def uploadFile(fname, uri, dbname):
print 'Upload contents of %s to %s/%s' % (fname, uri, dbname)
# Connect to the database
theServer = Server(uri)
db = theServer.get_or_create_db(dbname)
# Loop on file for upload
reader = DictReader(open(fname, 'rU'), dialect = 'excel')
# For bulk upload
docs = list()
checkpoint = 100
for doc in reader:
# Convert strings that are really numbers into ints and floats
newdoc = parseDoc(doc)
# Check if doc already exists in the DB
# If it already exists, update it
#if db.doc_exist(newdoc.get('_id')):
# newdoc['_rev'] = db.get_rev(newdoc.get('_id'))
docs.append(newdoc)
if len(docs) % checkpoint == 0:
docs = upload(db, docs)
# Upload the lasr batch
docs = upload(db, docs)
def __init__(self, settings, crm, client_doc, invoice_cycle):
self.invoice_template_dir = settings.invoice_template_dir
self.output_dir = settings.invoice_output_dir
self.tax = float(settings.invoice_tax)
self.jinja_env = Environment(
loader=FileSystemLoader(self.invoice_template_dir)
)
self.jinja_env.filters['nl2br'] = filters.nl2br
self.jinja_env.filters['format_date'] = filters.format_date
self.crm = crm
self.client_id = client_doc['_id']
try:
self.extcrm_id = client_doc['extcrm_id']
except KeyError:
print(client_doc)
raise KeyError
self.client_doc = client_doc
self.settings = settings
self.invoice_cycle = invoice_cycle
self.invoice_nr = invoice_cycle.current_nr
server = Server(self.settings.couchdb_uri)
self.db = server.get_db(self.settings.couchdb_db)
if not Invoice.service_definitons:
Invoice.load_service_definitions(self.db)
self.setup_invoice()
class CouchDBServer(object):
def __init__(self):
self.__get_server_uri()
self.__authenticate()
self.__connect()
def __get_server_uri(self):
couchdb_port = config.couchdb.port if config.couchdb.protocol == 'http' else config.couchdb.ssl_port
self.__couchdb_uri = "%s://%s:%s" % (config.couchdb.protocol, config.couchdb.host, couchdb_port)
def __authenticate(self):
user, passwd = config.couchdb.user, config.couchdb.password
if all((user, passwd)):
auth = restkit.BasicAuth(user, passwd)
self.__auth_resource = CouchdbResource(filters=[auth])
else:
self.__auth_resource = None
def __connect(self):
self.__server = Server(uri=self.__couchdb_uri, resource_instance=self.__auth_resource)
def list_workspaces(self):
return filter(is_usable_workspace, self.__server.all_dbs())
def get_workspace_handler(self, ws_name):
return self.__server.get_db(ws_name)
def get_or_create_db(self, ws_name):
return self.__server.get_or_create_db(ws_name)
def __init__(self, databases):
""" initialize couchdbkit handler with COUCHDB_DATABASES
settings """
self.__dict__ = self.__shared_state__
# create databases sessions
for app_name, uri in databases:
try:
if isinstance(uri, tuple):
# case when you want to specify server uri
# and database name specifically. usefull
# when you proxy couchdb on some path
server_uri, dbname = uri
else:
server_uri, dbname = uri.rsplit("/", 1)
except ValueError:
raise ValueError("couchdb uri [%s:%s] invalid" % (
app_name, uri))
res = CouchdbResource(server_uri, timeout=COUCHDB_TIMEOUT)
server = Server(server_uri, resource_instance=res)
app_label = app_name.split('.')[-1]
self._databases[app_label] = server.get_or_create_db(dbname)
def main(views_directory, server_uri):
directory = os.path.abspath(views_directory)
server = Server(server_uri)
db = server.get_or_create_db("v1")
loader = FileSystemDocsLoader(directory)
loader.sync(db, debug=True, verbose=True)
def handle(self, *args, **options):
quiet = options.get('quiet', False)
prod_indexes = options.get('prod-indexes', False)
docs_root = os.path.normpath(settings.DOCUMENT_ROOT)
if not quiet:
self.stdout.write('Cleaning UP stored files in DOCUMENT_ROOT: %s \n' % docs_root)
shutil.rmtree(docs_root)
os.makedirs(docs_root)
if not quiet:
self.stdout.write('done\n')
if not quiet:
self.stdout.write('Deleting CouchDB debug mode databases.\n')
databases = settings.COUCHDB_DATABASES
server = Server()
try:
for database in databases:
dbname = database[0] + '_test'
if not quiet:
self.stdout.write('Deleting DB: %s\n' % dbname)
server.delete_db(dbname)
if not quiet:
self.stdout.write('done\n')
except Exception, e:
if not quiet:
self.stdout.write('Failed to delete debug databases in CouchDB: %s ' % e)
pass
def __init__(self, uri):
self._last_seq_ack = 0
getLogger(self).debug("Initializing CouchDBManager for url [%s]" % uri)
self._lostConnection = False
self.__uri = uri
self.__dbs = {}
self.__seq_nums = {}
self.__serv = NoConectionServer()
self.mutex = threading.Lock()
self._available = False
#setting the doc types to load from couch
def get_types(subclasses):
if len(subclasses):
head = subclasses[0]
tail = []
if len(subclasses[1:]):
tail = subclasses[1:]
return get_types(head.__subclasses__()) + [head.class_signature] + get_types(tail)
return []
self._model_object_types = get_types([ModelObject])
try:
if uri is not None:
self.testCouchUrl(uri)
url = urlparse(uri)
getLogger(self).debug("Setting user,pass %s %s" % (url.username, url.password))
self.__serv = Server(uri=uri)
#print dir(self.__serv)
self.__serv.resource_class.credentials = (url.username, url.password)
self._available = True
except:
getLogger(self).warn("No route to couchdb server on: %s" % uri)
getLogger(self).debug(traceback.format_exc())
def main():
#arguments parser
parser = argparse.ArgumentParser(prog='pushCwe', epilog="Example: ./%(prog)s.py")
parser.add_argument('-c', '--couchdburi', action='store', type=str,
dest='couchdb',default="http://127.0.0.1:5984",
help='Couchdb URL (default http://127.0.0.1:5984)')
#arguments put in variables
args = parser.parse_args()
#default value from ENV COUCHDB
couchdb = os.environ.get('COUCHDB')
#Else from argument
if not couchdb:
couchdb = args.couchdb
__serv = Server(uri = couchdb)
# reports = os.path.join(os.getcwd(), "views", "reports")
workspace = __serv.get_or_create_db("cwe")
# designer.push(reports, workspace, atomic = False)
with open('data/cwe.csv', 'r') as csvfile:
cwereader = csv.reader(csvfile, delimiter=',')
header = cwereader.next()
for cwe in cwereader:
cwe_doc = dict(zip(header, cwe))
workspace.save_doc(cwe_doc)
def run(self):
crm = get_crm_module(self.settings)
server = Server(self.settings.couchdb_uri)
db = server.get_db(self.settings.couchdb_db)
invoice_cycle = InvoiceCycle(self.settings.invoice_nr)
clients = db.view(
self._cmd("client_is_billable"), include_docs=True
)
counter = 0
for client in clients:
if not self.settings.limit_client_id or client['id'] == self.settings.limit_client_id:
# print(client['doc']['name'])
invoice = Invoice(
self.settings, crm=crm,
client_doc=client['doc'],
invoice_cycle=invoice_cycle
)
invoice_start_date = min(
[d['start_date'] for d in invoice.doc['services'].itervalues()]
)
if invoice_start_date < invoice_cycle.doc['end_date']:
invoice.render_pdf()
invoice_cycle.add_invoice(invoice)
counter += 1
print(".", end="")
if self.settings.max != 0 and counter >= self.settings.max:
break
print("\n", "total", invoice_cycle.get_total())
def __init__(self, settings, crm, client_doc, invoice_cycle):
self.invoice_template_dir = settings.invoice_template_dir
self.output_dir = settings.invoice_output_dir
self.tax = float(settings.invoice_tax)
self.home_country = settings.invoice_home_country if hasattr(settings, 'invoice_home_country', ) else None
self.jinja_env = Environment(
loader=FileSystemLoader(self.invoice_template_dir)
)
self.jinja_env.filters['nl2br'] = filters.nl2br
self.jinja_env.filters['format_date'] = filters.format_date
self.crm = crm
self.client_id = client_doc['_id']
try:
self.extcrm_id = client_doc['extcrm_id']
except KeyError:
extcrm_id = client_doc['extcrm_id'] if 'extcrm_id' in client_doc else "None"
print('\nNOT creating invoice missing extcrm_id:%s, %s' % (extcrm_id, client_doc['name']), client_doc)
self.client_doc = None
return
self.client_doc = client_doc
self.settings = settings
self.invoice_cycle = invoice_cycle
self.invoice_nr = invoice_cycle.current_nr
server = Server(self.settings.couchdb_uri)
self.db = server.get_db(self.settings.couchdb_db)
if not Invoice.service_definitons:
Invoice.load_service_definitions(self.db)
self.setup_invoice()
def get_prod_db(source_uri):
"""
Get the production database object since we need to get some doc_ids from the prod database
"""
prod_db_name = source_uri.split('/')[-1]
prod_server = Server(uri=source_uri[:-len(prod_db_name)])
prod_db = prod_server.get_db(prod_db_name)
return prod_db
def create_db():
"""create the db if it don't exist"""
from couchdbkit import Server
server = Server(settings.SERVER_URI)
try:
db = server.create_db(settings.DATABASE_NAME)
except:
pass
def _list_couch_docs(self, db_name="dmscouch_test"):
"""Downloads all the documents that are currently in CouchDB now"""
docs = {}
server = Server()
db = server.get_or_create_db(db_name)
r = db.view("dmscouch/all", include_docs=True)
for row in r:
docs[row["doc"]["_id"]] = row["doc"]
return docs
def setUp(self):
self.settings = {
"couchdb_uri": "http://*****:*****@cdb:5984",
"couchdb_db": "desk_tester",
}
s = Server(self.settings["couchdb_uri"])
self.s = s
s.create_db(self.settings['couchdb_db'])
self.up = CouchdbUploader(path=os.path.dirname(__file__), auth=('admin', 'admin'), **self.settings)
def __init__(self, server_uri):
server_uri, db_name, docid = parse_uri(server_uri)
if "@" in server_uri:
username, password, server_uri = parse_auth(server_uri)
self.server = Server(server_uri)
self.server.res.add_authorization(httpc.BasicAuth((uri.username, uri.password)))
else:
self.server = Server(server_uri)
self.db = self.server.get_or_create_db("couchdbproxy")
def init_db(uri, dbname, main_db=True):
"""Returns a db object and syncs the design documents on demand.
If main_db is set to true then all models will use that one by default.
"""
server = Server(uri)
db = server.get_or_create_db(dbname)
if main_db:
Document.set_db(db)
return db
def info(logger):
s = Server()
node_hello = s.info()
logger.info('== Configuration ==')
logger.info('CouchDB: %s ===' % node_hello)
logger.info('Target hosts: %s' % NODE_LIST)
logger.info('This host''s IP: %s', MY_IP)
logger.info('CouchDB expected at port %s' % COUCHDB_PORT)
logger.info('Tunnel user for SSH: %s' % TUNNEL_USER)
logger.info('Monitoring interval for connections: %s seconds.' % MONITOR_INTERVAL)
logger.info('Log path: %s' % LOG_PATH)
def setup(dbstring, path):
server_uri, db_name, docid = parse_uri(dbstring)
if "@" in server_uri:
username, password, server_uri = parse_auth(server_uri)
server = Server(server_uri)
server.res.add_authorization(httpc.BasicAuth((uri.username, uri.password)))
else:
server = Server(server_uri)
db = server.get_or_create_db("couchdbproxy")
loader = FileSystemDocsLoader(path)
loader.sync(db, verbose=True)
def get_detailed_status(options):
total_tests = 0
total_errors = 0
server = Server(options.node)
db = server.get_db(options.database)
doc = get_build_doc(db, options.build)
doc_content = db.open_doc(doc['_id'])
tests_list = doc_content['_attachments'].keys()
# TODO: Check for number of tests that ran
print "List of tests against %s are %s " % (options.build, tests_list)
failed_tests = []
# Data struct to store information per test_class (attachment)
# {test_name:{tests:<int>, errros:<int>, time:<float>}}
test_data = {}
for attachment, value in doc_content['_attachments'].items():
errors_count = 0
tests_count = 0
print "Fetching attachment %s " % attachment
file = db.fetch_attachment(doc, attachment)
file = file.encode('ascii', 'ignore')
xmldoc = etree.parse(StringIO.StringIO(file))
root = xmldoc.getroot()
for child in root:
attributes = child.attrib
for childish in child:
# To print the error
#print childish.text
failed_tests.append(attributes.get('name'))
attributes = root.attrib
# To get root element attributes
#print attributes
name = attributes.get('name')
errors_count = int(attributes.get('errors'))
tests_count = int(attributes.get('tests'))
total_tests += tests_count
total_errors += errors_count
print "Testname: %s, passed: %s, failed: %s" % (name, tests_count-errors_count,
errors_count)
test_data[name] = {}
test_data[name]['tests'] = tests_count
test_data[name]['errors'] = errors_count
test_data[name]['time'] = float(attributes.get('time'))
#print test_data
text = "Passed %s out of %s tests on %s build\n" % (total_tests-total_errors, total_tests,
options.build)
num_failed = len(failed_tests)
if num_failed > 0:
text += "\nList of Failed tests:\n"
for i in range(num_failed):
text += "%s: %s\n" % (i+1, failed_tests[i])
print text
return test_data
def _list_couch_docs(self, db_name='dmscouch_test'):
"""Downloads all the documents that are currently in CouchDB now"""
docs = {}
server = Server()
db = server.get_or_create_db(db_name)
r = db.view(
'dmscouch/all',
include_docs=True,
)
for row in r:
docs[row['doc']['_id']] = row['doc']
return docs
def upload_file(fname, uri, dbname): print 'Upload contents of %s to %s/%s' % (fname, uri, dbname) theServer = Server(uri, filters=[BasicAuth(cloudant_username, cloudant_password)]) db = theServer.get_or_create_db(dbname) reader = DictReader(open(fname, 'rU'), dialect = 'excel') docs = list() checkpoint = 100 for doc in reader: newdoc = parse_doc(doc) docs.append(newdoc) if len(docs)%checkpoint==0: docs = upload(db,docs) docs = upload(db,docs)
def push_cwe(couchdb_url, filename):
__serv = Server(uri=couchdb_url)
workspace = __serv.get_or_create_db("cwe")
with open(filename, 'r') as csvfile:
cwereader = csv.reader(csvfile, delimiter=',')
header = cwereader.next()
print "[*] Beginning upload"
for cwe in cwereader:
cwe_doc = dict(zip(header, cwe))
workspace.save_doc(cwe_doc)
print "[*] Upload finished"
def process_couchdb_changes():
server = Server(settings.COUCHDB_SERVER)
db = server.get_or_create_db("openelm")
consumer = Consumer(db)
sequence = SyncSequenceCache.objects.get(pk=1)
changes = consumer.fetch(filter="record/new_records", since=sequence.last_sequence_id)
if changes:
for change in changes["results"]:
record_id = change["id"]
copy_photo_for_record.delay(record_id)
send_new_record_email.delay(record_id)
sequence.last_sequence_id = changes["last_seq"]
sequence.save()
def before_request():
"""Make sure we are connected to the database for each request."""
username = app.config.get('COUCHDB_SERVER_USERNAME')
password = app.config.get('COUCHDB_SERVER_PASSWORD')
if username and password:
server = Server(app.config['COUCHDB_SERVER'],
filters=[BasicAuth(username, password)])
else:
server = Server(app.config['COUCHDB_SERVER'])
# create databases
g.mdb = server.get_or_create_db(app.config['COUCHDB_MESSAGES_DB'])
Message.set_db(g.mdb)
def test_create_and_get_db(self):
couch_manager = CouchDbManager(uri=CONF.getCouchURI())
couch_manager.createDb(self.dbname)
self.assertNotEquals(
couch_manager.getDb(self.dbname),
None,
"Db %s shouldn't be None" % self.dbname)
server = Server(uri=CONF.getCouchURI())
self.assertIn(
self.dbname,
server.all_dbs(),
"Db %s should be in the db list" % self.dbname)
def __getattr__( self, name ):
if not self._settings:
self._load()
if name == 'db' or name == 'connection':
if 'db' not in self._settings:
p = urlparse.urlparse(self._settings['db_url'])
database_uri= "{0.scheme}://{0.netloc}".format( p )
database_name= p.path
connection = Server(database_uri)
db = connection.get_or_create_db(database_name)
self._settings['connection']= connection
self._settings['db']= db
return self._settings[name]
def __init__(self, database_uri=None, dbname=None):
'''
Create a Server object, initiate connection and set up the databse URI
(Not particulary in that order :-)
Oh and also sync up the views from the on disk storage onto CouchDB
so we can use them for our maintainance operations.
'''
self.logger = configuration.get_logger(logging_instance=conflictlogging,
system_name='dbmaintainer')
configuration.info(self.logger)
self.logger.info('Database maintainer starting.')
if not database_uri:
database_uri = "http://127.0.0.1:5984"
if not dbname:
dbname = 'session_store'
self.dbname = dbname
self.db_uri = database_uri
self.dbserver = Server(self.db_uri)
loader = FileSystemDocsLoader('/root/relaxession/_design/')
try:
self.db = self.dbserver.get_or_create_db(self.dbname)
loader.sync(self.db, verbose=True)
except Exception as e:
self.logger.info('Init error: %s' % e)
sys.exit(1)
self.db.res.put('/_revs_limit',str(configuration.REVS_LIMIT))
def __init__(self, uri):
self._last_seq_ack = 0
model.api.log("Initializing CouchDBManager for url [%s]" % uri)
self._lostConnection = False
self.__uri = uri
self.__dbs = {}
self.__seq_nums = {}
self.__serv = NoConectionServer()
self.mutex = threading.Lock()
self._available = False
try:
self.testCouchUrl(uri)
self.__serv = Server(uri = uri)
self._available = True
except:
model.api.log("No route to couchdb server on: %s" % uri)
def __init__(self, serverURL, dbname, username, pwd):
self.serverURL = serverURL
self.dbname = dbname
self.username = username
self.pwd = pwd
self.server = Server(self.serverURL, filters=[BasicAuth(self.username, self.pwd)])
set_logging("info") # suppress DEBUG output of the couchdbkit/restkit
def __init__(self,config):
try:
# First figure out which document template we are using here.
if config.get_param('template') not in ['pdst','tslp','random']:
raise Exception('Template parameter not recognized. Try: pdst tslp random')
self.server = Server(uri='http://' + \
config.get_param('server') + \
':' + config.get_param('port'))
self.db = self.server.get_or_create_db(config.get_param('db'))
self. docu_class = None
if config.get_param('template') == 'tslp':
self.docu_class = TSLPDocument
if config.get_param('template') == 'pdst':
self.docu_class = PDSTDocument
elif config.get_param('template') == 'random':
self.docu_class = DemoDocument
self.docu_class.set_db(self.db)
# OK, if that all worked, consider the object initialized wrt couch,
# and store the other params we need to iterate.
self.maxcounts = int(config.get_param('total'))
self.increment = int(config.get_param('increment'))
self.verbose = config.get_param('verbose')
except Exception, msg:
raise Exception(msg)
def main(*argv, **kwargs):
s = Server(argv[0])
db = s[argv[1]]
vr = db.view('proc/proc1', reduce=False, **kwargs)
scriptDir = argv[2]
script = '$KDATA_ROOT/lib/KDataPy/scripts/dataprocess/runProc1.py'
scriptOut = os.path.join(scriptDir, 'qsubout')
scriptErr = os.path.join(scriptDir, 'qsubout')
maxDocs = 30 #for process 1, we can submit up to 30 files to be processed
#by a particular batch job. This is because this is a relatively fast process
#to convert a Samba file to a KData file and we want to play nice with
#the batch system in Lyon
docids = []
for row in vr:
doc = db[row['id']]
doc['status'] = 'proc1 queued'
db.save_doc(doc)
docids.append(row['id'])
if len(docids) == maxDocs:
docids = submitBatchJob(scriptOut, scriptErr, script, argv[0],
argv[1], docids)
#don't forget last chunk of data files
docids = submitBatchJob(scriptOut, scriptErr, script, argv[0], argv[1],
docids)
def submitBatchJob(scriptOut, scriptErr, script, server, dbname, docids):
if len(docids) == 0:
return []
command = 'qsub -P P_edelweis -o %s -e %s -l sps=1 -l vmem=1G -l fsize=1024M %s %s %s %s' % (
scriptOut, scriptErr, script, server, dbname, ' '.join(docids))
proc = subprocess.Popen(
command,
shell=True,
stdout=subprocess.PIPE,
)
val = proc.communicate()[0]
if val != '':
print val
#now we need to update the
#database documents with this batch job submission
#metadata
s = Server(server)
db = s[dbname]
docs = []
for anId in docids:
doc = db[anId]
if doc.has_key('batchJob') == False:
doc['batchJob'] = []
jobStuff = {}
jobStuff['number'] = int(val.split(' ')[2])
jobStuff['script'] = script
jobStuff['stdout'] = os.path.join(
scriptOut, os.path.basename(script)) + '.o' + str(
jobStuff['number'])
jobStuff['stderr'] = os.path.join(
scriptErr, os.path.basename(script)) + '.e' + str(
jobStuff['number'])
jobStuff['command'] = command
jobStuff['type'] = 'proc1'
jobStuff['message'] = val
jobStuff['date'] = str(datetime.datetime.now())
doc['batchJob'].append(jobStuff)
proc = {}
proc['batchjob'] = jobStuff['number']
doc['proc1'] = proc
docs.append(doc)
db.bulk_save(docs)
return [] #return an empty list
