def uploadFile(fname, uri, dbname): print 'Upload contents of %s to %s/%s' % (fname, uri, dbname) # Connect to the database theServer = Server(uri) db = theServer.get_or_create_db(dbname) # Loop on file for upload reader = DictReader(open(fname, 'rU'), dialect = 'excel') # For bulk upload docs = list() checkpoint = 100 for doc in reader: # Convert strings that are really numbers into ints and floats newdoc = parseDoc(doc) # Check if doc already exists in the DB # If it already exists, update it #if db.doc_exist(newdoc.get('_id')): # newdoc['_rev'] = db.get_rev(newdoc.get('_id')) docs.append(newdoc) if len(docs) % checkpoint == 0: docs = upload(db, docs) # Upload the lasr batch docs = upload(db, docs)
def __init__(self, settings, crm, client_doc, invoice_cycle): self.invoice_template_dir = settings.invoice_template_dir self.output_dir = settings.invoice_output_dir self.tax = float(settings.invoice_tax) self.jinja_env = Environment( loader=FileSystemLoader(self.invoice_template_dir) ) self.jinja_env.filters['nl2br'] = filters.nl2br self.jinja_env.filters['format_date'] = filters.format_date self.crm = crm self.client_id = client_doc['_id'] try: self.extcrm_id = client_doc['extcrm_id'] except KeyError: print(client_doc) raise KeyError self.client_doc = client_doc self.settings = settings self.invoice_cycle = invoice_cycle self.invoice_nr = invoice_cycle.current_nr server = Server(self.settings.couchdb_uri) self.db = server.get_db(self.settings.couchdb_db) if not Invoice.service_definitons: Invoice.load_service_definitions(self.db) self.setup_invoice()
class CouchDBServer(object): def __init__(self): self.__get_server_uri() self.__authenticate() self.__connect() def __get_server_uri(self): couchdb_port = config.couchdb.port if config.couchdb.protocol == 'http' else config.couchdb.ssl_port self.__couchdb_uri = "%s://%s:%s" % (config.couchdb.protocol, config.couchdb.host, couchdb_port) def __authenticate(self): user, passwd = config.couchdb.user, config.couchdb.password if all((user, passwd)): auth = restkit.BasicAuth(user, passwd) self.__auth_resource = CouchdbResource(filters=[auth]) else: self.__auth_resource = None def __connect(self): self.__server = Server(uri=self.__couchdb_uri, resource_instance=self.__auth_resource) def list_workspaces(self): return filter(is_usable_workspace, self.__server.all_dbs()) def get_workspace_handler(self, ws_name): return self.__server.get_db(ws_name) def get_or_create_db(self, ws_name): return self.__server.get_or_create_db(ws_name)
def __init__(self, databases): """ initialize couchdbkit handler with COUCHDB_DATABASES settings """ self.__dict__ = self.__shared_state__ # create databases sessions for app_name, uri in databases: try: if isinstance(uri, tuple): # case when you want to specify server uri # and database name specifically. usefull # when you proxy couchdb on some path server_uri, dbname = uri else: server_uri, dbname = uri.rsplit("/", 1) except ValueError: raise ValueError("couchdb uri [%s:%s] invalid" % ( app_name, uri)) res = CouchdbResource(server_uri, timeout=COUCHDB_TIMEOUT) server = Server(server_uri, resource_instance=res) app_label = app_name.split('.')[-1] self._databases[app_label] = server.get_or_create_db(dbname)
def main(views_directory, server_uri): directory = os.path.abspath(views_directory) server = Server(server_uri) db = server.get_or_create_db("v1") loader = FileSystemDocsLoader(directory) loader.sync(db, debug=True, verbose=True)
def handle(self, *args, **options): quiet = options.get('quiet', False) prod_indexes = options.get('prod-indexes', False) docs_root = os.path.normpath(settings.DOCUMENT_ROOT) if not quiet: self.stdout.write('Cleaning UP stored files in DOCUMENT_ROOT: %s \n' % docs_root) shutil.rmtree(docs_root) os.makedirs(docs_root) if not quiet: self.stdout.write('done\n') if not quiet: self.stdout.write('Deleting CouchDB debug mode databases.\n') databases = settings.COUCHDB_DATABASES server = Server() try: for database in databases: dbname = database[0] + '_test' if not quiet: self.stdout.write('Deleting DB: %s\n' % dbname) server.delete_db(dbname) if not quiet: self.stdout.write('done\n') except Exception, e: if not quiet: self.stdout.write('Failed to delete debug databases in CouchDB: %s ' % e) pass
def __init__(self, uri): self._last_seq_ack = 0 getLogger(self).debug("Initializing CouchDBManager for url [%s]" % uri) self._lostConnection = False self.__uri = uri self.__dbs = {} self.__seq_nums = {} self.__serv = NoConectionServer() self.mutex = threading.Lock() self._available = False #setting the doc types to load from couch def get_types(subclasses): if len(subclasses): head = subclasses[0] tail = [] if len(subclasses[1:]): tail = subclasses[1:] return get_types(head.__subclasses__()) + [head.class_signature] + get_types(tail) return [] self._model_object_types = get_types([ModelObject]) try: if uri is not None: self.testCouchUrl(uri) url = urlparse(uri) getLogger(self).debug("Setting user,pass %s %s" % (url.username, url.password)) self.__serv = Server(uri=uri) #print dir(self.__serv) self.__serv.resource_class.credentials = (url.username, url.password) self._available = True except: getLogger(self).warn("No route to couchdb server on: %s" % uri) getLogger(self).debug(traceback.format_exc())
def main(): #arguments parser parser = argparse.ArgumentParser(prog='pushCwe', epilog="Example: ./%(prog)s.py") parser.add_argument('-c', '--couchdburi', action='store', type=str, dest='couchdb',default="http://127.0.0.1:5984", help='Couchdb URL (default http://127.0.0.1:5984)') #arguments put in variables args = parser.parse_args() #default value from ENV COUCHDB couchdb = os.environ.get('COUCHDB') #Else from argument if not couchdb: couchdb = args.couchdb __serv = Server(uri = couchdb) # reports = os.path.join(os.getcwd(), "views", "reports") workspace = __serv.get_or_create_db("cwe") # designer.push(reports, workspace, atomic = False) with open('data/cwe.csv', 'r') as csvfile: cwereader = csv.reader(csvfile, delimiter=',') header = cwereader.next() for cwe in cwereader: cwe_doc = dict(zip(header, cwe)) workspace.save_doc(cwe_doc)
def run(self): crm = get_crm_module(self.settings) server = Server(self.settings.couchdb_uri) db = server.get_db(self.settings.couchdb_db) invoice_cycle = InvoiceCycle(self.settings.invoice_nr) clients = db.view( self._cmd("client_is_billable"), include_docs=True ) counter = 0 for client in clients: if not self.settings.limit_client_id or client['id'] == self.settings.limit_client_id: # print(client['doc']['name']) invoice = Invoice( self.settings, crm=crm, client_doc=client['doc'], invoice_cycle=invoice_cycle ) invoice_start_date = min( [d['start_date'] for d in invoice.doc['services'].itervalues()] ) if invoice_start_date < invoice_cycle.doc['end_date']: invoice.render_pdf() invoice_cycle.add_invoice(invoice) counter += 1 print(".", end="") if self.settings.max != 0 and counter >= self.settings.max: break print("\n", "total", invoice_cycle.get_total())
def __init__(self, settings, crm, client_doc, invoice_cycle): self.invoice_template_dir = settings.invoice_template_dir self.output_dir = settings.invoice_output_dir self.tax = float(settings.invoice_tax) self.home_country = settings.invoice_home_country if hasattr(settings, 'invoice_home_country', ) else None self.jinja_env = Environment( loader=FileSystemLoader(self.invoice_template_dir) ) self.jinja_env.filters['nl2br'] = filters.nl2br self.jinja_env.filters['format_date'] = filters.format_date self.crm = crm self.client_id = client_doc['_id'] try: self.extcrm_id = client_doc['extcrm_id'] except KeyError: extcrm_id = client_doc['extcrm_id'] if 'extcrm_id' in client_doc else "None" print('\nNOT creating invoice missing extcrm_id:%s, %s' % (extcrm_id, client_doc['name']), client_doc) self.client_doc = None return self.client_doc = client_doc self.settings = settings self.invoice_cycle = invoice_cycle self.invoice_nr = invoice_cycle.current_nr server = Server(self.settings.couchdb_uri) self.db = server.get_db(self.settings.couchdb_db) if not Invoice.service_definitons: Invoice.load_service_definitions(self.db) self.setup_invoice()
def get_prod_db(source_uri): """ Get the production database object since we need to get some doc_ids from the prod database """ prod_db_name = source_uri.split('/')[-1] prod_server = Server(uri=source_uri[:-len(prod_db_name)]) prod_db = prod_server.get_db(prod_db_name) return prod_db
def create_db(): """create the db if it don't exist""" from couchdbkit import Server server = Server(settings.SERVER_URI) try: db = server.create_db(settings.DATABASE_NAME) except: pass
def _list_couch_docs(self, db_name="dmscouch_test"): """Downloads all the documents that are currently in CouchDB now""" docs = {} server = Server() db = server.get_or_create_db(db_name) r = db.view("dmscouch/all", include_docs=True) for row in r: docs[row["doc"]["_id"]] = row["doc"] return docs
def setUp(self): self.settings = { "couchdb_uri": "http://*****:*****@cdb:5984", "couchdb_db": "desk_tester", } s = Server(self.settings["couchdb_uri"]) self.s = s s.create_db(self.settings['couchdb_db']) self.up = CouchdbUploader(path=os.path.dirname(__file__), auth=('admin', 'admin'), **self.settings)
def __init__(self, server_uri): server_uri, db_name, docid = parse_uri(server_uri) if "@" in server_uri: username, password, server_uri = parse_auth(server_uri) self.server = Server(server_uri) self.server.res.add_authorization(httpc.BasicAuth((uri.username, uri.password))) else: self.server = Server(server_uri) self.db = self.server.get_or_create_db("couchdbproxy")
def init_db(uri, dbname, main_db=True): """Returns a db object and syncs the design documents on demand. If main_db is set to true then all models will use that one by default. """ server = Server(uri) db = server.get_or_create_db(dbname) if main_db: Document.set_db(db) return db
def info(logger): s = Server() node_hello = s.info() logger.info('== Configuration ==') logger.info('CouchDB: %s ===' % node_hello) logger.info('Target hosts: %s' % NODE_LIST) logger.info('This host''s IP: %s', MY_IP) logger.info('CouchDB expected at port %s' % COUCHDB_PORT) logger.info('Tunnel user for SSH: %s' % TUNNEL_USER) logger.info('Monitoring interval for connections: %s seconds.' % MONITOR_INTERVAL) logger.info('Log path: %s' % LOG_PATH)
def setup(dbstring, path): server_uri, db_name, docid = parse_uri(dbstring) if "@" in server_uri: username, password, server_uri = parse_auth(server_uri) server = Server(server_uri) server.res.add_authorization(httpc.BasicAuth((uri.username, uri.password))) else: server = Server(server_uri) db = server.get_or_create_db("couchdbproxy") loader = FileSystemDocsLoader(path) loader.sync(db, verbose=True)
def get_detailed_status(options): total_tests = 0 total_errors = 0 server = Server(options.node) db = server.get_db(options.database) doc = get_build_doc(db, options.build) doc_content = db.open_doc(doc['_id']) tests_list = doc_content['_attachments'].keys() # TODO: Check for number of tests that ran print "List of tests against %s are %s " % (options.build, tests_list) failed_tests = [] # Data struct to store information per test_class (attachment) # {test_name:{tests:<int>, errros:<int>, time:<float>}} test_data = {} for attachment, value in doc_content['_attachments'].items(): errors_count = 0 tests_count = 0 print "Fetching attachment %s " % attachment file = db.fetch_attachment(doc, attachment) file = file.encode('ascii', 'ignore') xmldoc = etree.parse(StringIO.StringIO(file)) root = xmldoc.getroot() for child in root: attributes = child.attrib for childish in child: # To print the error #print childish.text failed_tests.append(attributes.get('name')) attributes = root.attrib # To get root element attributes #print attributes name = attributes.get('name') errors_count = int(attributes.get('errors')) tests_count = int(attributes.get('tests')) total_tests += tests_count total_errors += errors_count print "Testname: %s, passed: %s, failed: %s" % (name, tests_count-errors_count, errors_count) test_data[name] = {} test_data[name]['tests'] = tests_count test_data[name]['errors'] = errors_count test_data[name]['time'] = float(attributes.get('time')) #print test_data text = "Passed %s out of %s tests on %s build\n" % (total_tests-total_errors, total_tests, options.build) num_failed = len(failed_tests) if num_failed > 0: text += "\nList of Failed tests:\n" for i in range(num_failed): text += "%s: %s\n" % (i+1, failed_tests[i]) print text return test_data
def _list_couch_docs(self, db_name='dmscouch_test'): """Downloads all the documents that are currently in CouchDB now""" docs = {} server = Server() db = server.get_or_create_db(db_name) r = db.view( 'dmscouch/all', include_docs=True, ) for row in r: docs[row['doc']['_id']] = row['doc'] return docs
def upload_file(fname, uri, dbname): print 'Upload contents of %s to %s/%s' % (fname, uri, dbname) theServer = Server(uri, filters=[BasicAuth(cloudant_username, cloudant_password)]) db = theServer.get_or_create_db(dbname) reader = DictReader(open(fname, 'rU'), dialect = 'excel') docs = list() checkpoint = 100 for doc in reader: newdoc = parse_doc(doc) docs.append(newdoc) if len(docs)%checkpoint==0: docs = upload(db,docs) docs = upload(db,docs)
def push_cwe(couchdb_url, filename): __serv = Server(uri=couchdb_url) workspace = __serv.get_or_create_db("cwe") with open(filename, 'r') as csvfile: cwereader = csv.reader(csvfile, delimiter=',') header = cwereader.next() print "[*] Beginning upload" for cwe in cwereader: cwe_doc = dict(zip(header, cwe)) workspace.save_doc(cwe_doc) print "[*] Upload finished"
def process_couchdb_changes(): server = Server(settings.COUCHDB_SERVER) db = server.get_or_create_db("openelm") consumer = Consumer(db) sequence = SyncSequenceCache.objects.get(pk=1) changes = consumer.fetch(filter="record/new_records", since=sequence.last_sequence_id) if changes: for change in changes["results"]: record_id = change["id"] copy_photo_for_record.delay(record_id) send_new_record_email.delay(record_id) sequence.last_sequence_id = changes["last_seq"] sequence.save()
def before_request(): """Make sure we are connected to the database for each request.""" username = app.config.get('COUCHDB_SERVER_USERNAME') password = app.config.get('COUCHDB_SERVER_PASSWORD') if username and password: server = Server(app.config['COUCHDB_SERVER'], filters=[BasicAuth(username, password)]) else: server = Server(app.config['COUCHDB_SERVER']) # create databases g.mdb = server.get_or_create_db(app.config['COUCHDB_MESSAGES_DB']) Message.set_db(g.mdb)
def test_create_and_get_db(self): couch_manager = CouchDbManager(uri=CONF.getCouchURI()) couch_manager.createDb(self.dbname) self.assertNotEquals( couch_manager.getDb(self.dbname), None, "Db %s shouldn't be None" % self.dbname) server = Server(uri=CONF.getCouchURI()) self.assertIn( self.dbname, server.all_dbs(), "Db %s should be in the db list" % self.dbname)
def __getattr__( self, name ): if not self._settings: self._load() if name == 'db' or name == 'connection': if 'db' not in self._settings: p = urlparse.urlparse(self._settings['db_url']) database_uri= "{0.scheme}://{0.netloc}".format( p ) database_name= p.path connection = Server(database_uri) db = connection.get_or_create_db(database_name) self._settings['connection']= connection self._settings['db']= db return self._settings[name]
def __init__(self, database_uri=None, dbname=None): ''' Create a Server object, initiate connection and set up the databse URI (Not particulary in that order :-) Oh and also sync up the views from the on disk storage onto CouchDB so we can use them for our maintainance operations. ''' self.logger = configuration.get_logger(logging_instance=conflictlogging, system_name='dbmaintainer') configuration.info(self.logger) self.logger.info('Database maintainer starting.') if not database_uri: database_uri = "http://127.0.0.1:5984" if not dbname: dbname = 'session_store' self.dbname = dbname self.db_uri = database_uri self.dbserver = Server(self.db_uri) loader = FileSystemDocsLoader('/root/relaxession/_design/') try: self.db = self.dbserver.get_or_create_db(self.dbname) loader.sync(self.db, verbose=True) except Exception as e: self.logger.info('Init error: %s' % e) sys.exit(1) self.db.res.put('/_revs_limit',str(configuration.REVS_LIMIT))
def __init__(self, uri): self._last_seq_ack = 0 model.api.log("Initializing CouchDBManager for url [%s]" % uri) self._lostConnection = False self.__uri = uri self.__dbs = {} self.__seq_nums = {} self.__serv = NoConectionServer() self.mutex = threading.Lock() self._available = False try: self.testCouchUrl(uri) self.__serv = Server(uri = uri) self._available = True except: model.api.log("No route to couchdb server on: %s" % uri)
def __init__(self, serverURL, dbname, username, pwd): self.serverURL = serverURL self.dbname = dbname self.username = username self.pwd = pwd self.server = Server(self.serverURL, filters=[BasicAuth(self.username, self.pwd)]) set_logging("info") # suppress DEBUG output of the couchdbkit/restkit
def __init__(self,config): try: # First figure out which document template we are using here. if config.get_param('template') not in ['pdst','tslp','random']: raise Exception('Template parameter not recognized. Try: pdst tslp random') self.server = Server(uri='http://' + \ config.get_param('server') + \ ':' + config.get_param('port')) self.db = self.server.get_or_create_db(config.get_param('db')) self. docu_class = None if config.get_param('template') == 'tslp': self.docu_class = TSLPDocument if config.get_param('template') == 'pdst': self.docu_class = PDSTDocument elif config.get_param('template') == 'random': self.docu_class = DemoDocument self.docu_class.set_db(self.db) # OK, if that all worked, consider the object initialized wrt couch, # and store the other params we need to iterate. self.maxcounts = int(config.get_param('total')) self.increment = int(config.get_param('increment')) self.verbose = config.get_param('verbose') except Exception, msg: raise Exception(msg)
def main(*argv, **kwargs): s = Server(argv[0]) db = s[argv[1]] vr = db.view('proc/proc1', reduce=False, **kwargs) scriptDir = argv[2] script = '$KDATA_ROOT/lib/KDataPy/scripts/dataprocess/runProc1.py' scriptOut = os.path.join(scriptDir, 'qsubout') scriptErr = os.path.join(scriptDir, 'qsubout') maxDocs = 30 #for process 1, we can submit up to 30 files to be processed #by a particular batch job. This is because this is a relatively fast process #to convert a Samba file to a KData file and we want to play nice with #the batch system in Lyon docids = [] for row in vr: doc = db[row['id']] doc['status'] = 'proc1 queued' db.save_doc(doc) docids.append(row['id']) if len(docids) == maxDocs: docids = submitBatchJob(scriptOut, scriptErr, script, argv[0], argv[1], docids) #don't forget last chunk of data files docids = submitBatchJob(scriptOut, scriptErr, script, argv[0], argv[1], docids)
def submitBatchJob(scriptOut, scriptErr, script, server, dbname, docids): if len(docids) == 0: return [] command = 'qsub -P P_edelweis -o %s -e %s -l sps=1 -l vmem=1G -l fsize=1024M %s %s %s %s' % ( scriptOut, scriptErr, script, server, dbname, ' '.join(docids)) proc = subprocess.Popen( command, shell=True, stdout=subprocess.PIPE, ) val = proc.communicate()[0] if val != '': print val #now we need to update the #database documents with this batch job submission #metadata s = Server(server) db = s[dbname] docs = [] for anId in docids: doc = db[anId] if doc.has_key('batchJob') == False: doc['batchJob'] = [] jobStuff = {} jobStuff['number'] = int(val.split(' ')[2]) jobStuff['script'] = script jobStuff['stdout'] = os.path.join( scriptOut, os.path.basename(script)) + '.o' + str( jobStuff['number']) jobStuff['stderr'] = os.path.join( scriptErr, os.path.basename(script)) + '.e' + str( jobStuff['number']) jobStuff['command'] = command jobStuff['type'] = 'proc1' jobStuff['message'] = val jobStuff['date'] = str(datetime.datetime.now()) doc['batchJob'].append(jobStuff) proc = {} proc['batchjob'] = jobStuff['number'] doc['proc1'] = proc docs.append(doc) db.bulk_save(docs) return [] #return an empty list