def __init__(self, uri, couch_exception_callback): super(CouchDbManager, self).__init__() getLogger(self).debug( "Initializing CouchDBManager for url [%s]" % uri) self._lostConnection = False self.__uri = uri self._available = False self.couch_exception_callback = couch_exception_callback test_couch_thread = threading.Thread(target=self.continuosly_check_connection) test_couch_thread.daemon = True test_couch_thread.start() try: if uri is not None: self.testCouchUrl(uri) url = urlparse(uri) getLogger(self).debug( "Setting user,pass %s %s" % (url.username, url.password)) self.__serv = Server(uri=uri) self.__serv.resource_class.credentials = (url.username, url.password) self._available = True self.pushReports() self._loadDbs() except: getLogger(self).warn("No route to couchdb server on: %s" % uri) getLogger(self).debug(traceback.format_exc())
def __init__(self, databases): """ initialize couchdbkit handler with COUCHDB_DATABASES settings """ self.__dict__ = self.__shared_state__ # create databases sessions for app_name, uri in databases: try: if isinstance(uri, tuple): # case when you want to specify server uri # and database name specifically. usefull # when you proxy couchdb on some path server_uri, dbname = uri else: server_uri, dbname = uri.rsplit("/", 1) except ValueError: raise ValueError("couchdb uri [%s:%s] invalid" % ( app_name, uri)) res = CouchdbResource(server_uri, timeout=COUCHDB_TIMEOUT) server = Server(server_uri, resource_instance=res) app_label = app_name.split('.')[-1] self._databases[app_label] = (server, dbname)
def __init__(self, databases): """ initialize couchdbkit handler with COUCHDB_DATABASES settings """ self.__dict__ = self.__shared_state__ # Convert old style to new style if isinstance(databases, (list, tuple)): databases = dict((app_name, { 'URL': uri }) for app_name, uri in databases) # create databases sessions for app_name, app_setting in six.iteritems(databases): uri = app_setting['URL'] try: if isinstance(uri, (list, tuple)): # case when you want to specify server uri # and database name specifically. usefull # when you proxy couchdb on some path server_uri, dbname = uri else: server_uri, dbname = uri.rsplit("/", 1) except ValueError: raise ValueError("couchdb uri [%s:%s] invalid" % (app_name, uri)) server = Server(server_uri) app_label = app_name.split('.')[-1] self._databases[app_label] = (server, dbname)
def main(): #arguments parser parser = argparse.ArgumentParser(prog='pushCwe', epilog="Example: ./%(prog)s.py") parser.add_argument('-c', '--couchdburi', action='store', type=str, dest='couchdb', default="http://127.0.0.1:5984", help='Couchdb URL (default http://127.0.0.1:5984)') #arguments put in variables args = parser.parse_args() #default value from ENV COUCHDB couchdb = os.environ.get('COUCHDB') #Else from argument if not couchdb: couchdb = args.couchdb __serv = Server(uri=couchdb) workspace = __serv.get_or_create_db("cwe") with open('data/cwe.csv', 'r') as csvfile: cwereader = csv.reader(csvfile, delimiter=',') header = cwereader.next() for cwe in cwereader: cwe_doc = dict(zip(header, cwe)) workspace.save_doc(cwe_doc)
def main(*argv, **kwargs): s = Server(argv[0]) db = s[argv[1]] vr = db.view('proc/proc1', reduce=False, **kwargs) scriptDir = argv[2] script = '$KDATA_ROOT/lib/KDataPy/scripts/dataprocess/runProc1.py' scriptOut = os.path.join(scriptDir, 'qsubout') scriptErr = os.path.join(scriptDir, 'qsubout') maxDocs = 30 #for process 1, we can submit up to 30 files to be processed #by a particular batch job. This is because this is a relatively fast process #to convert a Samba file to a KData file and we want to play nice with #the batch system in Lyon docids = [] for row in vr: doc = db[row['id']] doc['status'] = 'proc1 queued' db.save_doc(doc) docids.append(row['id']) if len(docids) == maxDocs: docids = submitBatchJob(scriptOut, scriptErr, script, argv[0], argv[1], docids) #don't forget last chunk of data files docids = submitBatchJob(scriptOut, scriptErr, script, argv[0], argv[1], docids)
def __init__(self, serverURL, dbname, username, pwd): self.serverURL = serverURL self.dbname = dbname self.username = username self.pwd = pwd self.server = Server(self.serverURL, filters=[BasicAuth(self.username, self.pwd)])
def handle(self, *args, **options): quiet = options.get('quiet', False) prod_indexes = options.get('prod-indexes', False) docs_root = os.path.normpath(settings.DOCUMENT_ROOT) if not quiet: self.stdout.write( 'Cleaning UP stored files in DOCUMENT_ROOT: %s \n' % docs_root) shutil.rmtree(docs_root) os.makedirs(docs_root) if not quiet: self.stdout.write('done\n') if not quiet: self.stdout.write('Deleting CouchDB debug mode databases.\n') databases = settings.COUCHDB_DATABASES server = Server() try: for database in databases: dbname = database[0] + '_test' if not quiet: self.stdout.write('Deleting DB: %s\n' % dbname) server.delete_db(dbname) if not quiet: self.stdout.write('done\n') except Exception, e: if not quiet: self.stdout.write( 'Failed to delete debug databases in CouchDB: %s ' % e) pass
def main(db_name, db_user, db_pass, db_host, sequence_file, zone_dir, **tls_args): # Starting Sequence for change stream sequence = sequence_read(sequence_file) click.echo('Skipping %s changes.' % sequence) # CouchDB Connection tls_args['cert_reqs'] = ssl.CERT_REQUIRED tls_args['ssl_version'] = ssl.PROTOCOL_TLSv1_2 auth = CouchdbResource(filters=[BasicAuth(db_user, db_pass)], **tls_args) server = Server(uri=db_host, resource_instance=auth) db = server[db_name] if sequence == 0: click.echo('Fast track syncing all zones...') c = Consumer(db) result = c.fetch(descending=True, limit=1) # Fast track to this sequence sequence = result['last_seq'] # Go get all the current zones. zones = c.fetch() for zone in zones['results']: domain = zone['id'] try: doc = db.get(docid=domain) except ResourceNotFound, e: click.echo('%s not found (this is normal if the zone was deleted)' % domain) else: zone_update(domain, doc['data'], zone_dir) sequence_write(sequence_file, sequence) # Keep track of our sync point click.echo('Fast track syncing done')
def main(argv): """ Main method. This method performs the following tasks: 1. Parse command line arguments 2. Retrieve credentials and connect to Cloudant and WebHDFS 3. Connect to the Cloudant `_changes` feed for checkpointed document consumption 4. Process each change individually. 5. Upon exception throwing, store the latest checkpoint to local file and exit. """ #add options into the parser parser = configureOptions() (options, args) = parser.parse_args() checkRequiredArguments(options, parser) print options # configurations last_seq = options.last_seq #get credential perm_file = '%s/.clou' % os.environ['HOME'] creds = get_creds(perm_file) #connect to source database s = Server('https://%s:%s@%s' % (creds['cloudant_user'], creds['cloudant_pwd'], options.uri)) db = s[options.dbname] #print db.info() #connect to target hdfs cluster hdfs = PyWebHdfsClient(host=options.hdfs_host, port=options.hdfs_port, user_name=creds['hdfs_user']) hdfs.make_dir(options.hdfs_path) #and here we consume the cloudant `_changes` feed counter = 0 changestream = ChangesStream(db, include_docs=True, heartbeat=True, since=last_seq) for c in changestream: #print c try: if counter % 100 == 0: checkpoint(last_seq) seq = processChange(hdfs, c, options.hdfs_path) if seq: # protect against the last line being blank last_seq = seq counter += 1 except Exception: traceback.print_exc() checkpoint(last_seq) os._exit(1) checkpoint(last_seq)
def deleteFromView(server, databaseName, procname, **kwargs): ''' This script will remove a "procname" key from database run documents. You can give this script a single run number, or a range of runs numbers. (For now, it will remove the proc from ALL partition files within a single run. Additionally, you can specify options to the db.view('proc/daqdoc') call by including key=value pairs to the command line args kwargs should at least contain startkey and endkey ''' s = Server(server) db = s[databaseName] if kwargs['startkey'] == None or kwargs['endkey'] == None: print 'you should supply a range of runs to use with **kwargs["startkey"] and kwargs["endkey"]' return vr = db.view('proc/daqdoc', reduce = False, **kwargs) for row in vr: if row.has_key('doc'): doc = row['doc'] else: doc = db.get(row['id']) print 'checking', doc['_id'] if doc.has_key(procname): print 'deleting', procname, 'from', doc['_id'] del doc[procname] if procname == 'proc0': doc['status'] = 'closed' else: doc['status'] = 'good' db.save_doc(doc)
def main(*argv): ''' This script queries the edwdb database (on edwdbik.fzk.de) and returns the runs of a particular type. This script can also list the types of runs that are available Here are the examples for each one 1) ./listRunsByType.py "calibration gamma" 2) ./listRunsByType.py list ''' s = Server('https://edwdbik.fzk.de:6984') db = s['datadb'] if len(argv) == 0: print 'You need to supply an argument' help(main) return arg1 = argv[0] if arg1 == 'list': vr = db.view('header/runcondition', group_level=1) print 'Run Types' for row in vr: print row['key'][0] else: print 'looking for runs of type', arg1 vr = db.view('header/runcondition', endkey = [arg1,''], startkey = [arg1+'\ufff0', ''], group_level = 2, descending = True) for row in vr: print row['key'][1]
def get_prod_db(source_uri): """ Get the production database object since we need to get some doc_ids from the prod database """ prod_db_name = source_uri.split('/')[-1] prod_server = Server(uri=source_uri[:-len(prod_db_name)]) prod_db = prod_server.get_db(prod_db_name) return prod_db
def __init__(self, serverURL, dbname, username, pwd): self.serverURL = serverURL self.dbname = dbname self.username = username self.pwd = pwd self.server = Server(self.serverURL, filters=[BasicAuth(self.username, self.pwd)]) set_logging('info') # suppress DEBUG output of the couchdbkit/restkit
def setUpClass(cls): super(TestLogging, cls).setUpClass() cls.server = Server() cls.db = cls.server.create_db('couchdbkit_test') doc = {"name": "Yesterday's Heroes"} cls.db.save_doc(doc) cls.doc_id = doc["_id"] cls.log = CaptureLogOutput(mod.request_logger.name)
def prep_connection(self, uri, db_name): # create server object self.server = Server(uri=uri) # create database self.db = self.server.get_or_create_db(db_name) # associate local objects to the db Book.set_db(self.db)
def reconnect(self): ret_val = False ur = self.__uri if CouchdbManager.testCouch(ur): self.__serv = Server(uri=ur) self.__dbs.clear() self._lostConnection = False ret_val = True return ret_val
def test_create_and_get_db(self): couch_manager = CouchDbManager(uri=CONF.getCouchURI()) couch_manager.createDb(self.dbname) self.assertNotEquals(couch_manager.getDb(self.dbname), None, "Db %s shouldn't be None" % self.dbname) server = Server(uri=CONF.getCouchURI()) self.assertIn(self.dbname, server.all_dbs(), "Db %s should be in the db list" % self.dbname)
def main(*argv): ''' This script queries the edwdb database (on edwdbik.fzk.de) and returns information found in the Muon Veto DAQ Map documents. You can do three things with this script. 1. obtain the information for a particular muon module 2. list the available fields to choose from in the muon veto daq map 3. search for the muon module that matches a particular field. Here are the examples for each one 1) ./vetoDaqDb.py moduleinfo 3 2) ./vetoDaqDb.py list 3) ./vetoDaqDb.py "HV channel" 67 ''' s = Server('https://edwdbik.fzk.de:6984') db = s['muonvetohardwaremap'] if len(argv) == 0: print 'You need to supply some arguments' help(main) return arg0 = argv[0] if len(argv) > 1: value = formatvalue(argv[1]) #print 'Searching Muon Veto DAQ Map for', key, '=', value if arg0 == 'moduleinfo': vr = db.view('map/module', reduce=False, key=value) for row in vr: doc = db.get(row['id']) for k, val in doc.items(): if k != '_id' and k != '_rev': print k, val print '\n' elif arg0 == 'list': vr = db.view('map/keys', group=True) for row in vr: if row['key'] != '_id' and k['key'] != '_rev': print row['key'] else: vr = db.view('map/key_values', key=arg0) for row in vr: if row['value'] == value: doc = db.get(row['id']) print 'module:', doc['muonmodule'], 'end:', doc[ 'end'], ' date_valid', doc['date_valid']
def get_detailed_status(options): total_tests = 0 total_errors = 0 server = Server(options.node) db = server.get_db(options.database) doc = get_build_doc(db, options.build) doc_content = db.open_doc(doc['_id']) tests_list = doc_content['_attachments'].keys() # TODO: Check for number of tests that ran print "List of tests against %s are %s " % (options.build, tests_list) failed_tests = [] # Data struct to store information per test_class (attachment) # {test_name:{tests:<int>, errros:<int>, time:<float>}} test_data = {} for attachment, value in doc_content['_attachments'].items(): errors_count = 0 tests_count = 0 print "Fetching attachment %s " % attachment file = db.fetch_attachment(doc, attachment) file = file.encode('ascii', 'ignore') xmldoc = etree.parse(StringIO.StringIO(file)) root = xmldoc.getroot() for child in root: attributes = child.attrib for childish in child: # To print the error #print childish.text failed_tests.append(attributes.get('name')) attributes = root.attrib # To get root element attributes #print attributes name = attributes.get('name') errors_count = int(attributes.get('errors')) tests_count = int(attributes.get('tests')) total_tests += tests_count total_errors += errors_count print "Testname: %s, passed: %s, failed: %s" % ( name, tests_count - errors_count, errors_count) test_data[name] = {} test_data[name]['tests'] = tests_count test_data[name]['errors'] = errors_count test_data[name]['time'] = float(attributes.get('time')) #print test_data text = "Passed %s out of %s tests on %s build\n" % ( total_tests - total_errors, total_tests, options.build) num_failed = len(failed_tests) if num_failed > 0: text += "\nList of Failed tests:\n" for i in range(num_failed): text += "%s: %s\n" % (i + 1, failed_tests[i]) print text return test_data
def submitBatchJob(scriptOut, scriptErr, script, server, dbname, docids): if len(docids) == 0: return [] command = 'qsub -P P_edelweis -o %s -e %s -l sps=1 -l vmem=1G -l fsize=1024M %s %s %s %s' % ( scriptOut, scriptErr, script, server, dbname, ' '.join(docids)) proc = subprocess.Popen( command, shell=True, stdout=subprocess.PIPE, ) val = proc.communicate()[0] if val != '': print val #now we need to update the #database documents with this batch job submission #metadata s = Server(server) db = s[dbname] docs = [] for anId in docids: doc = db[anId] if doc.has_key('batchJob') == False: doc['batchJob'] = [] jobStuff = {} jobStuff['number'] = int(val.split(' ')[2]) jobStuff['script'] = script jobStuff['stdout'] = os.path.join( scriptOut, os.path.basename(script)) + '.o' + str( jobStuff['number']) jobStuff['stderr'] = os.path.join( scriptErr, os.path.basename(script)) + '.e' + str( jobStuff['number']) jobStuff['command'] = command jobStuff['type'] = 'proc1' jobStuff['message'] = val jobStuff['date'] = str(datetime.datetime.now()) doc['batchJob'].append(jobStuff) proc = {} proc['batchjob'] = jobStuff['number'] doc['proc1'] = proc docs.append(doc) db.bulk_save(docs) return [] #return an empty list
def _list_couch_docs(self, db_name='dmscouch_test'): """Downloads all the documents that are currently in CouchDB now""" docs = {} server = Server() db = server.get_or_create_db(db_name) r = db.view( 'dmscouch/all', include_docs=True, ) for row in r: docs[row['doc']['_id']] = row['doc'] return docs
def push_cwe(couchdb_url, filename): __serv = Server(uri=couchdb_url) workspace = __serv.get_or_create_db("cwe") with open(filename, 'r') as csvfile: cwereader = csv.reader(csvfile, delimiter=',') header = cwereader.next() print "[*] Beginning upload" for cwe in cwereader: cwe_doc = dict(zip(header, cwe)) workspace.save_doc(cwe_doc) print "[*] Upload finished"
def init_app(self, app): """Bind an app to a CouchDBKit instance and initialize the database. :param app: The application to which the CouchDBKit instance should be bound. """ self.app = app self.app.config.setdefault('COUCHDB_SERVER', 'http://localhost:5984/') self.app.config.setdefault('COUCHDB_DATABASE', None) self.app.config.setdefault('COUCHDB_KEEPALIVE', None) self.app.config.setdefault('COUCHDB_VIEWS', '_design') server_uri = app.config.get('COUCHDB_SERVER') pool_keepalive = app.config.get('COUCHDB_KEEPALIVE') if pool_keepalive is not None: mgr = Manager(max_conn=pool_keepalive) server = Server(server_uri, manager=mgr) else: server = Server(server_uri) self.server = server self.init_db()
def process_couchdb_changes(): server = Server(settings.COUCHDB_SERVER) db = server.get_or_create_db('openelm') consumer = Consumer(db) sequence = SyncSequenceCache.objects.get(pk=1) changes = consumer.fetch(filter='record/new_records', since=sequence.last_sequence_id) if changes: for change in changes['results']: record_id = change['id'] copy_photo_for_record.delay(record_id) send_new_record_email.delay(record_id) sequence.last_sequence_id = changes['last_seq'] sequence.save()
def main(): s = Server('http://*****:*****@127.0.0.1:5984') db = s['datadb'] c = Consumer(db) #start listening since = current update sequence. #callback function is run.main #heartbeat every minute to keep the connection alive. c.wait(callback, since=db.info()['update_seq'], filter='proc/newproc0', feed='continuous', heartbeat=60000)
def upload_file(fname, uri, dbname): print 'Upload contents of %s to %s/%s' % (fname, uri, dbname) theServer = Server( uri, filters=[BasicAuth(cloudant_username, cloudant_password)]) db = theServer.get_or_create_db(dbname) reader = DictReader(open(fname, 'rU'), dialect='excel') docs = list() checkpoint = 100 for doc in reader: newdoc = parse_doc(doc) docs.append(newdoc) if len(docs) % checkpoint == 0: docs = upload(db, docs) docs = upload(db, docs)
def sync_all_app(self, uri, dbname, views): ''' Create a database session for each databases, then start the syncing process. Databases are described by tuples containing the application name and the database URL in which views must be synchronized. @param uri: Uri of the couchdb server @param dbname: Database name @param views: Name of the views ''' for view in views: res = CouchdbResource(uri, timeout=COUCHDB_TIMEOUT) server = Server(uri, resource_instance=res) self.sync(server, dbname, view, views[view])
def __init__(self, databases): """ initialize couchdbkit handler with COUCHDB_DATABASES settings """ self.__dict__ = self.__shared_state__ # Convert old style to new style if isinstance(databases, (list, tuple)): databases = dict((app_name, { 'URL': uri }) for app_name, uri in databases) # create databases sessions for app_name, app_setting in databases.items(): uri = app_setting['URL'] # Do not send credentials when they are both None as admin party will give a 401 user = app_setting.get('USER') password = app_setting.get('PASSWORD') filters = [BasicAuth(user, password) ] if (user or password) is not None else [] try: if isinstance(uri, (list, tuple)): # case when you want to specify server uri # and database name specifically. usefull # when you proxy couchdb on some path server_uri, dbname = uri else: server_uri, dbname = uri.rsplit("/", 1) except ValueError: raise ValueError("couchdb uri [%s:%s] invalid" % (app_name, uri)) res = CouchdbResource(server_uri, timeout=COUCHDB_TIMEOUT, filters=filters) server = Server(server_uri, resource_instance=res) app_label = app_name.split('.')[-1] self._databases[app_label] = (server, dbname)
def __init__(self, uri): self._last_seq_ack = 0 model.api.log("Initializing CouchDBManager for url [%s]" % uri) self._lostConnection = False self.__uri = uri self.__dbs = {} self.__seq_nums = {} self.__serv = NoConectionServer() self.mutex = threading.Lock() self._available = False try: self.testCouchUrl(uri) url = urlparse(uri) print("Setting user,pass %s %s" % (url.username, url.password)) self.__serv = Server(uri=uri) #print dir(self.__serv) self.__serv.resource_class.credentials = (url.username, url.password) self._available = True except: model.api.log("No route to couchdb server on: %s" % uri) print(traceback.format_exc())
def __init__(self, uri): super(CouchDbManager, self).__init__() getLogger(self).debug( "Initializing CouchDBManager for url [%s]" % uri) self._lostConnection = False self.__uri = uri self.__serv = NoConectionServer() self._available = False try: if uri is not None: self.testCouchUrl(uri) url = urlparse(uri) getLogger(self).debug( "Setting user,pass %s %s" % (url.username, url.password)) self.__serv = Server(uri=uri) self.__serv.resource_class.credentials = (url.username, url.password) self._available = True self.pushReports() self._loadDbs() except: getLogger(self).warn("No route to couchdb server on: %s" % uri) getLogger(self).debug(traceback.format_exc())