def __init__(self, uri, couch_exception_callback):
super(CouchDbManager, self).__init__()
getLogger(self).debug(
"Initializing CouchDBManager for url [%s]" % uri)
self._lostConnection = False
self.__uri = uri
self._available = False
self.couch_exception_callback = couch_exception_callback
test_couch_thread = threading.Thread(target=self.continuosly_check_connection)
test_couch_thread.daemon = True
test_couch_thread.start()
try:
if uri is not None:
self.testCouchUrl(uri)
url = urlparse(uri)
getLogger(self).debug(
"Setting user,pass %s %s" % (url.username, url.password))
self.__serv = Server(uri=uri)
self.__serv.resource_class.credentials = (url.username, url.password)
self._available = True
self.pushReports()
self._loadDbs()
except:
getLogger(self).warn("No route to couchdb server on: %s" % uri)
getLogger(self).debug(traceback.format_exc())
def __init__(self, databases):
""" initialize couchdbkit handler with COUCHDB_DATABASES
settings """
self.__dict__ = self.__shared_state__
# create databases sessions
for app_name, uri in databases:
try:
if isinstance(uri, tuple):
# case when you want to specify server uri
# and database name specifically. usefull
# when you proxy couchdb on some path
server_uri, dbname = uri
else:
server_uri, dbname = uri.rsplit("/", 1)
except ValueError:
raise ValueError("couchdb uri [%s:%s] invalid" % (
app_name, uri))
res = CouchdbResource(server_uri, timeout=COUCHDB_TIMEOUT)
server = Server(server_uri, resource_instance=res)
app_label = app_name.split('.')[-1]
self._databases[app_label] = (server, dbname)
def __init__(self, databases):
""" initialize couchdbkit handler with COUCHDB_DATABASES
settings """
self.__dict__ = self.__shared_state__
# Convert old style to new style
if isinstance(databases, (list, tuple)):
databases = dict((app_name, {
'URL': uri
}) for app_name, uri in databases)
# create databases sessions
for app_name, app_setting in six.iteritems(databases):
uri = app_setting['URL']
try:
if isinstance(uri, (list, tuple)):
# case when you want to specify server uri
# and database name specifically. usefull
# when you proxy couchdb on some path
server_uri, dbname = uri
else:
server_uri, dbname = uri.rsplit("/", 1)
except ValueError:
raise ValueError("couchdb uri [%s:%s] invalid" %
(app_name, uri))
server = Server(server_uri)
app_label = app_name.split('.')[-1]
self._databases[app_label] = (server, dbname)
def main():
#arguments parser
parser = argparse.ArgumentParser(prog='pushCwe',
epilog="Example: ./%(prog)s.py")
parser.add_argument('-c',
'--couchdburi',
action='store',
type=str,
dest='couchdb',
default="http://127.0.0.1:5984",
help='Couchdb URL (default http://127.0.0.1:5984)')
#arguments put in variables
args = parser.parse_args()
#default value from ENV COUCHDB
couchdb = os.environ.get('COUCHDB')
#Else from argument
if not couchdb:
couchdb = args.couchdb
__serv = Server(uri=couchdb)
workspace = __serv.get_or_create_db("cwe")
with open('data/cwe.csv', 'r') as csvfile:
cwereader = csv.reader(csvfile, delimiter=',')
header = cwereader.next()
for cwe in cwereader:
cwe_doc = dict(zip(header, cwe))
workspace.save_doc(cwe_doc)
def main(*argv, **kwargs):
s = Server(argv[0])
db = s[argv[1]]
vr = db.view('proc/proc1', reduce=False, **kwargs)
scriptDir = argv[2]
script = '$KDATA_ROOT/lib/KDataPy/scripts/dataprocess/runProc1.py'
scriptOut = os.path.join(scriptDir, 'qsubout')
scriptErr = os.path.join(scriptDir, 'qsubout')
maxDocs = 30 #for process 1, we can submit up to 30 files to be processed
#by a particular batch job. This is because this is a relatively fast process
#to convert a Samba file to a KData file and we want to play nice with
#the batch system in Lyon
docids = []
for row in vr:
doc = db[row['id']]
doc['status'] = 'proc1 queued'
db.save_doc(doc)
docids.append(row['id'])
if len(docids) == maxDocs:
docids = submitBatchJob(scriptOut, scriptErr, script, argv[0],
argv[1], docids)
#don't forget last chunk of data files
docids = submitBatchJob(scriptOut, scriptErr, script, argv[0], argv[1],
docids)
def __init__(self, serverURL, dbname, username, pwd):
self.serverURL = serverURL
self.dbname = dbname
self.username = username
self.pwd = pwd
self.server = Server(self.serverURL,
filters=[BasicAuth(self.username, self.pwd)])
def handle(self, *args, **options):
quiet = options.get('quiet', False)
prod_indexes = options.get('prod-indexes', False)
docs_root = os.path.normpath(settings.DOCUMENT_ROOT)
if not quiet:
self.stdout.write(
'Cleaning UP stored files in DOCUMENT_ROOT: %s \n' % docs_root)
shutil.rmtree(docs_root)
os.makedirs(docs_root)
if not quiet:
self.stdout.write('done\n')
if not quiet:
self.stdout.write('Deleting CouchDB debug mode databases.\n')
databases = settings.COUCHDB_DATABASES
server = Server()
try:
for database in databases:
dbname = database[0] + '_test'
if not quiet:
self.stdout.write('Deleting DB: %s\n' % dbname)
server.delete_db(dbname)
if not quiet:
self.stdout.write('done\n')
except Exception, e:
if not quiet:
self.stdout.write(
'Failed to delete debug databases in CouchDB: %s ' % e)
pass
def main(db_name, db_user, db_pass, db_host, sequence_file, zone_dir, **tls_args):
# Starting Sequence for change stream
sequence = sequence_read(sequence_file)
click.echo('Skipping %s changes.' % sequence)
# CouchDB Connection
tls_args['cert_reqs'] = ssl.CERT_REQUIRED
tls_args['ssl_version'] = ssl.PROTOCOL_TLSv1_2
auth = CouchdbResource(filters=[BasicAuth(db_user, db_pass)], **tls_args)
server = Server(uri=db_host, resource_instance=auth)
db = server[db_name]
if sequence == 0:
click.echo('Fast track syncing all zones...')
c = Consumer(db)
result = c.fetch(descending=True, limit=1)
# Fast track to this sequence
sequence = result['last_seq']
# Go get all the current zones.
zones = c.fetch()
for zone in zones['results']:
domain = zone['id']
try:
doc = db.get(docid=domain)
except ResourceNotFound, e:
click.echo('%s not found (this is normal if the zone was deleted)' % domain)
else:
zone_update(domain, doc['data'], zone_dir)
sequence_write(sequence_file, sequence) # Keep track of our sync point
click.echo('Fast track syncing done')
def main(argv):
"""
Main method.
This method performs the following tasks:
1. Parse command line arguments
2. Retrieve credentials and connect to Cloudant and WebHDFS
3. Connect to the Cloudant `_changes` feed for checkpointed document
consumption
4. Process each change individually.
5. Upon exception throwing, store the latest checkpoint to local file and
exit.
"""
#add options into the parser
parser = configureOptions()
(options, args) = parser.parse_args()
checkRequiredArguments(options, parser)
print options
# configurations
last_seq = options.last_seq
#get credential
perm_file = '%s/.clou' % os.environ['HOME']
creds = get_creds(perm_file)
#connect to source database
s = Server('https://%s:%s@%s' %
(creds['cloudant_user'], creds['cloudant_pwd'], options.uri))
db = s[options.dbname]
#print db.info()
#connect to target hdfs cluster
hdfs = PyWebHdfsClient(host=options.hdfs_host,
port=options.hdfs_port,
user_name=creds['hdfs_user'])
hdfs.make_dir(options.hdfs_path)
#and here we consume the cloudant `_changes` feed
counter = 0
changestream = ChangesStream(db,
include_docs=True,
heartbeat=True,
since=last_seq)
for c in changestream:
#print c
try:
if counter % 100 == 0:
checkpoint(last_seq)
seq = processChange(hdfs, c, options.hdfs_path)
if seq: # protect against the last line being blank
last_seq = seq
counter += 1
except Exception:
traceback.print_exc()
checkpoint(last_seq)
os._exit(1)
checkpoint(last_seq)
def deleteFromView(server, databaseName, procname, **kwargs):
'''
This script will remove a "procname" key from database run documents. You can give this script
a single run number, or a range of runs numbers. (For now, it will remove the proc from ALL partition files within
a single run.
Additionally, you can specify options to the db.view('proc/daqdoc') call by including key=value pairs to the command line args
kwargs should at least contain startkey and endkey
'''
s = Server(server)
db = s[databaseName]
if kwargs['startkey'] == None or kwargs['endkey'] == None:
print 'you should supply a range of runs to use with **kwargs["startkey"] and kwargs["endkey"]'
return
vr = db.view('proc/daqdoc', reduce = False, **kwargs)
for row in vr:
if row.has_key('doc'):
doc = row['doc']
else:
doc = db.get(row['id'])
print 'checking', doc['_id']
if doc.has_key(procname):
print 'deleting', procname, 'from', doc['_id']
del doc[procname]
if procname == 'proc0':
doc['status'] = 'closed'
else:
doc['status'] = 'good'
db.save_doc(doc)
def main(*argv):
'''
This script queries the edwdb database (on edwdbik.fzk.de) and returns the runs
of a particular type. This script can also list the types of runs that are available
Here are the examples for each one
1) ./listRunsByType.py "calibration gamma"
2) ./listRunsByType.py list
'''
s = Server('https://edwdbik.fzk.de:6984')
db = s['datadb']
if len(argv) == 0:
print 'You need to supply an argument'
help(main)
return
arg1 = argv[0]
if arg1 == 'list':
vr = db.view('header/runcondition', group_level=1)
print 'Run Types'
for row in vr:
print row['key'][0]
else:
print 'looking for runs of type', arg1
vr = db.view('header/runcondition', endkey = [arg1,''], startkey = [arg1+'\ufff0', ''], group_level = 2, descending = True)
for row in vr:
print row['key'][1]
def get_prod_db(source_uri):
"""
Get the production database object since we need to get some doc_ids from the prod database
"""
prod_db_name = source_uri.split('/')[-1]
prod_server = Server(uri=source_uri[:-len(prod_db_name)])
prod_db = prod_server.get_db(prod_db_name)
return prod_db
def __init__(self, serverURL, dbname, username, pwd):
self.serverURL = serverURL
self.dbname = dbname
self.username = username
self.pwd = pwd
self.server = Server(self.serverURL,
filters=[BasicAuth(self.username, self.pwd)])
set_logging('info') # suppress DEBUG output of the couchdbkit/restkit
def setUpClass(cls):
super(TestLogging, cls).setUpClass()
cls.server = Server()
cls.db = cls.server.create_db('couchdbkit_test')
doc = {"name": "Yesterday's Heroes"}
cls.db.save_doc(doc)
cls.doc_id = doc["_id"]
cls.log = CaptureLogOutput(mod.request_logger.name)
def prep_connection(self, uri, db_name):
# create server object
self.server = Server(uri=uri)
# create database
self.db = self.server.get_or_create_db(db_name)
# associate local objects to the db
Book.set_db(self.db)
def reconnect(self):
ret_val = False
ur = self.__uri
if CouchdbManager.testCouch(ur):
self.__serv = Server(uri=ur)
self.__dbs.clear()
self._lostConnection = False
ret_val = True
return ret_val
def test_create_and_get_db(self):
couch_manager = CouchDbManager(uri=CONF.getCouchURI())
couch_manager.createDb(self.dbname)
self.assertNotEquals(couch_manager.getDb(self.dbname), None,
"Db %s shouldn't be None" % self.dbname)
server = Server(uri=CONF.getCouchURI())
self.assertIn(self.dbname, server.all_dbs(),
"Db %s should be in the db list" % self.dbname)
def main(*argv):
'''
This script queries the edwdb database (on edwdbik.fzk.de) and returns information
found in the Muon Veto DAQ Map documents. You can do three things with this script.
1. obtain the information for a particular muon module
2. list the available fields to choose from in the muon veto daq map
3. search for the muon module that matches a particular field.
Here are the examples for each one
1) ./vetoDaqDb.py moduleinfo 3
2) ./vetoDaqDb.py list
3) ./vetoDaqDb.py "HV channel" 67
'''
s = Server('https://edwdbik.fzk.de:6984')
db = s['muonvetohardwaremap']
if len(argv) == 0:
print 'You need to supply some arguments'
help(main)
return
arg0 = argv[0]
if len(argv) > 1:
value = formatvalue(argv[1])
#print 'Searching Muon Veto DAQ Map for', key, '=', value
if arg0 == 'moduleinfo':
vr = db.view('map/module', reduce=False, key=value)
for row in vr:
doc = db.get(row['id'])
for k, val in doc.items():
if k != '_id' and k != '_rev':
print k, val
print '\n'
elif arg0 == 'list':
vr = db.view('map/keys', group=True)
for row in vr:
if row['key'] != '_id' and k['key'] != '_rev':
print row['key']
else:
vr = db.view('map/key_values', key=arg0)
for row in vr:
if row['value'] == value:
doc = db.get(row['id'])
print 'module:', doc['muonmodule'], 'end:', doc[
'end'], ' date_valid', doc['date_valid']
def get_detailed_status(options):
total_tests = 0
total_errors = 0
server = Server(options.node)
db = server.get_db(options.database)
doc = get_build_doc(db, options.build)
doc_content = db.open_doc(doc['_id'])
tests_list = doc_content['_attachments'].keys()
# TODO: Check for number of tests that ran
print "List of tests against %s are %s " % (options.build, tests_list)
failed_tests = []
# Data struct to store information per test_class (attachment)
# {test_name:{tests:<int>, errros:<int>, time:<float>}}
test_data = {}
for attachment, value in doc_content['_attachments'].items():
errors_count = 0
tests_count = 0
print "Fetching attachment %s " % attachment
file = db.fetch_attachment(doc, attachment)
file = file.encode('ascii', 'ignore')
xmldoc = etree.parse(StringIO.StringIO(file))
root = xmldoc.getroot()
for child in root:
attributes = child.attrib
for childish in child:
# To print the error
#print childish.text
failed_tests.append(attributes.get('name'))
attributes = root.attrib
# To get root element attributes
#print attributes
name = attributes.get('name')
errors_count = int(attributes.get('errors'))
tests_count = int(attributes.get('tests'))
total_tests += tests_count
total_errors += errors_count
print "Testname: %s, passed: %s, failed: %s" % (
name, tests_count - errors_count, errors_count)
test_data[name] = {}
test_data[name]['tests'] = tests_count
test_data[name]['errors'] = errors_count
test_data[name]['time'] = float(attributes.get('time'))
#print test_data
text = "Passed %s out of %s tests on %s build\n" % (
total_tests - total_errors, total_tests, options.build)
num_failed = len(failed_tests)
if num_failed > 0:
text += "\nList of Failed tests:\n"
for i in range(num_failed):
text += "%s: %s\n" % (i + 1, failed_tests[i])
print text
return test_data
def submitBatchJob(scriptOut, scriptErr, script, server, dbname, docids):
if len(docids) == 0:
return []
command = 'qsub -P P_edelweis -o %s -e %s -l sps=1 -l vmem=1G -l fsize=1024M %s %s %s %s' % (
scriptOut, scriptErr, script, server, dbname, ' '.join(docids))
proc = subprocess.Popen(
command,
shell=True,
stdout=subprocess.PIPE,
)
val = proc.communicate()[0]
if val != '':
print val
#now we need to update the
#database documents with this batch job submission
#metadata
s = Server(server)
db = s[dbname]
docs = []
for anId in docids:
doc = db[anId]
if doc.has_key('batchJob') == False:
doc['batchJob'] = []
jobStuff = {}
jobStuff['number'] = int(val.split(' ')[2])
jobStuff['script'] = script
jobStuff['stdout'] = os.path.join(
scriptOut, os.path.basename(script)) + '.o' + str(
jobStuff['number'])
jobStuff['stderr'] = os.path.join(
scriptErr, os.path.basename(script)) + '.e' + str(
jobStuff['number'])
jobStuff['command'] = command
jobStuff['type'] = 'proc1'
jobStuff['message'] = val
jobStuff['date'] = str(datetime.datetime.now())
doc['batchJob'].append(jobStuff)
proc = {}
proc['batchjob'] = jobStuff['number']
doc['proc1'] = proc
docs.append(doc)
db.bulk_save(docs)
return [] #return an empty list
def _list_couch_docs(self, db_name='dmscouch_test'):
"""Downloads all the documents that are currently in CouchDB now"""
docs = {}
server = Server()
db = server.get_or_create_db(db_name)
r = db.view(
'dmscouch/all',
include_docs=True,
)
for row in r:
docs[row['doc']['_id']] = row['doc']
return docs
def push_cwe(couchdb_url, filename):
__serv = Server(uri=couchdb_url)
workspace = __serv.get_or_create_db("cwe")
with open(filename, 'r') as csvfile:
cwereader = csv.reader(csvfile, delimiter=',')
header = cwereader.next()
print "[*] Beginning upload"
for cwe in cwereader:
cwe_doc = dict(zip(header, cwe))
workspace.save_doc(cwe_doc)
print "[*] Upload finished"
def init_app(self, app):
"""Bind an app to a CouchDBKit instance and initialize the database.
:param app: The application to which the CouchDBKit instance should be
bound.
"""
self.app = app
self.app.config.setdefault('COUCHDB_SERVER', 'http://localhost:5984/')
self.app.config.setdefault('COUCHDB_DATABASE', None)
self.app.config.setdefault('COUCHDB_KEEPALIVE', None)
self.app.config.setdefault('COUCHDB_VIEWS', '_design')
server_uri = app.config.get('COUCHDB_SERVER')
pool_keepalive = app.config.get('COUCHDB_KEEPALIVE')
if pool_keepalive is not None:
mgr = Manager(max_conn=pool_keepalive)
server = Server(server_uri, manager=mgr)
else:
server = Server(server_uri)
self.server = server
self.init_db()
def process_couchdb_changes():
server = Server(settings.COUCHDB_SERVER)
db = server.get_or_create_db('openelm')
consumer = Consumer(db)
sequence = SyncSequenceCache.objects.get(pk=1)
changes = consumer.fetch(filter='record/new_records', since=sequence.last_sequence_id)
if changes:
for change in changes['results']:
record_id = change['id']
copy_photo_for_record.delay(record_id)
send_new_record_email.delay(record_id)
sequence.last_sequence_id = changes['last_seq']
sequence.save()
def main():
s = Server('http://*****:*****@127.0.0.1:5984')
db = s['datadb']
c = Consumer(db)
#start listening since = current update sequence.
#callback function is run.main
#heartbeat every minute to keep the connection alive.
c.wait(callback,
since=db.info()['update_seq'],
filter='proc/newproc0',
feed='continuous',
heartbeat=60000)
def upload_file(fname, uri, dbname):
print 'Upload contents of %s to %s/%s' % (fname, uri, dbname)
theServer = Server(
uri, filters=[BasicAuth(cloudant_username, cloudant_password)])
db = theServer.get_or_create_db(dbname)
reader = DictReader(open(fname, 'rU'), dialect='excel')
docs = list()
checkpoint = 100
for doc in reader:
newdoc = parse_doc(doc)
docs.append(newdoc)
if len(docs) % checkpoint == 0:
docs = upload(db, docs)
docs = upload(db, docs)
def sync_all_app(self, uri, dbname, views):
'''
Create a database session for each databases, then start the syncing
process.
Databases are described by tuples containing the application name and
the database URL in which views must be synchronized.
@param uri: Uri of the couchdb server
@param dbname: Database name
@param views: Name of the views
'''
for view in views:
res = CouchdbResource(uri, timeout=COUCHDB_TIMEOUT)
server = Server(uri, resource_instance=res)
self.sync(server, dbname, view, views[view])
def __init__(self, databases):
""" initialize couchdbkit handler with COUCHDB_DATABASES
settings """
self.__dict__ = self.__shared_state__
# Convert old style to new style
if isinstance(databases, (list, tuple)):
databases = dict((app_name, {
'URL': uri
}) for app_name, uri in databases)
# create databases sessions
for app_name, app_setting in databases.items():
uri = app_setting['URL']
# Do not send credentials when they are both None as admin party will give a 401
user = app_setting.get('USER')
password = app_setting.get('PASSWORD')
filters = [BasicAuth(user, password)
] if (user or password) is not None else []
try:
if isinstance(uri, (list, tuple)):
# case when you want to specify server uri
# and database name specifically. usefull
# when you proxy couchdb on some path
server_uri, dbname = uri
else:
server_uri, dbname = uri.rsplit("/", 1)
except ValueError:
raise ValueError("couchdb uri [%s:%s] invalid" %
(app_name, uri))
res = CouchdbResource(server_uri,
timeout=COUCHDB_TIMEOUT,
filters=filters)
server = Server(server_uri, resource_instance=res)
app_label = app_name.split('.')[-1]
self._databases[app_label] = (server, dbname)
def __init__(self, uri):
self._last_seq_ack = 0
model.api.log("Initializing CouchDBManager for url [%s]" % uri)
self._lostConnection = False
self.__uri = uri
self.__dbs = {}
self.__seq_nums = {}
self.__serv = NoConectionServer()
self.mutex = threading.Lock()
self._available = False
try:
self.testCouchUrl(uri)
url = urlparse(uri)
print("Setting user,pass %s %s" % (url.username, url.password))
self.__serv = Server(uri=uri)
#print dir(self.__serv)
self.__serv.resource_class.credentials = (url.username,
url.password)
self._available = True
except:
model.api.log("No route to couchdb server on: %s" % uri)
print(traceback.format_exc())
def __init__(self, uri):
super(CouchDbManager, self).__init__()
getLogger(self).debug(
"Initializing CouchDBManager for url [%s]" % uri)
self._lostConnection = False
self.__uri = uri
self.__serv = NoConectionServer()
self._available = False
try:
if uri is not None:
self.testCouchUrl(uri)
url = urlparse(uri)
getLogger(self).debug(
"Setting user,pass %s %s" % (url.username, url.password))
self.__serv = Server(uri=uri)
self.__serv.resource_class.credentials = (url.username, url.password)
self._available = True
self.pushReports()
self._loadDbs()
except:
getLogger(self).warn("No route to couchdb server on: %s" % uri)
getLogger(self).debug(traceback.format_exc())
