class DatabaseAPI(object):
"""
Serves as an API object that can be passed around. See above for usage
Attributes:
cacheStore : CacheStore object
permStore : PermStore object
"""
def __init__(self):
self.duration_cacheStoreSet = 0.0
self.duration_permStoreSet = 0.0
self.duration_cacheStoreGet = 0.0
self.duration_permStoreGet = 0.0
self.cacheStore = CacheStore()
self.permStore = PermStore()
self.broker = None
def submit_job(self,app_id,exp_uid,task,task_args_json,namespace=None,ignore_result=True,time_limit=0):
if self.broker == None:
self.broker = next.broker.broker.JobBroker()
if namespace==None:
result = self.broker.applyAsync(app_id,exp_uid,task,task_args_json,ignore_result=ignore_result)
else:
result = self.broker.applySyncByNamespace(app_id,exp_uid,task,task_args_json,namespace=namespace,ignore_result=ignore_result,time_limit=time_limit)
return result
def exists(self,bucket_id,doc_uid,key):
"""
Checks existence of key.
Inputs:
(string) bucket_id, (string) doc_uid, (string) key
Outputs:
(bool) exists, (bool) didSucceed, (string) message
Usage: ::\n
exists,didSucceed,message = db.exists(bucket_id,doc_uid,key)
"""
try:
if USE_CACHE:
# attempts to read file from cache first
keyExistsInCacheStore,didSucceed,message = self.cacheStore.exists(constants.app_data_database_id,bucket_id,doc_uid,key)
if not didSucceed:
return None,False,'DatabaseAPI.cacheStore.exists(key) failed'
elif didSucceed and keyExistsInCacheStore:
return True,True,''
# attempts to read from permanent store
keyExistsInPermStore,didSucceed,message = self.permStore.exists(constants.app_data_database_id,bucket_id,doc_uid,key)
if not didSucceed:
return None,False,'DatabaseAPI.permStore.exists(key) failed'
elif didSucceed and keyExistsInPermStore:
return True,True,''
return False,True,''
except:
error = "DatabaseAPI.exists Failed with unknown exception"
return None,False,error
def get(self,bucket_id,doc_uid,key):
"""
Get a value corresponding to key, returns None if no key exists
Inputs:
(string) bucket_id, (string) doc_uid, (string) key
Outputs:
(python object) value, (bool) didSucceed, (string) message
Usage: ::\n
value,didSucceed,message = db.get(bucket_id,doc_uid,key)
"""
try:
if USE_CACHE:
# attempts to read file from cache first
keyExistsInCacheStore,didSucceed,message,dt = utils.timeit(self.cacheStore.exists)(constants.app_data_database_id,bucket_id,doc_uid,key)
self.duration_cacheStoreGet += dt
if not didSucceed:
return None,False,'get.cacheStore.exists(key) failed'
if keyExistsInCacheStore:
value,didSucceed,message,dt = utils.timeit(self.cacheStore.get)(constants.app_data_database_id,bucket_id,doc_uid,key)
self.duration_cacheStoreGet += dt
if not didSucceed:
return None,False,message
return value,True,'From Cache'
else:
# attempts to read from permanent store
keyExistsInPermStore,didSucceed,message,dt = utils.timeit(self.permStore.exists)(constants.app_data_database_id,bucket_id,doc_uid,key)
self.duration_permStoreGet += dt
if not didSucceed:
return None,False,'get.permStore.exists(key) failed'
if keyExistsInPermStore:
value,didSucceed,message,dt = utils.timeit(self.permStore.get)(constants.app_data_database_id,bucket_id,doc_uid,key)
self.duration_permStoreGet += dt
if not didSucceed:
return None,False,message
didSucceed,message = self.cacheStore.set(constants.app_data_database_id,bucket_id,doc_uid,key,value)
if not didSucceed:
return None,False,message
return value,True,'Hit PermStore'
# could not find file
else:
return None,True,'Not in Database'
else:
# not using cache
value,didSucceed,message,dt = utils.timeit(self.permStore.get)(constants.app_data_database_id,bucket_id,doc_uid,key)
self.duration_permStoreGet += dt
if not didSucceed:
return None,False,message
return value,True,'Hit PermStore'
except:
return None,False,'DatabaseAPI.get Failed with unknown exception'
def increment(self,bucket_id,doc_uid,key,value=1):
"""
increments a key by amount value. If key does not exist, sets {key:value}
Inputs:
(string) bucket_id, (string) doc_uid, (string) key, (int) value
Outputs:
(int) new_value, (bool) didSucceed, (string) message
Usage: ::\n
new_value,didSucceed,message = db.increment(bucket_id,doc_uid,key,value)
"""
try:
if USE_CACHE:
# need to implement cache!!
#############################
new_value,didSucceed,message,dt = utils.timeit(self.permStore.increment)(constants.app_data_database_id,bucket_id,doc_uid,key,value)
self.duration_permStoreSet += dt
if not didSucceed:
return None,False,message
return new_value,True,'Hit PermStore'
else:
new_value,didSucceed,message,dt = utils.timeit(self.permStore.increment)(constants.app_data_database_id,bucket_id,doc_uid,key,value)
self.duration_permStoreSet += dt
if not didSucceed:
return None,False,message
return new_value,True,'Hit PermStore'
except:
return None,False,'DatabaseAPI.increment Failed with unknown exception'
def get_list(self,bucket_id,doc_uid,key):
"""
Get a value corresponding to key, returns None if no key exists
Inputs:
(string) bucket_id, (string) doc_uid, (string) key
Outputs:
(list) value, (bool) didSucceed, (string) message
Usage: ::\n
value,didSucceed,message = db.get_list(bucket_id,doc_uid,key)
"""
try:
if USE_CACHE:
# attempts to read file from cache first
keyExistsInCacheStore,didSucceed,message,dt = utils.timeit(self.cacheStore.exists)(constants.app_data_database_id,bucket_id,doc_uid,key)
self.duration_cacheStoreGet += dt
if not didSucceed:
return None,False,'get.cacheStore.exists(key) failed'
if keyExistsInCacheStore:
value,didSucceed,message,dt = utils.timeit(self.cacheStore.get_list)(constants.app_data_database_id,bucket_id,doc_uid,key)
self.duration_cacheStoreGet += dt
if not didSucceed:
return None,False,message
return value,True,'From Cache'
else:
value,didSucceed,message,dt = utils.timeit(self.permStore.get_list)(constants.app_data_database_id,bucket_id,doc_uid,key)
self.duration_permStoreGet += dt
if not didSucceed:
return None,False,message
if value!=None:
didSucceed,message = self.cacheStore.set_list(constants.app_data_database_id,bucket_id,doc_uid,key,value)
if not didSucceed:
return None,False,message
return value,True,'Hit PermStore'
# could not find file
else:
return None,True,'Not in Database'
else:
# not in cache
value,didSucceed,message,dt = utils.timeit(self.permStore.get_list)(constants.app_data_database_id,bucket_id,doc_uid,key)
self.duration_permStoreGet += dt
if not didSucceed:
return None,False,message
return value,True,'Hit PermStore'
except:
return None,False,'DatabaseAPI.get Failed with unknown exception'
def append_list(self,bucket_id,doc_uid,key,value):
"""
Appends a {key,value_list} (if already exists, replaces)
Inputs:
(string) bucket_id, (string) doc_uid, (string) key, (list) value
Outputs:
(bool) didSucceed, (string) message
Usage: ::\n
didSucceed,message = db.set_list(bucket_id,doc_uid,key,value)
"""
try:
if USE_CACHE:
# attempts to read file from cache first
keyExistsInCacheStore,didSucceed,message,dt = utils.timeit(self.cacheStore.exists)(constants.app_data_database_id,bucket_id,doc_uid,key)
self.duration_cacheStoreGet += dt
if not didSucceed:
return False,message
if keyExistsInCacheStore:
didSucceedCache,message,dt = utils.timeit(self.cacheStore.append_list)(constants.app_data_database_id,bucket_id,doc_uid,key,value)
self.duration_cacheStoreSet += dt
didSucceedPerm,messagePerm,dt = utils.timeit(self.permStore.append_list)(constants.app_data_database_id,bucket_id,doc_uid,key,value)
self.duration_permStoreSet += dt
else:
didSucceedPerm,messagePerm,dt = utils.timeit(self.permStore.append_list)(constants.app_data_database_id,bucket_id,doc_uid,key,value)
self.duration_permStoreSet += dt
value_list,didSucceedPerm,message,dt = utils.timeit(self.permStore.get_list)(constants.app_data_database_id,bucket_id,doc_uid,key)
self.duration_permStoreGet += dt
didSucceedCache,message = self.cacheStore.set_list(constants.app_data_database_id,bucket_id,doc_uid,key,value_list)
if not didSucceed:
return False,message
if didSucceedCache and didSucceedPerm:
return True,''
else:
return False,message
else:
didSucceedPerm,messagePerm,dt = utils.timeit(self.permStore.append_list)(constants.app_data_database_id,bucket_id,doc_uid,key,value)
self.duration_permStoreSet += dt
return didSucceedPerm,messagePerm
except:
error = "DatabaseAPI.append_list Failed with unknown exception"
return False,error
def set_list(self,bucket_id,doc_uid,key,value):
"""
Sets a {key,value_list} (if already exists, replaces)
Inputs:
(string) bucket_id, (string) doc_uid, (string) key, (list) value
Outputs:
(bool) didSucceed, (string) message
Usage: ::\n
didSucceed,message = db.set_list(bucket_id,doc_uid,key,value)
"""
try:
if USE_CACHE:
# writes to cache first
didSucceedCache,messageCache,dt = utils.timeit(self.cacheStore.set_list)(constants.app_data_database_id,bucket_id,doc_uid,key,value)
self.duration_cacheStoreSet += dt
# then writes to permanent store
didSucceedPerm,messagePerm,dt = utils.timeit(self.permStore.set_list)(constants.app_data_database_id,bucket_id,doc_uid,key,value)
self.duration_permStoreSet += dt
if didSucceedCache and didSucceedPerm:
return True,''
else:
return False,messageCache + '\n' + messagePerm
else:
didSucceedPerm,messagePerm,dt = utils.timeit(self.permStore.set_list)(constants.app_data_database_id,bucket_id,doc_uid,key,value)
self.duration_permStoreSet += dt
return didSucceedPerm,messagePerm
except:
error = "DatabaseAPI.set Failed with unknown exception"
return False,error
def set_doc(self,bucket_id,doc_uid,doc):
"""
Sets a document with doc_uid
Inputs:
(dict) doc
Outputs:
None
Usage: ::\n
db.set_doc(key,value)
"""
didSucceed,message = self.permStore.setDoc(constants.app_data_database_id,bucket_id,doc_uid,doc)
return didSucceed,message
def get_doc(self,bucket_id,doc_uid):
"""
Gets doc in bucket_id that corresponds to doc_uid
Inputs:
(string) bucket_id, (string) doc_uid
Outputs:
(dict) doc, (bool) didSucceed, (string) message
Usage: ::\n
doc,didSucceed,message = db.getDoc(bucket_id,doc_uid)
"""
return self.permStore.getDoc(constants.app_data_database_id,bucket_id,doc_uid)
def get_docs_with_filter(self,bucket_id,pattern_dict):
"""
Retrieves all docs in bucket_id that match (i.e. contain) pattern_dict
Inputs:
(string) bucket_id, (dict of string values) pattern_dict
Outputs:
(list of dict) docs, (bool) didSucceed, (string) message
Usage: ::\n
docs,didSucceed,message = db.getDocsByPattern(bucket_id,pattern_dict)
"""
return self.permStore.getDocsByPattern(constants.app_data_database_id,bucket_id,pattern_dict)
def set(self,bucket_id,doc_uid,key,value):
"""
Sets a {key,value} (if already exists, replaces)
Inputs:
(string) bucket_id, (string) doc_uid, (string) key, (string) value
Outputs:
(bool) didSucceed, (string) message
Usage: ::\n
didSucceed,message = db.set(bucket_id,doc_uid,key,value)
"""
try:
if USE_CACHE:
# writes to cache first
didSucceedCache,messageCache,dt = utils.timeit(self.cacheStore.set)(constants.app_data_database_id,bucket_id,doc_uid,key,value)
self.duration_cacheStoreSet += dt
# then writes to permanent store
didSucceedPerm,messagePerm,dt = utils.timeit(self.permStore.set)(constants.app_data_database_id,bucket_id,doc_uid,key,value)
self.duration_permStoreSet += dt
if didSucceedCache and didSucceedPerm:
return True,''
else:
return False,messageCache + '\n' + messagePerm
else:
didSucceedPerm,messagePerm,dt = utils.timeit(self.permStore.set)(constants.app_data_database_id,bucket_id,doc_uid,key,value)
self.duration_permStoreSet += dt
return didSucceedPerm,messagePerm
except:
error = "DatabaseAPI.set Failed with unknown exception"
return False,error
def delete(self,bucket_id,doc_uid,key):
"""
Deletes {key:value} associated with given key
Inputs:
(string) bucket_id, (string) doc_uid, (string) key
Outputs:
(bool) didSucceed, (string) message
Usage: ::\n
didSucceed,message = db.delete(bucket_id,doc_uid,key)
"""
try:
if USE_CACHE:
# attempts to delete file from cache first
didSucceed,message = self.cacheStore.delete(constants.app_data_database_id,bucket_id,doc_uid,key)
if not didSucceed:
return False,'DatabaseAPI.cacheStore.delete(key) failed'
# attempts to delete from permanent store
didSucceed,message = self.permStore.delete(constants.app_data_database_id,bucket_id,doc_uid,key)
if not didSucceed:
return False,'DatabaseAPI.permStore.delete(key) failed'
return True,''
else:
didSucceed,message = self.permStore.delete(constants.app_data_database_id,bucket_id,doc_uid,key)
return didSucceed,message
except:
error = "DatabaseAPI.delete Failed with unknown exception"
return False,error
def ensure_index(self,bucket_id,index_dict):
"""
Adds index defined on index_dict to bucket_id
Inputs:
(string) bucket_id, (dict) index_dict
Outputs:
(string) index_info, (bool) didSucceed, (string) message
Usage: ::\n
didSucceed,message = db.get_index_info('rand_data',{'num_eyes':1,'exp_uid',1})
"""
didSucceed,message = self.permStore.create_index(constants.app_data_database_id,bucket_id,index_dict)
return didSucceed,message
def drop_all_indexes(self,bucket_id):
"""
Deletes all indexes defined on bucket_id
Inputs:
(string) bucket_id
Outputs:
(bool) didSucceed, (string) message
Usage: ::\n
didSucceed,message = db.drop_all_indexes(bucket_id)
"""
didSucceed,message = self.permStore.drop_all_indexes(constants.app_data_database_id,bucket_id)
return didSucceed,message
def delete_docs_with_filter(self,bucket_id,pattern_dict):
"""
Deletes all docs in bucket_id that match (i.e. contain) pattern_dict
Inputs:
(string) bucket_id, (dict of string values) pattern_dict
Outputs:
(bool) didSucceed, (string) message
Usage: ::\n
didSucceed,message = db.deleteDocsByPattern(bucket_id,key,value)
"""
docs,didSucceed,message = self.permStore.getDocsByPattern(constants.app_data_database_id,bucket_id,pattern_dict)
for doc in docs:
try:
doc_uid = doc['doc_uid']
didSucceed,message = self.cacheStore.deleteDoc(constants.app_data_database_id,bucket_id,doc_uid)
except:
pass
return self.permStore.deleteDocsByPattern(constants.app_data_database_id,bucket_id,pattern_dict)
def log(self,bucket_id,log_dict):
"""
Saves log_dict to PermStore as an individual document for later recall.
Inputs:
(string) bucket_id, (dict with string values) log_dict
Outputs:
(bool) didSucceed, (string) message
Usage: ::\n
didSucceed,message = db.log(bucket_id,doc_uid)
"""
timestamp = utils.datetimeNow()
log_dict.update({ 'timestamp':timestamp })
return self.permStore.setDoc(constants.logs_database_id,bucket_id,None,log_dict)
def get_logs_with_filter(self,bucket_id,pattern_dict):
"""
Retrieves all logs in bucket_id that match (i.e. contain) pattern_dict
Inputs:
(string) bucket_id, (dict of string values) pattern_dict
Outputs:
(list of dict) logs, (bool) didSucceed, (string) message
Usage: ::\n
logs,didSucceed,message = db.getLogsByPattern(bucket_id,pattern_dict)
"""
return self.permStore.getDocsByPattern(constants.logs_database_id,bucket_id,pattern_dict)
def delete_logs_with_filter(self,bucket_id,pattern_dict):
"""
Deletes all logs in bucket_id that match (i.e. contain) pattern_dict
Inputs:
(string) bucket_id, (dict of string values) pattern_dict
Outputs:
(bool) didSucceed, (string) message
Usage: ::\n
didSucceed,message = db.deleteLogsByPattern(bucket_id,key,value)
"""
return self.permStore.deleteDocsByPattern(constants.logs_database_id,bucket_id,pattern_dict)
def assertConnection(self):
"""
Asserts that the API has successfully connected to both the CacheStore and PermStore
Inputs:
None
Outputs:
(bool) areConnected, (string) message
Usage: ::\n
didSucceed,message = db.assertConnection()
"""
try:
if USE_CACHE:
cacheStoreConnected = self.cacheStore.assertConnection()
permStoreConnected = self.permStore.assertConnection()
if (cacheStoreConnected) and (permStoreConnected):
return True,''
elif (not cacheStoreConnected) and (not permStoreConnected):
return False,'Failed to connect to CacheStore and PermStore'
elif (not cacheStoreConnected) and (permStoreConnected):
return False,'Failed to connect to CacheStore'
elif (cacheStoreConnected) and (not permStoreConnected):
return False,'Failed to connect to PermStore'
else:
permStoreConnected = self.permStore.assertConnection()
if (permStoreConnected):
return True,'Cache dsiabled'
else:
return False,'Cache disabled, no connection from Permstore'
except:
error = "DatabaseAPI.assertConnection unknown exception"
return False, error
def flushDocCache(self,bucket_id,doc_uid):
"""
Deletes all app data in the Cache corresponding to bucket_id,doc_uid,
Note: This does NOT delete anything in PermStore (i.e. this is a safe operation)
Inputs:
(string) bucket_id, (string) doc_uid
Outputs:
(bool) didSucceed, (string) message
Usage: ::\n
didSucceed,message = db.flushDocCache(bucket_id,doc_uid)
"""
# delete everything in cache having to do with experiment
# get alg_list if it exists and delete all children to exp_uid
algListExists,didSucceed,message = self.exists(bucket_id,doc_uid,'alg_list')
if not didSucceed:
return False,message
if algListExists:
alg_list,didSucceed,message = self.get(bucket_id,doc_uid,'alg_list')
if not didSucceed:
return False,message
exp_uid,didSucceed,message = self.get(bucket_id,doc_uid,'exp_uid')
if not didSucceed:
return False,message
for alg_id in alg_list:
child_doc_uid = utils.getDocUID(exp_uid,alg_id)
didSucceed,message = self.cacheStore.deleteDoc(constants.app_data_database_id,bucket_id,child_doc_uid)
if not didSucceed:
return False,message
didSucceed,message = self.cacheStore.deleteDoc(constants.app_data_database_id,bucket_id,doc_uid)
if not didSucceed:
return False,message
return True,''
def flushCache(self):
"""
Deletes everything from the Cache.
Note: This does NOT delete anything in PermStore (i.e. this is a safe operation)
Inputs:
None
Outputs:
(bool) didSucceed, (string) message
Usage: ::\n
didSucceed,message = db.flushCache(bucket_id,doc_uid)
"""
return self.cacheStore.deleteAll()
def irreversibly_delete_everything(self):
if USE_CACHE:
self.cacheStore.deleteAll()
self.permStore.deleteAll()
def getDocNames(self,bucket_id):
"""
Get list of doc_uids correspding to all the docs in the bucket corresponding to the given bucket_id
Inputs:
(string) bucket_id
Outputs:
([(string) doc_uid, ... ]) docNames, (bool) didSucceed, (string) message
Usage: ::\n
docNames,didSucceed,message = db.getDocNames(bucket_id)
"""
try:
doc_uids,didSucceed,message = self.permStore.getDocNames(constants.app_data_database_id,bucket_id)
if not didSucceed:
return None,False,message
return doc_uids,True,''
except:
error = "DatabaseAPI.getDocNames Failed with unknown exception"
return None,False,error
def getBucketNames(self):
"""
Get list of bucket_ids
Inputs:
None
Outputs:
([(string) bucket_id, ... ]) bucketNames, (bool) didSucceed, (string) message
Usage: ::\n
bucketNames,didSucceed,message = db.getBucketNames()
"""
try:
bucket_ids,didSucceed,message = self.permStore.getBucketNames(constants.app_data_database_id)
if not didSucceed:
return None,False,message
return bucket_ids,True,''
except:
error = "DatabaseAPI.getDocNames Failed with unknown exception"
return None,False,error
def inspectDatabase(self):
"""
Returns string describing the entire app data database
Inputs:
None
Outputs:
(string) description, (bool) didSucceed, (string) message
Usage: ::\n
didSucceed,message = db.inspectDatabase()
"""
try:
output_str = ''
bucket_ids,didSucceed,message = self.permStore.getBucketNames(constants.app_data_database_id)
for bucket_id in bucket_ids:
bucket_id_str = str(bucket_id)
output_str += bucket_id_str + " { \n"
doc_uids,didSucceed,message = self.permStore.getDocNames(constants.app_data_database_id,bucket_id)
if doc_uids!=None :
for doc_uid in doc_uids:
doc_uid_str = str(doc_uid)
output_str += "\t" + doc_uid_str + " { \n"
doc,didSucceed,message = self.permStore.getDoc(constants.app_data_database_id,bucket_id,doc_uid)
for key in doc:
pickled_value = doc[key]
try:
value = cPickle.loads(pickled_value)
except:
value = pickled_value
str_value = str(value)
if len(str_value)>constants.maxStringLengthInInspectDatabase:
str_value = str_value[0:constants.maxStringLengthInInspectDatabase]+ " ..."
output_str += "\t\t" + key + " : " + str_value + "\n"
output_str += "\t" + "}" + "\n"
output_str += "}" + "\n"
return output_str,True,''
except:
error = "DatabaseAPI.inspectDatabase() incurred an unknown exception "
return None,False,error
def inspectDoc(self,bucket_id,doc_uid):
"""
Returns string describing the document of the particular bucket_id, doc_uid
Inputs:
(string) bucket_id, (string) doc_uid
Outputs:
(string) description, (bool) didSucceed, (string) message
Usage: ::\n
didSucceed,message = db.inspectDatabase()
"""
try:
output_str = ''
output_str += doc_uid + " { \n"
doc,didSucceed,message = self.permStore.getDoc(constants.app_data_database_id,bucket_id,doc_uid)
for key in doc:
pickled_value = doc[key]
try:
value = cPickle.loads(pickled_value)
except:
value = pickled_value
str_value = str(value)
if len(str_value)>constants.maxStringLengthInInspectDatabase:
str_value = str_value[0:max_value_string_length]+ " ..."
output_str += "\t" + key + " : " + str_value + "\n"
output_str += "}" + "\n"
return output_str,True,''
except:
return None,False,'DatabaseAPI.inspectDoc unknown exception'
class LoggerAPI(object):
"""
Serves as an API object that can be passed around. See above for usage
Attributes:
permStore : PermStore object
"""
def __init__(self):
self.permStore = PermStore()
def log(self, bucket_id, log_dict):
"""
Saves log_dict to PermStore as an individual document for later recall.
Inputs:
(string) bucket_id, (dict with string values) log_dict
Outputs:
(bool) didSucceed, (string) message
Usage: ::\n
didSucceed,message = db.log(bucket_id,doc_uid)
"""
return self.permStore.setDoc(constants.logs_database_id, bucket_id,
None, log_dict)
def ensure_index(self, bucket_id, index_dict):
"""
Adds index defined on index_dict to bucket_id
Inputs:
(string) bucket_id, (dict) index_dict
Outputs:
(string) index_info, (bool) didSucceed, (string) message
Usage: ::\n
didSucceed,message = db.get_index_info('rand_data',{'num_eyes':1,'exp_uid',1})
"""
didSucceed, message = self.permStore.create_index(
constants.logs_database_id, bucket_id, index_dict)
return didSucceed, message
def drop_all_indexes(self, bucket_id):
"""
Deletes all indexes defined on bucket_id
Inputs:
(string) bucket_id
Outputs:
(bool) didSucceed, (string) message
Usage: ::\n
didSucceed,message = db.drop_all_indexes(bucket_id)
"""
didSucceed, message = self.permStore.drop_all_indexes(
constants.logs_database_id, bucket_id)
return didSucceed, message
def get_logs_with_filter(self, bucket_id, pattern_dict):
"""
Retrieves all logs in bucket_id that match (i.e. contain) pattern_dict
Inputs:
(string) bucket_id, (dict of string values) pattern_dict
Outputs:
(list of dict) logs, (bool) didSucceed, (string) message
Usage: ::\n
logs,didSucceed,message = db.getLogsByPattern(bucket_id,pattern_dict)
"""
return self.permStore.getDocsByPattern(constants.logs_database_id,
bucket_id, pattern_dict)
def delete_logs_with_filter(self, bucket_id, pattern_dict):
"""
Deletes all logs in bucket_id that match (i.e. contain) pattern_dict
Inputs:
(string) bucket_id, (dict of string values) pattern_dict
Outputs:
(bool) didSucceed, (string) message
Usage: ::\n
didSucceed,message = db.deleteLogsByPattern(bucket_id,key,value)
"""
return self.permStore.deleteDocsByPattern(constants.logs_database_id,
bucket_id, pattern_dict)
def assertConnection(self):
"""
Asserts that the API has successfully connected to the PermStore
Inputs:
None
Outputs:
(bool) areConnected, (string) message
Usage: ::\n
didSucceed,message = db.assertConnection()
"""
permStoreConnected = self.permStore.assertConnection()
if (permStoreConnected):
return True, ''
else:
return False, 'no connection from Permstore'
def irreversibly_delete_everything(self):
self.permStore.deleteAll()
class DatabaseAPI(object):
"""
Serves as an API object that can be passed around. See above for usage
Attributes:
cacheStore : CacheStore object
permStore : PermStore object
"""
def __init__(self):
self.duration_cacheStoreSet = 0.0
self.duration_permStoreSet = 0.0
self.duration_cacheStoreGet = 0.0
self.duration_permStoreGet = 0.0
self.cacheStore = CacheStore()
self.permStore = PermStore()
self.broker = None
def submit_job(self,app_id,exp_uid,task,task_args_json,namespace=None,ignore_result=True,time_limit=0, alg_id=None, alg_label=None):
if self.broker == None:
self.broker = next.broker.broker.JobBroker()
if namespace==None:
result = self.broker.applyAsync(app_id,exp_uid,task,task_args_json,ignore_result=ignore_result)
else:
result = self.broker.applySyncByNamespace(app_id,exp_uid,
alg_id, alg_label,
task,task_args_json,namespace=namespace,
ignore_result=ignore_result,time_limit=time_limit)
return result
def exists(self,bucket_id,doc_uid,key):
"""
Checks existence of key.
Inputs:
(string) bucket_id, (string) doc_uid, (string) key
Outputs:
(bool) exists, (bool) didSucceed, (string) message
Usage: ::\n
exists,didSucceed,message = db.exists(bucket_id,doc_uid,key)
"""
try:
if USE_CACHE:
# attempts to read file from cache first
keyExistsInCacheStore,didSucceed,message = self.cacheStore.exists(constants.app_data_database_id,bucket_id,doc_uid,key)
if not didSucceed:
return None,False,'DatabaseAPI.cacheStore.exists(key) failed'
elif didSucceed and keyExistsInCacheStore:
return True,True,''
# attempts to read from permanent store
keyExistsInPermStore,didSucceed,message = self.permStore.exists(constants.app_data_database_id,bucket_id,doc_uid,key)
if not didSucceed:
return None,False,'DatabaseAPI.permStore.exists(key) failed'
elif didSucceed and keyExistsInPermStore:
return True,True,''
return False,True,''
except:
error = "DatabaseAPI.exists Failed with unknown exception"
return None,False,error
def get(self,bucket_id,doc_uid,key):
"""
Get a value corresponding to key, returns None if no key exists
Inputs:
(string) bucket_id, (string) doc_uid, (string) key
Outputs:
(python object) value, (bool) didSucceed, (string) message
Usage: ::\n
value,didSucceed,message = db.get(bucket_id,doc_uid,key)
"""
try:
if USE_CACHE:
# attempts to read file from cache first
response, dt = utils.timeit(self.cacheStore.exists)(constants.app_data_database_id,bucket_id,doc_uid,key)
keyExistsInCacheStore,didSucceed,message = response
self.duration_cacheStoreGet += dt
if not didSucceed:
return None,False,'get.cacheStore.exists(key) failed'
if keyExistsInCacheStore:
response,dt = utils.timeit(self.cacheStore.get)(constants.app_data_database_id,bucket_id,doc_uid,key)
value,didSucceed,message = response
self.duration_cacheStoreGet += dt
if not didSucceed:
return None,False,message
return value,True,'From Cache'
else:
# attempts to read from permanent store
response,dt = utils.timeit(self.permStore.exists)(constants.app_data_database_id,bucket_id,doc_uid,key)
keyExistsInPermStore,didSucceed,message = response
self.duration_permStoreGet += dt
if not didSucceed:
return None,False,'get.permStore.exists(key) failed'
if keyExistsInPermStore:
response,dt = utils.timeit(self.permStore.get)(constants.app_data_database_id,bucket_id,doc_uid,key)
value,didSucceed,message = response
self.duration_permStoreGet += dt
if not didSucceed:
return None,False,message
didSucceed,message = self.cacheStore.set(constants.app_data_database_id,bucket_id,doc_uid,key,value)
if not didSucceed:
return None,False,message
return value,True,'Hit PermStore'
# could not find file
else:
return None,True,'Not in Database'
else:
# not using cache
response,dt = utils.timeit(self.permStore.get)(constants.app_data_database_id,bucket_id,doc_uid,key)
value,didSucceed,message = response
self.duration_permStoreGet += dt
if not didSucceed:
return None,False,message
return value,True,'Hit PermStore'
except:
return None,False,'DatabaseAPI.get Failed with unknown exception'
def get_and_delete(self,bucket_id,doc_uid,key):
"""
returns value associated with key and then deltes {key:value}.
If key does not exist, returns None
Inputs:
(string) bucket_id, (string) doc_uid, (string) key
Outputs:
(bool) didSucceed, (string) message
Usage: ::\n
didSucceed,message = db.get_and_delete(bucket_id,doc_uid,key)
"""
try:
if USE_CACHE:
# not implemented
raise
else:
# not using cache
response,dt = utils.timeit(self.permStore.get_and_delete)(constants.app_data_database_id,bucket_id,doc_uid,key)
value,didSucceed,message = response
self.duration_permStoreGet += dt
if not didSucceed:
return None,False,message
return value,True,'Hit PermStore'
except:
return None,False,'DatabaseAPI.get Failed with unknown exception'
def get_many(self,bucket_id,doc_uid,key_list):
"""
Get values corresponding to keys in key_list, returns None if no key exists
Inputs:
(string) bucket_id, (string) doc_uid, (list of string) key_list
Outputs:
(dict of {key1:value1,key2:value2}) return_dict, (bool) didSucceed, (string) message
Usage: ::\n
return_dict,didSucceed,message = db.get_many(bucket_id,doc_uid,key_list)
"""
try:
if USE_CACHE:
raise
else:
# not using cache
response,dt = utils.timeit(self.permStore.get_many)(constants.app_data_database_id,bucket_id,doc_uid,key_list)
return_dict,didSucceed,message = response
self.duration_permStoreSet += dt
if not didSucceed:
return None,False,message
return return_dict,True,'Hit PermStore'
except:
return None,False,'DatabaseAPI.get Failed with unknown exception'
def increment(self,bucket_id,doc_uid,key,value=1):
"""
increments a key by amount value. If key does not exist, sets {key:value}
Inputs:
(string) bucket_id, (string) doc_uid, (string) key, (int) value
Outputs:
(int) new_value, (bool) didSucceed, (string) message
Usage: ::\n
new_value,didSucceed,message = db.increment(bucket_id,doc_uid,key,value)
"""
try:
if USE_CACHE:
# need to implement cache!!
#############################
response,dt = utils.timeit(self.permStore.increment)(constants.app_data_database_id,bucket_id,doc_uid,key,value)
new_value,didSucceed,message = response
self.duration_permStoreSet += dt
if not didSucceed:
return None,False,message
return new_value,True,'Hit PermStore'
else:
response,dt = utils.timeit(self.permStore.increment)(constants.app_data_database_id,bucket_id,doc_uid,key,value)
new_value,didSucceed,message = response
self.duration_permStoreSet += dt
if not didSucceed:
return None,False,message
return new_value,True,'Hit PermStore'
except:
return None,False,'DatabaseAPI.increment Failed with unknown exception'
def increment_many(self,bucket_id,doc_uid,key_value_dict):
"""
increments a key by amount value. If key does not exist, sets {key:value}
Inputs:
(string) bucket_id, (string) doc_uid, ({(str)key1:(float)value1,(int)key2:(float) value2}) key_value_dict
Outputs:
(bool) didSucceed, (string) message
Usage: ::\n
didSucceed,message = db.increment_many(bucket_id,doc_uid,key_value_dict)
"""
try:
if USE_CACHE:
# need to implement cache!!
#############################
response,dt = utils.timeit(self.permStore.increment_many)(constants.app_data_database_id,bucket_id,doc_uid,key_value_dict)
new_key_value_dict,didSucceed,message = response
self.duration_permStoreSet += dt
if not didSucceed:
return None,False,message
return new_key_value_dict,True,'Hit PermStore'
else:
response,dt = utils.timeit(self.permStore.increment_many)(constants.app_data_database_id,bucket_id,doc_uid,key_value_dict)
new_key_value_dict,didSucceed,message = response
self.duration_permStoreSet += dt
if not didSucceed:
return None,False,message
return new_key_value_dict,True,'Hit PermStore'
except:
return None,False,'DatabaseAPI.increment_many Failed with unknown exception'
def get_list(self,bucket_id,doc_uid,key):
"""
Get a value corresponding to key, returns None if no key exists
Inputs:
(string) bucket_id, (string) doc_uid, (string) key
Outputs:
(list) value, (bool) didSucceed, (string) message
Usage: ::\n
value,didSucceed,message = db.get_list(bucket_id,doc_uid,key)
"""
try:
if USE_CACHE:
# attempts to read file from cache first
response,dt = utils.timeit(self.cacheStore.exists)(constants.app_data_database_id,bucket_id,doc_uid,key)
keyExistsInCacheStore,didSucceed,message = response
self.duration_cacheStoreGet += dt
if not didSucceed:
return None,False,'get.cacheStore.exists(key) failed'
if keyExistsInCacheStore:
response,dt = utils.timeit(self.cacheStore.get_list)(constants.app_data_database_id,bucket_id,doc_uid,key)
value,didSucceed,message = response
self.duration_cacheStoreGet += dt
if not didSucceed:
return None,False,message
return value,True,'From Cache'
else:
response,dt = utils.timeit(self.permStore.get_list)(constants.app_data_database_id,bucket_id,doc_uid,key)
value,didSucceed,message = response
self.duration_permStoreGet += dt
if not didSucceed:
return None,False,message
if value!=None:
didSucceed,message = self.cacheStore.set_list(constants.app_data_database_id,bucket_id,doc_uid,key,value)
if not didSucceed:
return None,False,message
return value,True,'Hit PermStore'
# could not find file
else:
return None,True,'Not in Database'
else:
# not in cache
response,dt = utils.timeit(self.permStore.get_list)(constants.app_data_database_id,bucket_id,doc_uid,key)
value,didSucceed,message = response
self.duration_permStoreGet += dt
if not didSucceed:
return None,False,message
return value,True,'Hit PermStore'
except:
return None,False,'DatabaseAPI.get Failed with unknown exception'
def pop_list(self, bucket_id, doc_uid, key, value):
"""
Inputs:
(string) bucket_id, (string) doc_uid, (string) key, (int) value
value=-1 pops the last item of the list
value=0 pops the first item of the list
Outputs:
(python object) value, (bool) didSucceed, (string) message
Usage: ::\n
didSucceed,message = db.pop_list(bucket_id,doc_uid,key,value)
"""
try:
response, dt = utils.timeit(self.permStore.pop_list)(constants.app_data_database_id,
bucket_id, doc_uid, key, value)
value, didSucceedPerm, messagePerm = response
self.duration_permStoreSet += dt
return value, didSucceedPerm, messagePerm
except Exception as e:
error = "DatabaseAPI.pop_list failed with exception: {}".format(e)
utils.debug_print(error)
return None, False, error
def append_list(self,bucket_id,doc_uid,key,value):
"""
Appends a {key,value_list} (if already exists, replaces)
Inputs:
(string) bucket_id, (string) doc_uid, (string) key, (list) value
Outputs:
(bool) didSucceed, (string) message
Usage: ::\n
didSucceed,message = db.set_list(bucket_id,doc_uid,key,value)
"""
try:
if USE_CACHE:
# attempts to read file from cache first
response,dt = utils.timeit(self.cacheStore.exists)(constants.app_data_database_id,bucket_id,doc_uid,key)
keyExistsInCacheStore,didSucceed,message = response
self.duration_cacheStoreGet += dt
if not didSucceed:
return False,message
if keyExistsInCacheStore:
response,dt = utils.timeit(self.cacheStore.append_list)(constants.app_data_database_id,bucket_id,doc_uid,key,value)
didSucceedCache,message = response
self.duration_cacheStoreSet += dt
response,dt = utils.timeit(self.permStore.append_list)(constants.app_data_database_id,bucket_id,doc_uid,key,value)
didSucceedPerm,messagePerm = response
self.duration_permStoreSet += dt
else:
response,dt = utils.timeit(self.permStore.append_list)(constants.app_data_database_id,bucket_id,doc_uid,key,value)
didSucceedPerm,messagePerm = response
self.duration_permStoreSet += dt
response,dt = utils.timeit(self.permStore.get_list)(constants.app_data_database_id,bucket_id,doc_uid,key)
value_list,didSucceedPerm,message = response
self.duration_permStoreGet += dt
didSucceedCache,message = self.cacheStore.set_list(constants.app_data_database_id,bucket_id,doc_uid,key,value_list)
if not didSucceed:
return False,message
if didSucceedCache and didSucceedPerm:
return True,''
else:
return False,message
else:
response,dt = utils.timeit(self.permStore.append_list)(constants.app_data_database_id,bucket_id,doc_uid,key,value)
didSucceedPerm,messagePerm = response
self.duration_permStoreSet += dt
return didSucceedPerm,messagePerm
except:
error = "DatabaseAPI.append_list Failed with unknown exception"
return False,error
def set_list(self,bucket_id,doc_uid,key,value):
"""
Sets a {key,value_list} (if already exists, replaces)
Inputs:
(string) bucket_id, (string) doc_uid, (string) key, (list) value
Outputs:
(bool) didSucceed, (string) message
Usage: ::\n
didSucceed,message = db.set_list(bucket_id,doc_uid,key,value)
"""
try:
if USE_CACHE:
# writes to cache first
response,dt = utils.timeit(self.cacheStore.set_list)(constants.app_data_database_id,bucket_id,doc_uid,key,value)
didSucceedCache,messageCache = response
self.duration_cacheStoreSet += dt
# then writes to permanent store
response,dt = utils.timeit(self.permStore.set_list)(constants.app_data_database_id,bucket_id,doc_uid,key,value)
didSucceedPerm,messagePerm = response
self.duration_permStoreSet += dt
if didSucceedCache and didSucceedPerm:
return True,''
else:
return False,messageCache + '\n' + messagePerm
else:
response,dt = utils.timeit(self.permStore.set_list)(constants.app_data_database_id,bucket_id,doc_uid,key,value)
didSucceedPerm,messagePerm = response
self.duration_permStoreSet += dt
return didSucceedPerm,messagePerm
except:
error = "DatabaseAPI.set Failed with unknown exception"
return False,error
def set_doc(self,bucket_id,doc_uid,doc):
"""
Sets a document with doc_uid
Inputs:
(dict) doc
Outputs:
None
Usage: ::\n
db.set_doc(key,value)
"""
didSucceed,message = self.permStore.setDoc(constants.app_data_database_id,bucket_id,doc_uid,doc)
return didSucceed,message
def get_doc(self,bucket_id,doc_uid):
"""
Gets doc in bucket_id that corresponds to doc_uid
Inputs:
(string) bucket_id, (string) doc_uid
Outputs:
(dict) doc, (bool) didSucceed, (string) message
Usage: ::\n
doc,didSucceed,message = db.getDoc(bucket_id,doc_uid)
"""
return self.permStore.getDoc(constants.app_data_database_id,bucket_id,doc_uid)
def get_docs_with_filter(self,bucket_id,pattern_dict):
"""
Retrieves all docs in bucket_id that match (i.e. contain) pattern_dict
Inputs:
(string) bucket_id, (dict of string values) pattern_dict
Outputs:
(list of dict) docs, (bool) didSucceed, (string) message
Usage: ::\n
docs,didSucceed,message = db.getDocsByPattern(bucket_id,pattern_dict)
"""
t = self.permStore.getDocsByPattern(constants.app_data_database_id,bucket_id,pattern_dict)
return t
def set(self,bucket_id,doc_uid,key,value):
"""
Sets a {key,value} (if already exists, replaces)
Inputs:
(string) bucket_id, (string) doc_uid, (string) key, (string) value
Outputs:
(bool) didSucceed, (string) message
Usage: ::\n
didSucceed,message = db.set(bucket_id,doc_uid,key,value)
"""
try:
if USE_CACHE:
# writes to cache first
response,dt = utils.timeit(self.cacheStore.set)(constants.app_data_database_id,bucket_id,doc_uid,key,value)
didSucceedCache,messageCache = response
self.duration_cacheStoreSet += dt
# then writes to permanent store
response,dt = utils.timeit(self.permStore.set)(constants.app_data_database_id,bucket_id,doc_uid,key,value)
didSucceedPerm,messagePerm = response
self.duration_permStoreSet += dt
if didSucceedCache and didSucceedPerm:
return True,''
else:
return False,messageCache + '\n' + messagePerm
else:
response,dt = utils.timeit(self.permStore.set)(constants.app_data_database_id,bucket_id,doc_uid,key,value)
didSucceedPerm,messagePerm = response
self.duration_permStoreSet += dt
return didSucceedPerm,messagePerm
except:
error = "DatabaseAPI.set Failed with unknown exception"
return False,error
def set_many(self,bucket_id,doc_uid,key_value_dict):
"""
sets key, values in dict. If key does not exist, sets {key:value}
Inputs:
(string) bucket_id, (string) doc_uid, ({(str)key1:(float)value1,(int)key2:(float) value2}) key_value_dict
Outputs:
(bool) didSucceed, (string) message
Usage: ::\n
didSucceed,message = db.set_many(bucket_id,doc_uid,key_value_dict)
"""
try:
if USE_CACHE:
# need to implement cache!!
#############################
response,dt = utils.timeit(self.permStore.set_many)(constants.app_data_database_id,bucket_id,doc_uid,key_value_dict)
didSucceed,message = response
self.duration_permStoreSet += dt
if not didSucceed:
return False,message
return True,'Hit PermStore'
else:
response,dt = utils.timeit(self.permStore.set_many)(constants.app_data_database_id,bucket_id,doc_uid,key_value_dict)
didSucceed,message = response
self.duration_permStoreSet += dt
if not didSucceed:
return False,message
return True,'Hit PermStore'
except:
return None,False,'DatabaseAPI.set_many Failed with unknown exception'
def delete(self,bucket_id,doc_uid,key):
"""
Deletes {key:value} associated with given key
Inputs:
(string) bucket_id, (string) doc_uid, (string) key
Outputs:
(bool) didSucceed, (string) message
Usage: ::\n
didSucceed,message = db.delete(bucket_id,doc_uid,key)
"""
try:
if USE_CACHE:
# attempts to delete file from cache first
didSucceed,message = self.cacheStore.delete(constants.app_data_database_id,bucket_id,doc_uid,key)
if not didSucceed:
return False,'DatabaseAPI.cacheStore.delete(key) failed'
# attempts to delete from permanent store
didSucceed,message = self.permStore.delete(constants.app_data_database_id,bucket_id,doc_uid,key)
if not didSucceed:
return False,'DatabaseAPI.permStore.delete(key) failed'
return True,''
else:
didSucceed,message = self.permStore.delete(constants.app_data_database_id,bucket_id,doc_uid,key)
return didSucceed,message
except:
error = "DatabaseAPI.delete Failed with unknown exception"
return False,error
def ensure_index(self,bucket_id,index_dict):
"""
Adds index defined on index_dict to bucket_id
Inputs:
(string) bucket_id, (dict) index_dict
Outputs:
(string) index_info, (bool) didSucceed, (string) message
Usage: ::\n
didSucceed,message = db.get_index_info('rand_data',{'num_eyes':1,'exp_uid',1})
"""
didSucceed,message = self.permStore.create_index(constants.app_data_database_id,bucket_id,index_dict)
return didSucceed,message
def drop_all_indexes(self,bucket_id):
"""
Deletes all indexes defined on bucket_id
Inputs:
(string) bucket_id
Outputs:
(bool) didSucceed, (string) message
Usage: ::\n
didSucceed,message = db.drop_all_indexes(bucket_id)
"""
didSucceed,message = self.permStore.drop_all_indexes(constants.app_data_database_id,bucket_id)
return didSucceed,message
def delete_docs_with_filter(self,bucket_id,pattern_dict):
"""
Deletes all docs in bucket_id that match (i.e. contain) pattern_dict
Inputs:
(string) bucket_id, (dict of string values) pattern_dict
Outputs:
(bool) didSucceed, (string) message
Usage: ::\n
didSucceed,message = db.deleteDocsByPattern(bucket_id,key,value)
"""
docs,didSucceed,message = self.permStore.getDocsByPattern(constants.app_data_database_id,bucket_id,pattern_dict)
for doc in docs:
try:
doc_uid = doc['doc_uid']
didSucceed,message = self.cacheStore.deleteDoc(constants.app_data_database_id,bucket_id,doc_uid)
except:
pass
return self.permStore.deleteDocsByPattern(constants.app_data_database_id,bucket_id,pattern_dict)
def log(self,bucket_id,log_dict):
"""
Saves log_dict to PermStore as an individual document for later recall.
Inputs:
(string) bucket_id, (dict with string values) log_dict
Outputs:
(bool) didSucceed, (string) message
Usage: ::\n
didSucceed,message = db.log(bucket_id,doc_uid)
"""
timestamp = utils.datetimeNow()
log_dict.update({ 'timestamp':timestamp })
return self.permStore.setDoc(constants.logs_database_id,bucket_id,None,log_dict)
def get_logs_with_filter(self,bucket_id,pattern_dict):
"""
Retrieves all logs in bucket_id that match (i.e. contain) pattern_dict
Inputs:
(string) bucket_id, (dict of string values) pattern_dict
Outputs:
(list of dict) logs, (bool) didSucceed, (string) message
Usage: ::\n
logs,didSucceed,message = db.getLogsByPattern(bucket_id,pattern_dict)
"""
return self.permStore.getDocsByPattern(constants.logs_database_id,bucket_id,pattern_dict)
def delete_logs_with_filter(self,bucket_id,pattern_dict):
"""
Deletes all logs in bucket_id that match (i.e. contain) pattern_dict
Inputs:
(string) bucket_id, (dict of string values) pattern_dict
Outputs:
(bool) didSucceed, (string) message
Usage: ::\n
didSucceed,message = db.deleteLogsByPattern(bucket_id,key,value)
"""
return self.permStore.deleteDocsByPattern(constants.logs_database_id,bucket_id,pattern_dict)
def assertConnection(self):
"""
Asserts that the API has successfully connected to both the CacheStore and PermStore
Inputs:
None
Outputs:
(bool) areConnected, (string) message
Usage: ::\n
didSucceed,message = db.assertConnection()
"""
try:
if USE_CACHE:
cacheStoreConnected = self.cacheStore.assertConnection()
permStoreConnected = self.permStore.assertConnection()
if (cacheStoreConnected) and (permStoreConnected):
return True,''
elif (not cacheStoreConnected) and (not permStoreConnected):
return False,'Failed to connect to CacheStore and PermStore'
elif (not cacheStoreConnected) and (permStoreConnected):
return False,'Failed to connect to CacheStore'
elif (cacheStoreConnected) and (not permStoreConnected):
return False,'Failed to connect to PermStore'
else:
permStoreConnected = self.permStore.assertConnection()
if (permStoreConnected):
return True,'Cache dsiabled'
else:
return False,'Cache disabled, no connection from Permstore'
except:
error = "DatabaseAPI.assertConnection unknown exception"
return False, error
def flushDocCache(self,bucket_id,doc_uid):
"""
Deletes all app data in the Cache corresponding to bucket_id,doc_uid,
Note: This does NOT delete anything in PermStore (i.e. this is a safe operation)
Inputs:
(string) bucket_id, (string) doc_uid
Outputs:
(bool) didSucceed, (string) message
Usage: ::\n
didSucceed,message = db.flushDocCache(bucket_id,doc_uid)
"""
# delete everything in cache having to do with experiment
# get alg_list if it exists and delete all children to exp_uid
algListExists,didSucceed,message = self.exists(bucket_id,doc_uid,'alg_list')
if not didSucceed:
return False,message
if algListExists:
alg_list,didSucceed,message = self.get(bucket_id,doc_uid,'alg_list')
if not didSucceed:
return False,message
exp_uid,didSucceed,message = self.get(bucket_id,doc_uid,'exp_uid')
if not didSucceed:
return False,message
for alg_id in alg_list:
child_doc_uid = utils.getDocUID(exp_uid,alg_id)
didSucceed,message = self.cacheStore.deleteDoc(constants.app_data_database_id,bucket_id,child_doc_uid)
if not didSucceed:
return False,message
didSucceed,message = self.cacheStore.deleteDoc(constants.app_data_database_id,bucket_id,doc_uid)
if not didSucceed:
return False,message
return True,''
def flushCache(self):
"""
Deletes everything from the Cache.
Note: This does NOT delete anything in PermStore (i.e. this is a safe operation)
Inputs:
None
Outputs:
(bool) didSucceed, (string) message
Usage: ::\n
didSucceed,message = db.flushCache(bucket_id,doc_uid)
"""
return self.cacheStore.deleteAll()
def irreversibly_delete_everything(self):
if USE_CACHE:
self.cacheStore.deleteAll()
self.permStore.deleteAll()
def getDocNames(self,bucket_id):
"""
Get list of doc_uids correspding to all the docs in the bucket corresponding to the given bucket_id
Inputs:
(string) bucket_id
Outputs:
([(string) doc_uid, ... ]) docNames, (bool) didSucceed, (string) message
Usage: ::\n
docNames,didSucceed,message = db.getDocNames(bucket_id)
"""
try:
doc_uids,didSucceed,message = self.permStore.getDocNames(constants.app_data_database_id,bucket_id)
if not didSucceed:
return None,False,message
return doc_uids,True,''
except:
error = "DatabaseAPI.getDocNames Failed with unknown exception"
return None,False,error
def getBucketNames(self):
"""
Get list of bucket_ids
Inputs:
None
Outputs:
([(string) bucket_id, ... ]) bucketNames, (bool) didSucceed, (string) message
Usage: ::\n
bucketNames,didSucceed,message = db.getBucketNames()
"""
try:
bucket_ids,didSucceed,message = self.permStore.getBucketNames(constants.app_data_database_id)
if not didSucceed:
return None,False,message
return bucket_ids,True,''
except:
error = "DatabaseAPI.getDocNames Failed with unknown exception"
return None,False,error
def inspectDatabase(self):
"""
Returns string describing the entire app data database
Inputs:
None
Outputs:
(string) description, (bool) didSucceed, (string) message
Usage: ::\n
didSucceed,message = db.inspectDatabase()
"""
try:
output_str = ''
bucket_ids,didSucceed,message = self.permStore.getBucketNames(constants.app_data_database_id)
for bucket_id in bucket_ids:
bucket_id_str = str(bucket_id)
output_str += bucket_id_str + " { \n"
doc_uids,didSucceed,message = self.permStore.getDocNames(constants.app_data_database_id,bucket_id)
if doc_uids!=None :
for doc_uid in doc_uids:
doc_uid_str = str(doc_uid)
output_str += "\t" + doc_uid_str + " { \n"
doc,didSucceed,message = self.permStore.getDoc(constants.app_data_database_id,bucket_id,doc_uid)
for key in doc:
pickled_value = doc[key]
try:
value = cPickle.loads(pickled_value)
except:
value = pickled_value
str_value = str(value)
if len(str_value)>constants.maxStringLengthInInspectDatabase:
str_value = str_value[0:constants.maxStringLengthInInspectDatabase]+ " ..."
output_str += "\t\t" + key + " : " + str_value + "\n"
output_str += "\t" + "}" + "\n"
output_str += "}" + "\n"
return output_str,True,''
except:
error = "DatabaseAPI.inspectDatabase() incurred an unknown exception "
return None,False,error
def inspectDoc(self,bucket_id,doc_uid):
"""
Returns string describing the document of the particular bucket_id, doc_uid
Inputs:
(string) bucket_id, (string) doc_uid
Outputs:
(string) description, (bool) didSucceed, (string) message
Usage: ::\n
didSucceed,message = db.inspectDatabase()
"""
try:
output_str = ''
output_str += doc_uid + " { \n"
doc,didSucceed,message = self.permStore.getDoc(constants.app_data_database_id,bucket_id,doc_uid)
for key in doc:
pickled_value = doc[key]
try:
value = cPickle.loads(pickled_value)
except:
value = pickled_value
str_value = str(value)
if len(str_value)>constants.maxStringLengthInInspectDatabase:
str_value = str_value[0:max_value_string_length]+ " ..."
output_str += "\t" + key + " : " + str_value + "\n"
output_str += "}" + "\n"
return output_str,True,''
except:
return None,False,'DatabaseAPI.inspectDoc unknown exception'
class LoggerAPI(object):
"""
Serves as an API object that can be passed around. See above for usage
Attributes:
permStore : PermStore object
"""
def __init__(self):
self.permStore = PermStore()
def log(self, bucket_id, log_dict):
"""
Saves log_dict to PermStore as an individual document for later recall.
Inputs:
(string) bucket_id, (dict with string values) log_dict
Outputs:
(bool) didSucceed, (string) message
Usage: ::\n
didSucceed,message = db.log(bucket_id,doc_uid)
"""
return self.permStore.setDoc(constants.logs_database_id, bucket_id, None, log_dict)
def ensure_index(self, bucket_id, index_dict):
"""
Adds index defined on index_dict to bucket_id
Inputs:
(string) bucket_id, (dict) index_dict
Outputs:
(string) index_info, (bool) didSucceed, (string) message
Usage: ::\n
didSucceed,message = db.get_index_info('rand_data',{'num_eyes':1,'exp_uid',1})
"""
didSucceed, message = self.permStore.create_index(constants.logs_database_id, bucket_id, index_dict)
return didSucceed, message
def drop_all_indexes(self, bucket_id):
"""
Deletes all indexes defined on bucket_id
Inputs:
(string) bucket_id
Outputs:
(bool) didSucceed, (string) message
Usage: ::\n
didSucceed,message = db.drop_all_indexes(bucket_id)
"""
didSucceed, message = self.permStore.drop_all_indexes(constants.logs_database_id, bucket_id)
return didSucceed, message
def get_logs_with_filter(self, bucket_id, pattern_dict):
"""
Retrieves all logs in bucket_id that match (i.e. contain) pattern_dict
Inputs:
(string) bucket_id, (dict of string values) pattern_dict
Outputs:
(list of dict) logs, (bool) didSucceed, (string) message
Usage: ::\n
logs,didSucceed,message = db.getLogsByPattern(bucket_id,pattern_dict)
"""
return self.permStore.getDocsByPattern(constants.logs_database_id, bucket_id, pattern_dict)
def delete_logs_with_filter(self, bucket_id, pattern_dict):
"""
Deletes all logs in bucket_id that match (i.e. contain) pattern_dict
Inputs:
(string) bucket_id, (dict of string values) pattern_dict
Outputs:
(bool) didSucceed, (string) message
Usage: ::\n
didSucceed,message = db.deleteLogsByPattern(bucket_id,key,value)
"""
return self.permStore.deleteDocsByPattern(constants.logs_database_id, bucket_id, pattern_dict)
def assertConnection(self):
"""
Asserts that the API has successfully connected to the PermStore
Inputs:
None
Outputs:
(bool) areConnected, (string) message
Usage: ::\n
didSucceed,message = db.assertConnection()
"""
permStoreConnected = self.permStore.assertConnection()
if permStoreConnected:
return True, ""
else:
return False, "no connection from Permstore"
def irreversibly_delete_everything(self):
self.permStore.deleteAll()