class BenchCodernityDB(BenchBase):
ID_FIELD = "_id"
def __init__(self, *args, **kwargs):
super(BenchCodernityDB, self).__init__(*args, **kwargs)
def create_database(self):
self.db = Database(self.db_name)
self.db.create()
self.db.add_index(WithSmallNumberIndex(self.db.path, "small_number"))
def delete_database(self):
self.db.close()
shutil.rmtree(self.db_name)
def create(self, record):
self.db.insert(record)
def get(self, key):
return self.db.get("id", key, with_doc=True)
def query(self, **kwargs):
key, val = kwargs.items()[0]
return list(self.db.get_many(key, val, limit=-1, with_doc=True))
def regenerate_indexes(temp_dir):
"""
"""
tmpdb = Database(temp_dir)
tmpdb.custom_header = coins_index.make_custom_header()
tmpdb.create()
refresh_indexes(tmpdb)
tmpdb.close()
return tmpdb
class cache:
"""
cache for word morphological analysis
"""
def __init__(self, ):
"""
Create Analex Cache
"""
self.cache = {
'checkedWords': {},
'FreqWords': {
'noun': {},
'verb': {},
'stopword': {}
},
}
self.db = Database('/tmp/qalsadiCache')
if not self.db.exists():
self.db.create()
x_ind = WithAIndex(self.db.path, 'a')
self.db.add_index(x_ind)
else:
self.db.open()
def __del__(self):
"""
Delete instance and clear cache
"""
self.cache = None
self.db.close()
def isAlreadyChecked(self, word):
try:
return bool(self.db.get('a', word))
except:
return False
#~ except: return False;
def getChecked(self, word):
x = self.db.get('a', word, with_doc=True)
y = x.get('doc', False)
if y: return y.get('d', [])
else: return []
def addChecked(self, word, data):
idata = {"a": word, 'd': data}
self.db.insert(idata)
def existsCacheFreq(self, word, wordtype):
return word in self.cache['FreqWords']
def getFreq(self, originalword, wordtype):
return self.cache['FreqWords'][wordtype].get(originalword, 0)
def addFreq(self, original, wordtype, freq):
self.cache['FreqWords'][wordtype][original] = freq
class DBImport:
'''
import scan: scans existing self.db and rebuilds config file
create self.db: creates self.db file, master index, question index and table index
'''
def __init__(self,passkey,xtraDB):
self.key = passkey
self.dbName = xtraDB
self.db=Database(self.dbName)
self.importScan()
def __del__(self):
if (self.db.opened):
self.db.close()
# ADD REBUILD OPTION
def importScan(self):
#read from config, as a check
self.db=Database(self.dbName)
if(self.db.exists()):
self.db.open()
self.db.id_ind.enc_key = self.key
for curr in self.db.all('id'): #since first passkey in self.db should be only one there, function only perfomed once
if curr['t'] == 'master':
masterKey=''.join(curr['_id'])
self.DBConfig = AppConfig()
self.DBConfig.putmap('databaseinfo','indexkey',masterKey)#masterkey=value
self.DBConfig.putmap('databaseinfo','databasename',self.dbName)
break
#add else statement for errors if couldnt be written for found
self.db.close()
return True
class cache :
"""
cache for word morphological analysis
"""
def __init__(self,):
"""
Create Analex Cache
"""
self.cache={'checkedWords':{},
'FreqWords':{'noun':{}, 'verb':{},'stopword':{}},
};
self.db = Database('/tmp/qalsadiCache')
if not self.db.exists():
self.db.create();
x_ind = WithAIndex(self.db.path, 'a')
self.db.add_index(x_ind)
else:
self.db.open();
def __del__(self):
"""
Delete instance and clear cache
"""
self.cache=None;
self.db.close();
def isAlreadyChecked(self, word):
try:
return bool(self.db.get('a', word))
except: return False
#~ except: return False;
def getChecked(self, word):
x = self.db.get('a', word, with_doc=True)
y= x.get('doc',False);
if y: return y.get('d',[])
else: return []
def addChecked(self, word, data):
idata = {"a":word,'d':data}
self.db.insert(idata)
def existsCacheFreq(self, word, wordtype):
return word in self.cache['FreqWords'];
def getFreq(self, originalword, wordtype):
return self.cache['FreqWords'][wordtype].get(originalword,0);
def addFreq(self, original, wordtype, freq):
self.cache['FreqWords'][wordtype][original]=freq;
def migrate(source, destination):
"""
Very basic for now
"""
dbs = Database(source)
dbt = Database(destination)
dbs.open()
dbt.create()
dbt.close()
for curr in os.listdir(os.path.join(dbs.path, "_indexes")):
if curr != "00id.py":
shutil.copyfile(os.path.join(dbs.path, "_indexes", curr), os.path.join(dbt.path, "_indexes", curr))
dbt.open()
for c in dbs.all("id"):
del c["_rev"]
dbt.insert(c)
return True
class DBImport:
'''
import scan: scans existing self.db and rebuilds config file
create self.db: creates self.db file, master index, question index and table index
'''
def __init__(self, passkey, xtraDB):
self.key = passkey
self.dbName = xtraDB
self.db = Database(self.dbName)
self.importScan()
def __del__(self):
if (self.db.opened):
self.db.close()
# ADD REBUILD OPTION
def importScan(self):
#read from config, as a check
self.db = Database(self.dbName)
if (self.db.exists()):
self.db.open()
self.db.id_ind.enc_key = self.key
for curr in self.db.all(
'id'
): #since first passkey in self.db should be only one there, function only perfomed once
if curr['t'] == 'master':
masterKey = ''.join(curr['_id'])
self.DBConfig = AppConfig()
self.DBConfig.putmap('databaseinfo', 'indexkey',
masterKey) #masterkey=value
self.DBConfig.putmap('databaseinfo', 'databasename',
self.dbName)
break
#add else statement for errors if couldnt be written for found
self.db.close()
return True
def main():
db = Database('/tmp/demo_secure')
key = 'abcdefgh'
id_ind = EncUniqueHashIndex(db.path, 'id', storage_class='Salsa20Storage')
db.set_indexes([id_ind])
db.create()
db.id_ind.enc_key = key
for x in xrange(100):
db.insert(dict(x=x, data='testing'))
db.close()
dbr = Database('/tmp/demo_secure')
dbr.open()
dbr.id_ind.enc_key = key
for curr in dbr.all('id', limit=5):
print curr
def main():
db = Database('/tmp/demo_secure')
key = 'abcdefgh'
id_ind = EncUniqueHashIndex(db.path, 'id', storage_class='Salsa20Storage')
db.set_indexes([id_ind])
db.create()
db.id_ind.enc_key = key
for x in xrange(100):
db.insert(dict(x=x, data='testing'))
db.close()
dbr = Database('/tmp/demo_secure')
dbr.open()
dbr.id_ind.enc_key = key
for curr in dbr.all('id', limit=5):
print curr
class Developer:
def __init__(self, passkey, dbname=None):
self.key = passkey
if (dbname == None):
self.DBConfig = AppConfig()
self.dbName = self.DBConfig.mapget('databaseinfo')['databasename']
else:
self.dbName = dbname
self.db = Database(self.dbName)
def dump(self):
if (self.db.exists()):
self.db.open()
self.db.id_ind.enc_key = self.key
for curr in self.db.all('id'):
print curr
self.db.close()
def migrate(source, destination):
"""
Very basic for now
"""
dbs = Database(source)
dbt = Database(destination)
dbs.open()
dbt.create()
dbt.close()
for curr in os.listdir(os.path.join(dbs.path, '_indexes')):
if curr != '00id.py':
shutil.copyfile(os.path.join(dbs.path, '_indexes', curr),
os.path.join(dbt.path, '_indexes', curr))
dbt.open()
for c in dbs.all('id'):
del c['_rev']
dbt.insert(c)
return True
def main():
db = Database("/tmp/demo_secure")
key = "abcdefgh"
id_ind = EncUniqueHashIndex(db.path, "id")
db.set_indexes([id_ind])
db.create()
db.id_ind.enc_key = key
print db.id_ind.storage
for x in xrange(100):
db.insert(dict(x=x, data="testing"))
db.close()
dbr = Database("/tmp/demo_secure")
dbr.open()
dbr.id_ind.enc_key = key
for curr in dbr.all("id", limit=5):
print curr
def CreateDB(self):
'''
创建nosql数据库
:param dbName:
:return:
'''
db = Database(os.path.join(self.__dbRoot,self.__dbName))
if db.exists():
return True,"DB Exist"
try:
# OutPutHelper.consolePrint("Create DB=%s, dbpath=%s"% (self.__dbName ,db.create()))
db.create()
# if indexname:
# x_ind = WithXIndex(db.path, indexname)
# db.add_index(x_ind)
except Exception as ex:
return False,"Create DB Failed"
db.close()
return True ,"Success"
pass
def recreate_db(chat_history_dir):
"""
"""
global _LocalStorage
try:
_LocalStorage.close()
except Exception as exc:
lg.warn('failed closing local storage : %r' % exc)
_LocalStorage = None
dbs = Database(chat_history_dir)
dbs.custom_header = message_index.make_custom_header()
temp_dir = os.path.join(settings.ChatHistoryDir(), 'tmp')
if os.path.isdir(temp_dir):
bpio._dir_remove(temp_dir)
orig_dir = os.path.join(settings.ChatHistoryDir(), 'orig')
if os.path.isdir(orig_dir):
bpio._dir_remove(orig_dir)
dbt = Database(temp_dir)
dbt.custom_header = message_index.make_custom_header()
source_opened = False
try:
dbs.open()
source_opened = True
except Exception as exc:
lg.warn('failed open local storage : %r' % exc)
# patch_flush_fsync(dbs)
dbt.create()
dbt.close()
refresh_indexes(dbt, reindex=False)
dbt.open()
# patch_flush_fsync(dbt)
if source_opened:
for c in dbs.all('id'):
del c['_rev']
dbt.insert(c)
dbt.close()
if source_opened:
dbs.close()
os.rename(dbs.path, orig_dir)
os.rename(dbt.path, dbs.path)
_LocalStorage = Database(chat_history_dir)
_LocalStorage.custom_header = message_index.make_custom_header()
db().open()
# patch_flush_fsync(db())
if refresh_indexes(db(), rewrite=False, reindex=False):
bpio._dir_remove(orig_dir)
lg.info('local DB re-created in %r' % chat_history_dir)
else:
lg.err('local DB is broken !!!')
class Cache(object):
"""
cache for word morphological analysis
"""
DB_PATH = os.path.join(os.path.expanduser('~'), '.qalsadiCache')
def __init__(self, dp_path = False):
"""
Create Analex Cache
"""
self.cache = {
'checkedWords': {},
'FreqWords': {
'noun': {},
'verb': {},
'stopword': {}
},
}
if not dp_path:
dp_path = self.DB_PATH
else:
dp_path = os.path.join(os.path.dirname(dp_path), '.qalsadiCache')
self.db = Database(dp_path)
if not self.db.exists():
self.db.create()
x_ind = WithAIndex(self.db.path, 'a')
self.db.add_index(x_ind)
else:
self.db.open()
def __del__(self):
"""
Delete instance and clear cache
"""
self.cache = None
self.db.close()
def is_already_checked(self, word):
""" return if ``word`` is already cached"""
try:
return bool(self.db.get('a', word))
except:
return False
#~ except: return False;
def get_checked(self, word):
""" return checked ``word`` form cache"""
xxx = self.db.get('a', word, with_doc=True)
yyy = xxx.get('doc', False)
if yyy:
return yyy.get('d', [])
else: return []
def add_checked(self, word, data):
""" add checked ``word`` form cache"""
idata = {"a": word, 'd': data}
self.db.insert(idata)
def exists_cache_freq(self, word, wordtype):
""" return if word exists in freq cache"""
return word in self.cache['FreqWords']
def get_freq(self, originalword, wordtype):
""" return ``word`` frequency form cache"""
return self.cache['FreqWords'][wordtype].get(originalword, 0)
def add_freq(self, original, wordtype, freq):
""" add ``original`` frequency ``freq`` to cache"""
self.cache['FreqWords'][wordtype][original] = freq
class SecuIn:
'''
Handles all data input into the database
'''
def __init__(self,passkey):
self.key = passkey
self.initQuestions = SecuQ(self.key)
self.DBConfig = AppConfig()
self.dbName = self.DBConfig.mapget('databaseinfo')['databasename']
self.db = Database(self.dbName)
initDay = DayEntry(self.key) # checks day hash or creates a new one
self.dayKey = initDay.dayKey
def questionDataIN(self,data):
'''
Data IN:
{'a' : 2, 'b': 14 , 'c': 11, 'd': 43, 'note' : 'hello'}
or
{ 'b': 14 , 'c': 11, 'd': 43, 'note' : 'hello'}
some entries may be missing
Data OUT: (NEVER DELETE ANYTIHNG :) )
{'date' : xx , _id: ###date2### , 'a':{'xxdate3xx':2},
'b':{'xxdate3xx':14},
'c':{'xxdate3xx':11},
'note':{'xxdate3xx':'you'}}
{'date' : xx , _id: ###date1### , 'a':{'xxdate1xx':1,'xxdate2xx':2},
'b':{'xxdate1xx':14,'xxdate2xx':14},
'c':{'xxdate1xx':11,'xxdate2xx':11},
'note':{'xxdate2xx':'hello','xxdate3xx':'you'}
'''
timeIN = getTimeStamp() #get now time
#initialize new questions
# get data, as doc {'date':'xx/xx/xxTxx:xx:xxxx','question1':'x','question2':'x'}, same as dic format
if(self.db.exists()):
self.db.open()
self.db.id_ind.enc_key = self.key
dayrow = self.db.get('id', self.dayKey, with_doc=True)
#this function assumes database already opened
# this is gonna be a tuple that is inserted directly
#convert data from javasript to python dict/json
# if (type(data) is str):
dataIN=eval(data) #{ 'b': 14 , 'c': 11, 'd': 43, 'note' : 'hello'}
datachanged = dataIN.keys()
for question in datachanged:
try:
dayrow[question][timeIN] = dataIN[question]
except KeyError: #first write to key, initiate
dayrow[question] = {}
dayrow[question][timeIN] = dataIN[question]
self.db.update(dayrow)
self.db.close()
self.initQuestions.questionsValidate(datachanged) #insert questions whos data had changed
#if all ok!
return True
class cache :
"""
cache for word morphological analysis
"""
DB_PATH = os.path.join(os.path.expanduser('~'), '.thaalabCache')
def __init__(self, cache_path=False):
"""
Create Analex Cache
"""
# use this dictionary as a local cache,
# The global db will be updated on destructing object
# get the database path
if hasattr(sys, 'frozen'): # only when running in py2exe this exists
base = sys.prefix
else: # otherwise this is a regular python script
base = os.path.dirname(os.path.realpath(__file__))
if not cache_path:
file_path = self.DB_PATH
else:
file_path = os.path.join(os.path.dirname(cache_path), '.thaalabCache')
self.cache={};
self.db = Database(file_path)
if not self.db.exists():
self.db.create();
x_ind = WithAIndex(self.db.path, 'a')
self.db.add_index(x_ind)
else:
self.db.open();
def __del__(self):
"""
Delete instance and clear cache
"""
self.cache=None;
self.db.close();
def update(self):
"""update data base """
#~ pass
for word in self.cache:
self.add_checked(word, self.cache[word])
def is_already_checked(self, word):
try:
return bool(self.db.get('a', word))
except:
return False
#~ except: return False;
def get_checked(self, word):
try:
x = self.db.get('a', word, with_doc=True)
y = x.get('doc',False);
if y:
return y.get('d',[])
else: return []
except:
return []
def add_checked(self, word, data):
idata = {"a":word,'d':data}
try:
saved = self.db.get('a', word, with_doc=True)
except:
saved = False
if saved:
saved['doc']['d'] = data
doc = saved['doc']
doc['update'] = True
self.db.update(doc)
else:
self.db.insert(idata)
def exists_cache_word(self, word):
""" test if word exists in cache"""
#if exists in cache dictionary
if word in self.cache:
return True
else: # test in database
if self.is_already_checked(word):
stored_data = self.get_checked(word)
self.cache[word] = stored_data
return bool(self.cache[word])
else:
# add null dict to the word index to avoid multiple database check
self.cache[word] = {}
return {}
def get_relation_freq(self, word_prev, word_cur, relation):
self.exists_cache_word(word_prev)
return self.cache.get(word_prev, {}).get(word_cur, {}).get(relation, 0);
def is_related(self, word_prev, word_cur):
""" test if two words are related"""
#serach in cache
self.exists_cache_word(word_prev)
# if exists in cache or database
return self.cache.get(word_prev, {}).get(word_cur, {});
def add_relation(self, word_prev, word_cur, relation):
#~ relation ='r'+str(relation)
if word_prev not in self.cache:
# test first that is in db cache
if self.is_already_checked(word_prev):
stored_data = self.get_checked(word_prev)
self.cache[word_prev] = stored_data
else: # create an new entry
self.cache[word_prev] = {word_cur:{relation:1, }, }
# word_prev exists
# add word_cur to previous dict
elif word_cur not in self.cache[word_prev]:
self.cache[word_prev][word_cur] = {relation:1,}
elif relation not in self.cache[word_prev][word_cur]:
self.cache[word_prev][word_cur][relation] = 1
else:
self.cache[word_prev][word_cur][relation] += 1
def display_all(self):
""" display all contents of data base """
#~ pass
print "aranasyn.cache: dislay all records in Thaalib Database """
for curr in self.db.all('a', with_doc=True):
print curr['doc']['a'], arepr(curr['doc']['d'])
class SecuQ:
'''
Abstract:
Handles all Questions, input and output input into the database
def questionGet(self):
- reloads all question database attributes
def questionInsert(self,data,descriptor='inclusive'):
-> data = input question dict
ex:
'a':{'active':'True','typ':'slider','range':'0-100','aggregate':True, 'multipoint':True}, ,,,,.................
- inclusive = Updates entried (overwrites) (safe)
- exclusive = deletes all entried not in input (dangerous)
def questionsValidate(self,data):
-> data = input list of questions [(keys)]
each question verified, if not initiated, will be initiated after
'''
def __init__(self, passkey):
self.key = passkey
self.indexdb = DBIndexSystem(self.key)
#self.indexdb.masterIndex
#self.indexdb.Qindex
#self.indexdb.Tindex
#self.indexdb.IndexedTable
#self.indexdb.dbName
self.db = Database(self.indexdb.dbName)
#init variables blank, avoid key errors
self.all = {}
self.valid = {}
self.active = {}
self.notactive = {}
self.unInit = {}
self.typ = {}
self.aggregate = {}
self.multipoint = {}
self.questionGet() #populate variables
#query: all , valid, true,unInit, false, inline
def questionGet(
self
): #the passkey in question should be loaded from config or index passkey
if (self.db.exists()):
self.db.open()
self.db.id_ind.enc_key = self.key
#select Qindex
Qindex = self.db.get('id', self.indexdb.Qindex, with_doc=True)
oQ = Qindex.copy()
# delete unnessesary to modify
# if u use del <key> it deletes all instances in all variables using same reference to dict
oQ.pop('_rev', None)
oQ.pop('_id', None)
oQ.pop('t', None)
#oQ.pop('questions', None)
#returns list in string for of all keys aka the question and metadata
#step through and assign
questionsSet = oQ.keys()
for question in questionsSet:
self.all[question] = oQ[question]
if oQ[question]['active'] == 'True':
self.active[question] = oQ[question]
if oQ[question]['active'] == 'unInit':
self.unInit[question] = oQ[question]
if (oQ[question]['active']
== 'unInit') | (oQ[question]['active'] == 'True'):
self.valid[question] = oQ[question]
if oQ[question]['active'] == 'False':
self.notactive[question] = oQ[question]
if oQ[question]['typ'] == 'note':
self.typ[question] = oQ[question]
try:
if oQ[question]['aggregate'] == 'True':
self.aggregate[question] = oQ[question]
except KeyError:
pass
try:
if oQ[question]['multipoint'] == 'True':
self.multipoint[question] = oQ[question]
except KeyError:
pass
self.db.close()
return True
'''
Qinfo=
{
'a':{'active':'True','typ':'slider','range':'0-100','aggregate':True, 'multipoint':True},
'b':{'active':'True','typ':'slider','range':'0-100','aggregate':True, 'multipoint':False},
'c':{'active':'True','typ':'slider','range':'0-100','aggregate':False, 'multipoint':True},
'd':{'active':'True','typ':'slider','range':'0-100','aggregate':False, 'multipoint':False},
'note':{'active':'True','typ':'note', 'multipoint':"False"}
}
'''
def questionInsert(self,
data,
descriptor='inclusive'
): # this will be a class later for ... infinite data
if (self.db.exists()):
self.db.open()
self.db.id_ind.enc_key = self.key
#select Qindex
Qindex = self.db.get('id', self.indexdb.Qindex, with_doc=True)
#must copy in this way for dictionaries or else all references are affected
oQ = Qindex.copy()
# delete unnessesary to modify
# if u use del <key> it deletes all instances in all variables using same reference to dict
oQ.pop('_rev', None)
oQ.pop('_id', None)
oQ.pop('t', None)
oQ.pop('questions', None)
# if (type(data) is str):
nQL = eval(str(data))
if (descriptor == "exclusive"):
#exclusive, new data always overwrites old data, deletes any data that is not new
# remmove old keys from row
for key in oQ.keys(
): #removes keys not in entry and overwrites everything
if key not in nQL.keys():
Qindex.pop(key, None)
if (descriptor == "inclusive"):
#only overwrites data, keeps old data that is unnaffected
pass
#oQ.update(nQL) # update existing keys to be written
Qindex.update(nQL) #updates existing keys in row
self.db.update(Qindex) #updates NoSQL
self.db.close()
self.questionGet()
return True
else:
print('CANNOT LOAD self.db')
return False
def questionsValidate(
self,
data): #turns all uesd questions true, insert list of questions
for question in data:
if question in self.unInit.keys():
updated = {}
updated[question] = self.unInit[question]
updated[question]['active'] = "True"
self.questionInsert(str(updated), "inclusive")
#update class variables
self.questionGet()
return True
class CodernityDataStore(object):
PATH_TYPE = 'path'
def __init__(self, db_path):
self.db = Database(db_path)
if self.db.exists():
self.db.open()
else:
self.db.create()
path_index = PathIndex(self.db.path, 'path')
self.db.add_index(path_index)
path_added_index = PathAddedIndex(self.db.path, 'path_added')
self.db.add_index(path_added_index)
@classmethod
def dt_str(cls, datetime):
return datetime.isoformat()[0:19]
def add_video(self, path, video, added=None):
logger.debug("add_video(%s, %s, %s)", path, video, added)
added = added or datetime.utcnow()
existing = list(self.db.get_many('path', path, with_doc=True))
video_data, video_type = Serializer.serialize_video(video)
data = dict(_t=self.PATH_TYPE, path=path, video_data=video_data, video_type=video_type,
downloads=dict(), added=self.dt_str(added))
self.db.insert(data)
for existing_path in existing:
self.db.delete(existing_path['doc'])
def add_download(self, path, provider, sub_id, language, score):
logger.debug("add_download(%s, %s, %s, %s, %d)", path, provider, sub_id, language, score)
data = self.db.get('path', path, with_doc=True)
path = data['doc']
download = dict(provider=provider, sub_id=sub_id, lang=str(language), score=score)
if str(language) in path['downloads']:
path['downloads'][str(language)].append(download)
else:
path['downloads'][str(language)] = [download]
self.db.update(path)
def get_downloads_for_video(self, path):
logger.debug("get_downloads_for_video(%s)", path)
data = self.db.get('path', path, with_doc=True)
return data['doc']['downloads']
@staticmethod
def exceeds_desired_score(video, score, desired_movie_score, desired_episode_score):
if isinstance(video, Episode):
return score >= desired_episode_score
elif isinstance(video, Movie):
return score >= desired_movie_score
def get_incomplete_videos(self, languages, desired_movie_score, desired_episode_score, ignore_older_than):
logger.debug("get_incomplete_videos(%s, %d, %d, %s)", languages, desired_movie_score, desired_episode_score, ignore_older_than)
within_date = self.db.get_many('path_added', start=self.dt_str(ignore_older_than), with_doc=True)
results = []
for path in (data['doc'] for data in within_date):
video = Serializer.deserialize_video(path['video_type'], path['video_data'])
needs = []
for lang in languages:
if str(lang) in path['downloads']:
current_score = max(download['score'] for download in path['downloads'][str(lang)])
if not self.exceeds_desired_score(video, current_score, desired_movie_score, desired_episode_score):
needs.append(dict(lang=lang, current_score=current_score))
else:
needs.append(dict(lang=lang, current_score=0))
if needs:
results.append(dict(path=path['path'], video=video, needs=needs))
logger.debug("found %d incomplete videos: %s", len(results), results)
return results
def close(self):
self.db.close()
class SecuQ:
'''
Abstract:
Handles all Questions, input and output input into the database
def questionGet(self):
- reloads all question database attributes
def questionInsert(self,data,descriptor='inclusive'):
-> data = input question dict
ex:
'a':{'active':'True','typ':'slider','range':'0-100','aggregate':True, 'multipoint':True}, ,,,,.................
- inclusive = Updates entried (overwrites) (safe)
- exclusive = deletes all entried not in input (dangerous)
def questionsValidate(self,data):
-> data = input list of questions [(keys)]
each question verified, if not initiated, will be initiated after
'''
def __init__(self,passkey):
self.key = passkey
self.indexdb = DBIndexSystem(self.key)
#self.indexdb.masterIndex
#self.indexdb.Qindex
#self.indexdb.Tindex
#self.indexdb.IndexedTable
#self.indexdb.dbName
self.db = Database(self.indexdb.dbName)
#init variables blank, avoid key errors
self.all = {}
self.valid = {}
self.active = {}
self.notactive = {}
self.unInit = {}
self.typ = {}
self.aggregate = {}
self.multipoint = {}
self.questionGet() #populate variables
#query: all , valid, true,unInit, false, inline
def questionGet(self): #the passkey in question should be loaded from config or index passkey
if(self.db.exists()):
self.db.open()
self.db.id_ind.enc_key = self.key
#select Qindex
Qindex = self.db.get('id', self.indexdb.Qindex, with_doc=True)
oQ = Qindex.copy()
# delete unnessesary to modify
# if u use del <key> it deletes all instances in all variables using same reference to dict
oQ.pop('_rev', None)
oQ.pop('_id', None)
oQ.pop('t', None)
#oQ.pop('questions', None)
#returns list in string for of all keys aka the question and metadata
#step through and assign
questionsSet = oQ.keys()
for question in questionsSet:
self.all[question] = oQ[question]
if oQ[question]['active'] == 'True':
self.active[question] = oQ[question]
if oQ[question]['active'] == 'unInit':
self.unInit[question] = oQ[question]
if (oQ[question]['active'] == 'unInit') | (oQ[question]['active'] == 'True'):
self.valid[question] = oQ[question]
if oQ[question]['active'] == 'False':
self.notactive[question] = oQ[question]
if oQ[question]['typ'] == 'note':
self.typ[question] = oQ[question]
try:
if oQ[question]['aggregate'] == 'True':
self.aggregate[question] = oQ[question]
except KeyError:
pass
try:
if oQ[question]['multipoint'] == 'True':
self.multipoint[question] = oQ[question]
except KeyError:
pass
self.db.close()
return True
'''
Qinfo=
{
'a':{'active':'True','typ':'slider','range':'0-100','aggregate':True, 'multipoint':True},
'b':{'active':'True','typ':'slider','range':'0-100','aggregate':True, 'multipoint':False},
'c':{'active':'True','typ':'slider','range':'0-100','aggregate':False, 'multipoint':True},
'd':{'active':'True','typ':'slider','range':'0-100','aggregate':False, 'multipoint':False},
'note':{'active':'True','typ':'note', 'multipoint':"False"}
}
'''
def questionInsert(self,data,descriptor='inclusive'):# this will be a class later for ... infinite data
if(self.db.exists()):
self.db.open()
self.db.id_ind.enc_key = self.key
#select Qindex
Qindex = self.db.get('id', self.indexdb.Qindex, with_doc=True)
#must copy in this way for dictionaries or else all references are affected
oQ= Qindex.copy()
# delete unnessesary to modify
# if u use del <key> it deletes all instances in all variables using same reference to dict
oQ.pop('_rev', None)
oQ.pop('_id', None)
oQ.pop('t', None)
oQ.pop('questions', None)
# if (type(data) is str):
nQL=eval(str(data))
if (descriptor == "exclusive"):
#exclusive, new data always overwrites old data, deletes any data that is not new
# remmove old keys from row
for key in oQ.keys(): #removes keys not in entry and overwrites everything
if key not in nQL.keys():
Qindex.pop(key,None)
if (descriptor == "inclusive"):
#only overwrites data, keeps old data that is unnaffected
pass
#oQ.update(nQL) # update existing keys to be written
Qindex.update(nQL) #updates existing keys in row
self.db.update(Qindex) #updates NoSQL
self.db.close()
self.questionGet()
return True
else:
print ('CANNOT LOAD self.db')
return False
def questionsValidate(self,data): #turns all uesd questions true, insert list of questions
for question in data:
if question in self.unInit.keys():
updated = {}
updated[question] = self.unInit[question]
updated[question]['active'] = "True"
self.questionInsert(str(updated),"inclusive")
#update class variables
self.questionGet()
return True
class cache:
"""
cache for word morphological analysis
"""
def __init__(self, ):
"""
Create Analex Cache
"""
# use this dictionary as a local cache,
# The global db will be updated on destructing object
self.cache = {}
self.db = Database('~/tmp/thaalibCache')
if not self.db.exists():
self.db.create()
x_ind = WithAIndex(self.db.path, 'a')
self.db.add_index(x_ind)
else:
self.db.open()
def __del__(self):
"""
Delete instance and clear cache
"""
self.cache = None
self.db.close()
def update(self):
"""update data base """
for word in self.cache:
self.add_checked(word, self.cache[word])
def is_already_checked(self, word):
try:
return bool(self.db.get('a', word))
except:
return False
#~ except: return False;
def get_checked(self, word):
try:
x = self.db.get('a', word, with_doc=True)
y = x.get('doc', False)
if y:
return y.get('d', [])
else:
return []
except:
return []
def add_checked(self, word, data):
idata = {"a": word, 'd': data}
try:
saved = self.db.get('a', word, with_doc=True)
except:
saved = False
if saved:
saved['doc']['d'] = data
doc = saved['doc']
doc['update'] = True
self.db.update(doc)
else:
self.db.insert(idata)
def exists_cache_word(self, word):
""" test if word exists in cache"""
#if exists in cache dictionary
if word in self.cache:
return True
else: # test in database
if self.is_already_checked(word):
stored_data = self.get_checked(word)
self.cache[word] = stored_data
return bool(self.cache[word])
else:
# add null dict to the word index to avoid multiple database check
self.cache[word] = {}
return {}
def get_relation_freq(self, word_prev, word_cur, relation):
self.exists_cache_word(word_prev)
return self.cache.get(word_prev, {}).get(word_cur,
{}).get(relation, 0)
def is_related(self, word_prev, word_cur):
""" test if two words are related"""
#serach in cache
self.exists_cache_word(word_prev)
# if exists in cache or database
return self.cache.get(word_prev, {}).get(word_cur, {})
def add_relation(self, word_prev, word_cur, relation):
#~ relation ='r'+str(relation)
if word_prev not in self.cache:
# test first that is in db cache
if self.is_already_checked(word_prev):
stored_data = self.get_checked(word_prev)
self.cache[word_prev] = stored_data
else: # create an new entry
self.cache[word_prev] = {
word_cur: {
relation: 1,
},
}
# word_prev exists
# add word_cur to previous dict
elif word_cur not in self.cache[word_prev]:
self.cache[word_prev][word_cur] = {
relation: 1,
}
elif relation not in self.cache[word_prev][word_cur]:
self.cache[word_prev][word_cur][relation] = 1
else:
self.cache[word_prev][word_cur][relation] += 1
def display_all(self):
""" display all contents of data base """
print "aranasyn.cache: dislay all records in Thaalib Database " ""
for curr in self.db.all('a', with_doc=True):
print curr['doc']['a'], arepr(curr['doc']['d'])
class SecuFrame: #in producion, key must be specified
def __init__(self, passkey, date_range='all'):
self.key = passkey
self.Qeng = SecuQ(self.key)
self.indexdb = DBIndexSystem(self.key)
#self.indexdb.masterIndex
#self.indexdb.Qindex
#self.indexdb.Tindex
#self.indexdb.IndexedTable
#self.indexdb.dbName
self.dayindex = DayEntry(self.key)
#self.dayindex.dayKey
self.DBConfig = AppConfig()
self.dbName = self.DBConfig.mapget('databaseinfo')['databasename']
self.db = Database(self.dbName)
self.dbparseable = self.db2json(daterange=date_range, clean=True)
def __del__(self):
if (self.db.opened):
self.db.close()
def db2json(self, daterange, clean=True):
'''
> daterange
- tuple datetime objects to specify range
(dateObj,dateObj)
'''
dfJSON = []
if (self.db.exists()):
self.db.open()
self.db.id_ind.enc_key = self.key
if daterange == "all":
if clean == True:
for currHash in self.indexdb.IndexedTable: #get row
curr = self.db.get('id', currHash, with_doc=True)
curr.pop('_id')
curr.pop('_rev')
dfJSON.append(curr)
self.db.close()
return dfJSON
if clean == False:
for currHash in self.indexdb.IndexedTable: #get row
curr = self.db.get('id', currHash, with_doc=True)
dfJSON.append(curr)
self.db.close()
return dfJSON
if daterange == "today":
if clean == True:
curr = self.db.get('id',
self.dayindex.dayKey,
with_doc=True)
curr.pop('_id')
curr.pop('_rev')
dfJSON.append(curr)
self.db.close()
return dfJSON
'''
#@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
if ((type(daterange) == tuple) & (len(daterange)<=2) & (daterange[0]<daterange[1]) &(type(daterange[0])==datetime.datetime) & (type(daterange[1])==datetime.datetime): #if it's a valid daterange
if clean == True
for curr in db.all('id'): #get row
currdto=dt.datetime.strptime(curr['date'],"%Y-%m-%d %H:%M:%S.%f")
if ( daterange[0] <= currdto <= daterange[1]):
curr.pop('_id')
curr.pop('_rev')
dfJSON.append(curr)
db.close()
return dfJSON
if clean == False:
for curr in db.all('id'): #get row
dfJSON.append(curr)
db.close()
return dfJSON
else: # raise some kindof exception
return False
#@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
'''
def iffo(self, daterange="all", obey=True):
self.dbfiltered = []
self.dbdatemapped = {}
self.infoIndex = self.Qeng.valid # {'a':{'active':'True','typ':'slider','range':'0-100','aggregate':'True', 'multipoint':'True'}}
if obey == True:
for curr in self.dbparseable: #get row
'''
{'date' : xx , 'a':{'xxdate1xx':1,'xxdate2xx':2},
'b':{'xxdate1xx':14,'xxdate2xx':14},
'c':{'xxdate1xx':11,'xxdate2xx':11},
'note':{'xxdate2xx':'hello','xxdate3xx':'you'}
}
'''
tmp = {} #holder that is constructed
rowDate = curr["date"] #'date' : xx
'''
date : xx
'''
questionsData = curr.keys() # returns a list
questionsData.remove('date')
'''
['a','b','c','note']
'''
#questionsData.remove('note')
for question in questionsData: #get question from list
try:
if (
self.infoIndex[question]['multipoint'] ==
"True"
): # & (self.infoIndex[question]['aggregate']== "False"): #display all points
multiP = curr[question].keys()
'''
in 'a'
['xxdate1xx','xxdate2xx']
'''
for point in multiP: #points are dates
try:
tmp[question][point] = curr[question][
point]
except KeyError:
tmp[question] = {}
tmp[question][point] = curr[question][
point]
try:
self.dbdatemapped[point][question] = curr[
question][point]
except KeyError: #avoid overwriting
self.dbdatemapped[point] = {}
self.dbdatemapped[point][question] = curr[
question][point]
if (self.infoIndex[question]['multipoint'] == "True"
) & (
self.infoIndex[question]['aggregate'] == "True"
): #display only one aggregate in it's own column
'''
creates unique key for aggregate
'''
datelist = curr[question].keys(
) #gets all dates within the question
datelist.sort() #ensure earliest to latest
aggregate_key_name = str(question) + "_aggregate"
tmp[aggregate_key_name] = {}
try: #as intigers
tmp[aggregate_key_name][rowDate] = 0
aggregate_sum = 0
for point in datelist:
aggregate_sum += curr[question][point]
except TypeError: #process aggregate function as concatenated strings
tmp[aggregate_key_name][rowDate] = ""
aggregate_sum = ""
for point in datelist:
aggregate_sum += curr[question][
point] + "\n"
try:
self.dbdatemapped[rowDate][
aggregate_key_name] = aggregate_sum
except KeyError:
self.dbdatemapped[rowDate] = {}
self.dbdatemapped[rowDate][
aggregate_key_name] = aggregate_sum
tmp[aggregate_key_name] = {}
tmp[aggregate_key_name][
rowDate] = aggregate_sum # replaces with single
if ((self.infoIndex[question]['multipoint'] == "False")
&
(self.infoIndex[question]['aggregate']
== "False")) | (self.infoIndex[question]['typ']
== "note"): #display only one
'''
Puts last entry under rowdate
'''
'''
NOTE HANDLING
in future this should select the most positive note based on sentiment analysis
- For now it will select the last note typed in
'''
datelist = curr[question].keys(
) #gets all dates within the question
pointKey = self.getLastDate(
datelist
) #selects most recent date from list (keys)
try:
tmp[question][rowDate] = curr[question][
pointKey] # replaces with single, most recent, point only
except KeyError:
tmp[question] = {}
tmp[question][rowDate] = curr[question][
pointKey] # replaces with single, most recent, point only
try:
self.dbdatemapped[rowDate][question] = curr[
question][pointKey]
except KeyError:
self.dbdatemapped[rowDate] = {}
self.dbdatemapped[rowDate][question] = curr[
question][pointKey]
if (self.infoIndex[question]['multipoint'] == "False"
) & (
self.infoIndex[question]['aggregate'] == "True"
): #display only one aggregate in it's own column
datelist = curr[question].keys(
) #gets all dates within the question
datelist.sort() #ensure earliest to latest
tmp[question] = {}
try: #as intigers
tmp[question][rowDate] = 0
aggregate_sum = 0
for point in datelist:
aggregate_sum += curr[question][point]
except TypeError: #process aggregate function as concatenated strings
tmp[question][rowDate] = ""
aggregate_sum = ""
for point in datelist:
aggregate_sum += curr[question][
point] + "\n"
#output
tmp[question][rowDate] = aggregate_sum
#remapping is additive
try:
self.dbdatemapped[rowDate][
question] = aggregate_sum
except KeyError:
self.dbdatemapped[rowDate] = {}
self.dbdatemapped[rowDate][
question] = aggregate_sum
except KeyError:
continue
self.dbfiltered.append(tmp)
return self
def igraph(self):
import datetime as dt
self.graphFrame = []
graphpoints = self.dbdatemapped.keys()
graphdates = []
for date in graphpoints:
try:
graphdates.append(
dt.datetime.strptime(date, "%Y-%m-%d %H:%M:%S.%f"))
except ValueError:
graphdates.append(
dt.datetime.strptime(date, "%Y-%m-%d %H:%M:%S"))
sortkeydto, pointerdts = zip(*sorted(zip(graphdates, graphpoints)))
for i in xrange(
0,
len(pointerdts)): # want {date: xxxISOxxx , a:x ,b:x ,note:x}
tmpRow = {}
tmpRow['date'] = sortkeydto[i].isoformat() + "Z"
for question in self.dbdatemapped[pointerdts[i]]:
tmpRow[question] = self.dbdatemapped[pointerdts[i]][question]
self.graphFrame.append(tmpRow)
return self
#map accordingly with date to iso format
def Agraph(self, neuroOutput):
import datetime as dt
self.neuroOutput = neuroOutput # [(dto,dto),(dto,dto),,,,]
self.AgraphFrame = []
graphpoints = self.dbdatemapped.keys()
graphdates = []
self.last_date = None
self.curr_date = None
self.neuro_scan_count = 0
self.neuro_highlight_complete = False
for date in graphpoints:
try:
graphdates.append(
dt.datetime.strptime(date, "%Y-%m-%d %H:%M:%S.%f"))
except ValueError:
graphdates.append(
dt.datetime.strptime(date, "%Y-%m-%d %H:%M:%S"))
sortkeydto, pointerdts = zip(*sorted(zip(graphdates, graphpoints)))
for i in xrange(
0,
len(pointerdts)): # want {date: xxxISOxxx , a:x ,b:x ,note:x}
tmpRow = {}
# set to white / transparent first
self.curr_date = sortkeydto[i]
if (self.neuro_highlight_complete == False):
tmpScanPos = divmod(self.neuro_scan_count,
2) # divisor answer, remainder
#print "tmpScanPos: " +str(tmpScanPos) + " self.neuro_scan_count: " + str(self.neuro_scan_count)
tmpNeuroDate = self.neuroOutput[tmpScanPos[0]][tmpScanPos[1]]
if (self.last_date == None): tmpRow["lineColor"] = "#FFFFFF"
elif (self.curr_date == tmpNeuroDate):
if (tmpScanPos[1] == 0):
tmpRow["lineColor"] = "#CC0000" #if start of range
if (tmpScanPos[1] == 1):
tmpRow["lineColor"] = "#FFFFFF" # if end of range
self.neuro_scan_count += 1
elif (self.last_date < tmpNeuroDate < self.curr_date):
if (tmpScanPos[1] == 0):
tmpRow["lineColor"] = "#CC0000" #if start of range
if (tmpScanPos[1] == 1):
tmpRow["lineColor"] = "#FFFFFF" # if end of range
self.neuro_scan_count += 1
if ((tmpScanPos[0] + tmpScanPos[0]) == len(neuroOutput)):
self.neuro_highlight_complete = True #checks if this should be the last iteration
tmpRow['date'] = sortkeydto[i].isoformat() + "Z"
for question in self.dbdatemapped[pointerdts[i]]:
tmpRow[question] = self.dbdatemapped[pointerdts[i]][question]
self.AgraphFrame.append(tmpRow)
self.last_date = sortkeydto[i]
#map accordingly with date to iso format
def dayresponse(self):
self.responseFrame = {}
try:
tmp = self.dbdatemapped[self.dayindex.todayDate]
except KeyError: #means there is no information for the daykey
return self
# remove aggregate keyword, json handles association
for question in tmp.keys():
cleankey = question.replace('_aggregate', '')
self.responseFrame[cleankey] = tmp[question]
return self
def orderedmap(self):
import datetime as dt
self.processFrameList = []
self.processFrameDict = {}
graphpoints = self.dbdatemapped.keys()
graphdates = []
for date in graphpoints:
try:
graphdates.append(
dt.datetime.strptime(date, "%Y-%m-%d %H:%M:%S.%f"))
except ValueError:
graphdates.append(
dt.datetime.strptime(date, "%Y-%m-%d %H:%M:%S"))
sortkeydto, pointerdts = zip(*sorted(zip(graphdates, graphpoints)))
for i in xrange(
0,
len(pointerdts)): # want {date: xxxISOxxx , a:x ,b:x ,note:x}
tmpRow = {}
tmpRow[sortkeydto[i]] = {}
self.processFrameDict[sortkeydto[i]] = {}
for question in self.dbdatemapped[pointerdts[i]]:
tmpRow[sortkeydto[i]][question] = self.dbdatemapped[
pointerdts[i]][question]
self.processFrameDict[sortkeydto[i]][
question] = self.dbdatemapped[pointerdts[i]][question]
self.processFrameList.append(tmpRow)
return self
def getLastDate(self, dates): #input a list of dates
dates.sort(reverse=True)
return dates[0] #output most recent date in subset
class CodernityDB(BaseService):
"""A service providing a codernity db interface."""
name = 'db'
default_config = dict(db=dict(path=''), app=dict(dir=''))
def __init__(self, app):
super(CodernityDB, self).__init__(app)
self.dbfile = os.path.join(self.app.config['app']['dir'],
self.app.config['db']['path'])
self.db = None
self.uncommitted = dict()
self.stop_event = Event()
self.db = Database(self.dbfile)
try:
log.info('opening db', path=self.dbfile)
self.db.open()
except DatabasePathException:
log.info('db does not exist, creating it', path=self.dbfile)
self.db.create()
self.db.add_index(MD5Index(self.dbfile, 'key'))
def _run(self):
self.stop_event.wait()
def stop(self):
# commit?
log.info('closing db')
if self.started:
self.db.close()
self.stop_event.set()
def get(self, key):
log.debug('getting entry', key=key)
if key in self.uncommitted:
if self.uncommitted[key] is None:
raise KeyError("key not in db")
return self.uncommitted[key]
try:
value = self.db.get('key', key, with_doc=True)['doc']['value']
except RecordNotFound:
raise KeyError("key not in db")
return compress.decompress(value)
def put(self, key, value):
log.debug('putting entry', key=key, value=value)
self.uncommitted[key] = value
def commit(self):
log.debug('committing', db=self)
for k, v in self.uncommitted.items():
if v is None:
doc = self.db.get('key', k, with_doc=True)['doc']
self.db.delete(doc)
else:
self.db.insert({'key': k, 'value': compress.compress(v)})
self.uncommitted.clear()
def delete(self, key):
log.debug('deleting entry', key=key)
self.uncommitted[key] = None
def __contains__(self, key):
try:
self.get(key)
except KeyError:
return False
return True
def __eq__(self, other):
return isinstance(other, self.__class__) and self.db == other.db
def __repr__(self):
return '<DB at %d uncommitted=%d>' % (id(self.db), len(self.uncommitted))
def inc_refcount(self, key, value):
self.put(key, value)
def dec_refcount(self, key):
pass
def revert_refcount_changes(self, epoch):
pass
def commit_refcount_changes(self, epoch):
pass
def cleanup(self, epoch):
pass
def put_temporarily(self, key, value):
self.inc_refcount(key, value)
self.dec_refcount(key)
class DBIndexSystem:
'''
QindexGet:
-- Get the question index key
Allows for simple calling to database variables
ex:
self.indexdb = DBIndexSystem()
#self.indexdb.masterIndex
#self.indexdb.Qindex
#self.indexdb.Tindex
#self.indexdb.IndexedTable
#self.indexdb.dbName
'''
def __init__(self, passkey):
self.key = passkey
self.DBConfig = AppConfig()
#check for self.db stuff
#IF IT DOESNT PASS THESE TESTS
#warn before deletion
self.dbName = self.DBConfig.mapget('databaseinfo')['databasename']
self.masterIndex = self.DBConfig.mapget('databaseinfo')['indexkey']
self.db = Database(self.dbName)
if (self.db.exists()):
self.db.open()
self.db.id_ind.enc_key = self.key
try:
self.Qindex = self.QindexGet() #question index key
self.Tindex = self.TindexGet() #table index key
self.IndexedTable = self.tableGet(
) #regular 'table' index. list of hash pointers in order
except:
print 'bad index'
self.db.close()
self.sanitycheck = False
else:
self.db.close()
self.sanitycheck = True
def __del__(self):
if (self.db.opened):
self.db.close()
def QindexGet(self):
masterRow = self.db.get('id', self.masterIndex, with_doc=True)
Qindexkey = masterRow['Qindex']
return Qindexkey # questions as [a,b,c,d,e,f,g,h,i]
#add try if line not found, your self.db is totally f****d eh
def TindexGet(self, tableName='Tindex'):
masterRow = self.db.get('id', self.masterIndex, with_doc=True)
Tindexkey = masterRow[tableName]
return Tindexkey # questions as [a,b,c,d,e,f,g,h,i]
#add try if line not found, your self.db is totally f****d eh
def tableGet(
self,
tableName='Tindex'
): #the self.key in question should be loaded from config or index self.key
Tindexkey = self.TindexGet(
tableName) #not self variable because this may be a custom table
Tindex = self.db.get('id', Tindexkey, with_doc=True)
return Tindex[
'table'] # table entries as as [######,######,######,######,######,]
def TindexPut(self,
data,
tableName='Tindex'): #enter ordered hash data into table
if (self.db.exists()):
self.db.open()
self.db.id_ind.enc_key = self.key
Tindexkey = self.TindexGet(tableName)
TindexRow = self.db.get('id', Tindexkey, with_doc=True)
try:
TindexRow['table'].append(data)
except KeyError:
TindexRow['table'] = []
TindexRow['table'].append(data)
self.db.update(TindexRow)
self.db.close()
#append net entry to current table array
#write table array to same index key
return True
def selectrow(self, idIN):
#check if already open
if (self.db.exists()):
if (self.db.opened == False):
self.db.open()
self.db.id_ind.enc_key = self.key
#select Qindex
data = self.db.get('id', idIN, with_doc=True)
self.db.close()
return data
else:
data = self.db.get('id', idIN, with_doc=True)
return data
def updaterow(self, data):
if (self.db.exists()):
if (self.db.opened == False):
self.db.open()
self.db.id_ind.enc_key = self.key
self.db.update(data) #must include _id, must be dict/json
self.db.close()
return True
else:
self.db.update(data) #must include _id, must be dict/json
return True
class DayEntry: #checker class
'''
checks day hash or creates a new one
once instatiated, it checks for:
- if day key in config coincideds with todays date
- if there isnt a date in config, it scans database for the one matching todays
- if no date in conifig, or it's the wrong date, new row is made (only if there isnt one with matching date in the entire self.db)
'''
def __init__(self, passkey):
self.todayDate = str(getDayStart())
self.key = passkey
self.DBConfig = AppConfig()
self.dayKey = None # setup befpore checking, avoid attribute error
self.dbName = self.DBConfig.mapget('databaseinfo')['databasename']
self.db = Database(self.dbName)
try:
self.dayKey = self.DBConfig.mapget('databaseinfo')['daykey']
except KeyError: # if notthin in config, check self.db for entry
daystatus = self.checkfordate()
if (daystatus == False):
self.makeDayRow()
self.DBConfig.putmap('databaseinfo', 'daykey', self.dayKey)
#if true do nothing, config file fixed
else:
daystatus = self.checkfordate(
) #if false, scans for right one, and fixes config
oldcompare = self.dayKey
self.dayKey = self.DBConfig.mapget('databaseinfo')['daykey']
if (daystatus == False) & (oldcompare == self.dayKey):
self.makeDayRow()
self.DBConfig.putmap('databaseinfo', 'daykey', self.dayKey)
if (daystatus == True): #everything all good
pass #nothing created just a check
def __del__(self):
if (self.db.opened):
self.db.close()
def makeDayRow(self):
if (self.checkfordate() == True): #already exists no need to write
return False
dbindex = DBIndexSystem(self.key)
dayrow = {}
dayrow["date"] = self.todayDate
if (self.db.exists() == True):
self.db.open()
self.db.id_ind.enc_key = self.key
self.db.insert(dayrow)
self.db.close() #must close first , no double opens
self.getDayRowID() # resfresh day key
dbindex.TindexPut(self.dayKey)
## would normally write to config file
return True
def getDayRowID(self): #gets row id by date
if (self.db.exists()):
self.db.open()
self.db.id_ind.enc_key = self.key
for curr in self.db.all('id'):
try:
if curr['date'] == str(self.todayDate):
dataop = curr['_id']
dataop = "".join(
dataop
) #_id is returned as a list of charaters, must be concatenated to string
self.db.close()
self.dayKey = dataop
return dataop #returns datestring
except KeyError:
continue
#break
#if it makes it here, entry doesnt exist
self.db.close()
return False #there is a probplem
def checkfordate(self): #checks for existance of that date in self.db
if (self.db.exists()):
self.db.open()
self.db.id_ind.enc_key = self.key
if (self.dayKey != None):
dayrow = self.db.get('id', self.dayKey, with_doc=True)
#doesnt account for if there is an entry in the config that doesnt exist
if dayrow['date'] == str(self.todayDate):
self.db.close()
return True
for curr in self.db.all('id'): #try to search
try:
if curr['date'] == str(self.todayDate):
self.DBConfig.putmap('databaseinfo', 'daykey', "".join(
curr['_id'])) #fix lost entry
self.db.close()
return False
except KeyError:
continue
#break
#if it makes it here, entry doesnt exist and nothing was remapped
self.db.close()
return False
class SecuFrame: #in producion, key must be specified
def __init__(self,passkey,date_range='all'):
self.key = passkey
self.Qeng = SecuQ(self.key)
self.indexdb = DBIndexSystem(self.key)
#self.indexdb.masterIndex
#self.indexdb.Qindex
#self.indexdb.Tindex
#self.indexdb.IndexedTable
#self.indexdb.dbName
self.dayindex = DayEntry(self.key)
#self.dayindex.dayKey
self.DBConfig = AppConfig()
self.dbName = self.DBConfig.mapget('databaseinfo')['databasename']
self.db = Database(self.dbName)
self.dbparseable = self.db2json(daterange=date_range,clean=True)
def __del__(self):
if (self.db.opened):
self.db.close()
def db2json(self,daterange,clean=True):
'''
> daterange
- tuple datetime objects to specify range
(dateObj,dateObj)
'''
dfJSON = []
if(self.db.exists()):
self.db.open()
self.db.id_ind.enc_key = self.key
if daterange == "all":
if clean == True:
for currHash in self.indexdb.IndexedTable: #get row
curr = self.db.get('id', currHash, with_doc=True)
curr.pop('_id')
curr.pop('_rev')
dfJSON.append(curr)
self.db.close()
return dfJSON
if clean == False:
for currHash in self.indexdb.IndexedTable: #get row
curr = self.db.get('id', currHash, with_doc=True)
dfJSON.append(curr)
self.db.close()
return dfJSON
if daterange == "today":
if clean == True:
curr = self.db.get('id', self.dayindex.dayKey, with_doc=True)
curr.pop('_id')
curr.pop('_rev')
dfJSON.append(curr)
self.db.close()
return dfJSON
'''
#@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
if ((type(daterange) == tuple) & (len(daterange)<=2) & (daterange[0]<daterange[1]) &(type(daterange[0])==datetime.datetime) & (type(daterange[1])==datetime.datetime): #if it's a valid daterange
if clean == True
for curr in db.all('id'): #get row
currdto=dt.datetime.strptime(curr['date'],"%Y-%m-%d %H:%M:%S.%f")
if ( daterange[0] <= currdto <= daterange[1]):
curr.pop('_id')
curr.pop('_rev')
dfJSON.append(curr)
db.close()
return dfJSON
if clean == False:
for curr in db.all('id'): #get row
dfJSON.append(curr)
db.close()
return dfJSON
else: # raise some kindof exception
return False
#@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
'''
def iffo(self, daterange = "all", obey = True):
self.dbfiltered = []
self.dbdatemapped = {}
self.infoIndex = self.Qeng.valid # {'a':{'active':'True','typ':'slider','range':'0-100','aggregate':'True', 'multipoint':'True'}}
if obey == True :
for curr in self.dbparseable: #get row
'''
{'date' : xx , 'a':{'xxdate1xx':1,'xxdate2xx':2},
'b':{'xxdate1xx':14,'xxdate2xx':14},
'c':{'xxdate1xx':11,'xxdate2xx':11},
'note':{'xxdate2xx':'hello','xxdate3xx':'you'}
}
'''
tmp={} #holder that is constructed
rowDate = curr["date"] #'date' : xx
'''
date : xx
'''
questionsData = curr.keys() # returns a list
questionsData.remove('date')
'''
['a','b','c','note']
'''
#questionsData.remove('note')
for question in questionsData: #get question from list
try:
if (self.infoIndex[question]['multipoint']== "True"): # & (self.infoIndex[question]['aggregate']== "False"): #display all points
multiP = curr[question].keys()
'''
in 'a'
['xxdate1xx','xxdate2xx']
'''
for point in multiP: #points are dates
try:
tmp[question][point] = curr[question][point]
except KeyError:
tmp[question]={}
tmp[question][point] = curr[question][point]
try:
self.dbdatemapped[point][question] = curr[question][point]
except KeyError: #avoid overwriting
self.dbdatemapped[point] = {}
self.dbdatemapped[point][question] = curr[question][point]
if (self.infoIndex[question]['multipoint']== "True") & (self.infoIndex[question]['aggregate']== "True"): #display only one aggregate in it's own column
'''
creates unique key for aggregate
'''
datelist = curr[question].keys() #gets all dates within the question
datelist.sort() #ensure earliest to latest
aggregate_key_name = str(question)+"_aggregate"
tmp[aggregate_key_name]={}
try: #as intigers
tmp[aggregate_key_name][rowDate] = 0
aggregate_sum = 0
for point in datelist:
aggregate_sum += curr[question][point]
except TypeError: #process aggregate function as concatenated strings
tmp[aggregate_key_name][rowDate] = ""
aggregate_sum = ""
for point in datelist:
aggregate_sum += curr[question][point] + "\n"
try:
self.dbdatemapped[rowDate][aggregate_key_name] = aggregate_sum
except KeyError:
self.dbdatemapped[rowDate] = {}
self.dbdatemapped[rowDate][aggregate_key_name] = aggregate_sum
tmp[aggregate_key_name] = {}
tmp[aggregate_key_name][rowDate] = aggregate_sum # replaces with single
if ((self.infoIndex[question]['multipoint']== "False") & (self.infoIndex[question]['aggregate']== "False")) | (self.infoIndex[question]['typ']== "note"): #display only one
'''
Puts last entry under rowdate
'''
'''
NOTE HANDLING
in future this should select the most positive note based on sentiment analysis
- For now it will select the last note typed in
'''
datelist = curr[question].keys() #gets all dates within the question
pointKey = self.getLastDate(datelist) #selects most recent date from list (keys)
try:
tmp[question][rowDate] = curr[question][pointKey] # replaces with single, most recent, point only
except KeyError:
tmp[question]={}
tmp[question][rowDate] = curr[question][pointKey] # replaces with single, most recent, point only
try:
self.dbdatemapped[rowDate][question] = curr[question][pointKey]
except KeyError:
self.dbdatemapped[rowDate] = {}
self.dbdatemapped[rowDate][question] = curr[question][pointKey]
if (self.infoIndex[question]['multipoint']== "False") & (self.infoIndex[question]['aggregate']== "True"): #display only one aggregate in it's own column
datelist = curr[question].keys() #gets all dates within the question
datelist.sort() #ensure earliest to latest
tmp[question]={}
try: #as intigers
tmp[question][rowDate] = 0
aggregate_sum = 0
for point in datelist:
aggregate_sum += curr[question][point]
except TypeError: #process aggregate function as concatenated strings
tmp[question][rowDate] = ""
aggregate_sum = ""
for point in datelist:
aggregate_sum += curr[question][point] + "\n"
#output
tmp[question][rowDate] = aggregate_sum
#remapping is additive
try:
self.dbdatemapped[rowDate][question] = aggregate_sum
except KeyError:
self.dbdatemapped[rowDate] = {}
self.dbdatemapped[rowDate][question] = aggregate_sum
except KeyError:
continue
self.dbfiltered.append(tmp)
return self
def igraph(self):
import datetime as dt
self.graphFrame = []
graphpoints = self.dbdatemapped.keys()
graphdates = []
for date in graphpoints:
try:
graphdates.append(dt.datetime.strptime(date,"%Y-%m-%d %H:%M:%S.%f"))
except ValueError:
graphdates.append(dt.datetime.strptime(date,"%Y-%m-%d %H:%M:%S"))
sortkeydto, pointerdts = zip(*sorted(zip(graphdates, graphpoints)))
for i in xrange(0,len(pointerdts)): # want {date: xxxISOxxx , a:x ,b:x ,note:x}
tmpRow = {}
tmpRow['date'] = sortkeydto[i].isoformat() + "Z"
for question in self.dbdatemapped[pointerdts[i]]:
tmpRow[question] = self.dbdatemapped[pointerdts[i]][question]
self.graphFrame.append(tmpRow)
return self
#map accordingly with date to iso format
def Agraph(self,neuroOutput):
import datetime as dt
self.neuroOutput = neuroOutput # [(dto,dto),(dto,dto),,,,]
self.AgraphFrame = []
graphpoints = self.dbdatemapped.keys()
graphdates = []
self.last_date = None
self.curr_date = None
self.neuro_scan_count = 0
self.neuro_highlight_complete = False
for date in graphpoints:
try:
graphdates.append(dt.datetime.strptime(date,"%Y-%m-%d %H:%M:%S.%f"))
except ValueError:
graphdates.append(dt.datetime.strptime(date,"%Y-%m-%d %H:%M:%S"))
sortkeydto, pointerdts = zip(*sorted(zip(graphdates, graphpoints)))
for i in xrange(0,len(pointerdts)): # want {date: xxxISOxxx , a:x ,b:x ,note:x}
tmpRow = {}
# set to white / transparent first
self.curr_date = sortkeydto[i]
if (self.neuro_highlight_complete == False):
tmpScanPos = divmod(self.neuro_scan_count,2) # divisor answer, remainder
#print "tmpScanPos: " +str(tmpScanPos) + " self.neuro_scan_count: " + str(self.neuro_scan_count)
tmpNeuroDate = self.neuroOutput[tmpScanPos[0]][tmpScanPos[1]]
if ( self.last_date == None): tmpRow["lineColor"] = "#FFFFFF"
elif (self.curr_date == tmpNeuroDate):
if (tmpScanPos[1] == 0 ): tmpRow["lineColor"] = "#CC0000" #if start of range
if (tmpScanPos[1] == 1 ): tmpRow["lineColor"] = "#FFFFFF" # if end of range
self.neuro_scan_count +=1
elif(self.last_date < tmpNeuroDate < self.curr_date):
if (tmpScanPos[1] == 0 ): tmpRow["lineColor"] = "#CC0000" #if start of range
if (tmpScanPos[1] == 1 ): tmpRow["lineColor"] = "#FFFFFF" # if end of range
self.neuro_scan_count +=1
if ((tmpScanPos[0] + tmpScanPos[0]) == len(neuroOutput)): self.neuro_highlight_complete = True #checks if this should be the last iteration
tmpRow['date'] = sortkeydto[i].isoformat() + "Z"
for question in self.dbdatemapped[pointerdts[i]]:
tmpRow[question] = self.dbdatemapped[pointerdts[i]][question]
self.AgraphFrame.append(tmpRow)
self.last_date = sortkeydto[i]
#map accordingly with date to iso format
def dayresponse(self):
self.responseFrame = {}
try:
tmp = self.dbdatemapped[self.dayindex.todayDate]
except KeyError: #means there is no information for the daykey
return self
# remove aggregate keyword, json handles association
for question in tmp.keys():
cleankey = question.replace('_aggregate', '')
self.responseFrame[cleankey] = tmp[question]
return self
def orderedmap(self):
import datetime as dt
self.processFrameList = []
self.processFrameDict = {}
graphpoints = self.dbdatemapped.keys()
graphdates = []
for date in graphpoints:
try:
graphdates.append(dt.datetime.strptime(date,"%Y-%m-%d %H:%M:%S.%f"))
except ValueError:
graphdates.append(dt.datetime.strptime(date,"%Y-%m-%d %H:%M:%S"))
sortkeydto, pointerdts = zip(*sorted(zip(graphdates, graphpoints)))
for i in xrange(0,len(pointerdts)): # want {date: xxxISOxxx , a:x ,b:x ,note:x}
tmpRow = {}
tmpRow[sortkeydto[i]] = {}
self.processFrameDict[sortkeydto[i]] = {}
for question in self.dbdatemapped[pointerdts[i]]:
tmpRow[sortkeydto[i]][question] = self.dbdatemapped[pointerdts[i]][question]
self.processFrameDict[sortkeydto[i]][question] = self.dbdatemapped[pointerdts[i]][question]
self.processFrameList.append(tmpRow)
return self
def getLastDate(self,dates): #input a list of dates
dates.sort(reverse=True)
return dates[0] #output most recent date in subset
class DBSubsystem:
'''
import scan: scans existing self.db and rebuilds config file
create self.db: creates self.db file, master index, question index and table index
'''
def __init__(self, passkey, xtraDB=None):
self.DATABASE_SOFTWARE_VERSION = "0.3.1a"
self.key = passkey
self.DBConfig = AppConfig()
self.dbval = xtraDB
def __del__(self):
if (self.db.opened):
self.db.close()
# ADD REBUILD OPTION
def createDB(self):
if (self.creationCheck()):
self.buildDB()
return True
else:
return False
def creationCheck(self):
if (Integrity().checkExists() == False):
if (self.dbval != None):
self.DBConfig.createConfig()
self.DBConfig.putmap('databaseinfo', 'databasename',
self.dbval)
self.dbName = self.dbval
return True
else:
return False
else: #if integrity passed as ok existing
return False
def buildDB(self):
from _dbindex import EncUniqueHashIndex
self.dbName = self.DBConfig.mapget('databaseinfo')['databasename']
self.db = Database(self.dbName)
id_ind = EncUniqueHashIndex(self.db.path, 'id')
self.db.set_indexes([id_ind])
self.db.create()
self.db.id_ind.enc_key = self.key
self.db.close()
self.createMasterindex() #create master index passkey, only once
self.createQindex()
self.createTindex()
#add error handling
return True
'''
@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
Index Creation
@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
'''
def createMasterindex(self):
if (self.db.exists()):
self.db.open()
self.db.id_ind.enc_key = self.key
#this function assumes database
self.db.insert(
dict(t='master',
Qindex=None,
Tindex=None,
DBVersion=self.DATABASE_SOFTWARE_VERSION))
for curr in self.db.all(
'id'
): #since first passkey in self.db should be only one there, function only perfomed once
if curr['t'] == 'master':
self.masterIndex = ''.join(curr['_id'])
self.DBConfig.putmap('databaseinfo', 'indexkey',
self.masterIndex) #masterkey=value
break
#add else statement for errors if couldnt be written for found
self.db.close()
return self.masterIndex
def createQindex(self):
if (self.db.exists()):
self.db.open()
self.db.id_ind.enc_key = self.key
#this function assumes database
#insert question index
self.db.insert(dict(t='Qindex'))
#get question index passkey, form type qintex (t=xxxx)
for curr in self.db.all(
'id'
): #since first passkey in self.db should be only one there, function only perfomed once
if curr['t'] == 'Qindex':
self.Qindexkey = ''.join(curr['_id'])
break
#add else statement for errors if couldnt be written for found
#write Qindex passkey to master index
indexRow = self.db.get('id', self.masterIndex, with_doc=True)
#write question index passkey to master index
indexRow['Qindex'] = self.Qindexkey
self.db.update(indexRow)
self.db.close()
#wrote new Qindex passkey to master index passkey
def createTindex(self):
self.dbName = self.DBConfig.mapget('databaseinfo')['databasename']
self.masterIndex = self.DBConfig.mapget('databaseinfo')['indexkey']
self.db = Database(self.dbName)
if (self.db.exists()):
self.db.open()
self.db.id_ind.enc_key = self.key
#this function assumes database
#insert question index
self.db.insert(dict(t='Tindex', table=[]))
#get question index passkey, form type qintex (t=xxxx)
for curr in self.db.all(
'id'
): #since first passkey in self.db should be only one there, function only perfomed once
if curr['t'] == 'Tindex':
self.Tindexkey = ''.join(curr['_id'])
break
#add else statement for errors if couldnt be written for found
#write Qindex passkey to master index
indexRow = self.db.get('id', self.masterIndex, with_doc=True)
#write question index passkey to master index
indexRow['Tindex'] = self.Tindexkey
self.db.update(indexRow)
self.db.close()
#wrote new Qindex passkey to master index passkey
'''
