class KB_plotter(object):
def __init__(self,xval='date_float'):
self.xval = xval
self.rootNumber = 0
self.stof = DB()
self.roots_df = self.stof.get_kb_roots()
NavigationToolbar2.forward = self.next_button
NavigationToolbar2.back = self.back_button
def next_button(self, *args, **kwargs):
if self.rootNumber != self.roots_df.shape[0] - 1:
self.rootNumber = self.rootNumber + 1
self.show_plot()
def back_button(self, *args, **kwargs):
if self.rootNumber != 0:
self.rootNumber = self.rootNumber - 1
self.show_plot()
def get_root_id(self,index):
root = self.roots_df.iloc[index]
return root['id']
def get_area_name(self,index):
root = self.roots_df.iloc[index]
return root['area_name']
def show_plot(self):
plt.clf()
root_id = self.get_root_id(self.rootNumber)
area_name = self.get_area_name(self.rootNumber)
mdf = self.stof.get_kb_oodi_dd(root_id)
print mdf
plt.suptitle(area_name)
self.sub_plot(221, "MEP-aan",mdf[[self.xval,'mep_aan']],True,True,True,True)
self.sub_plot(222, "MEP-uit", mdf[[self.xval,'mep_uit']],True,True,True,True)
self.sub_plot(223, "Diff", mdf[[self.xval,'difference']],True,True,True,True)
self.sub_plot(224, "I-Flens", mdf[[self.xval,'iflens']],True,True,False,True)
plt.show()
def sub_plot(self,loc, name, mdf,mean=False,std=False,sin=False,lin=False):
mdf.columns = ['x', 'y']
(m,s) = stats.mean_std(mdf['y'])
mdf = mdf[mdf.y < m + 2 * s]
mdf = mdf[mdf.y > m - 2 * s]
(m, s) = stats.mean_std(mdf['y'])
plt.subplot(loc)
plt.title(name)
plt.scatter(mdf['x'],mdf['y'])
if mean: plt.plot([np.min(mdf['x']),np.max(mdf['x'])],[m,m],'g--')
if std:
plt.plot([np.min(mdf['x']),np.max(mdf['x'])],[m + s,m + s],'r:')
plt.plot([np.min(mdf['x']), np.max(mdf['x'])], [m - s, m - s],'r:')
if sin:
ls = np.linspace(np.min(mdf['x']), np.max(mdf['x']), 100)
sin_pred = reg.fit_sin(mdf['x'], mdf['y'])
plt.plot(ls,sin_pred['fitfunc'](ls))
if lin: plt.plot(mdf['x'], reg.fit_lin(mdf[['x']], mdf['y']))
class Search():
def __init__(self, db_name):
self.db = DB(db_name)
self.N = self.db.len()
def search(self, wordlist):
sortedlist = self.get_score_list(wordlist)
cnt = 0
url_list = []
for num, docscore in sortedlist:
cnt = cnt + 1
url_list.append(self.db.get_url_title(num))
if cnt > 20:
break
if cnt == 0:
return None
return url_list
def get_score_list(self, wordlist):
score = {}
for word in wordlist:
# 计算score
tf = {} # 词频
result = self.db.get_word_list(word)
if len(result) > 0:
doclist = result[0][0]
doclist = doclist.split(' ')
doclist = [int(x) for x in doclist]
print(len(doclist))
df = len(set(doclist)) # 包含该词的文档数，去掉重复出现的文档
idf = math.log(self.N / df) # 逆文档频率 inverse document frequency
print('idf : ', idf)
for num in doclist: # num对应的是url的id ，此处统计的是同一个词在同一篇文章中出现的次数
if num in tf:
tf[num] = tf[num] + 1
else:
tf[num] = 1
# tf 统计结束，计算score
for num in tf:
if num in score:
# 如果该num文档已经有分数了，则累加
score[num] = score[num] + tf[num] * idf
else:
score[num] = tf[num] * idf
sortedlist = sorted(score.items(), key=lambda d: d[1], reverse=True)
# score.items() 以列表的形式返回可遍历的(key,value) reverse = True 按照降序排列
return sortedlist
class DirWatch():
def __init__(self, fast5_path):
self.fast5_path = fast5_path
self.db = DB()
self.running = True
self.t1 = Thread(target=self.fast5_loop)
self.t1.start()
def fast5_loop(self):
while self.running:
fast5s = []
for fol, subs, files in os.walk(self.fast5_path):
if "added" in fol: continue
for file in files:
if not file.endswith(".fast5"): continue
if self.db.fast5_exists(file) == 0:
shutil.move(file, os.path.join("added", file))
fast5s.append(Fast5(filename=file, filepath=os.path.join(fol, "added", file)))
print("%s; Added %s to DB" %(str(datetime.now()), file))
self.db.insert_fast5s(fast5s)
sleep(60)
def close(self):
self.running = False
self.t1.join()
def __init__(self,xval='date_float'):
self.xval = xval
self.rootNumber = 0
self.stof = DB()
self.roots_df = self.stof.get_kb_roots()
NavigationToolbar2.forward = self.next_button
NavigationToolbar2.back = self.back_button
def __init__(self, file_path=None):
self.file_path = file_path
if file_path is None:
self.db = DB()
self.roots_df = self.db.get_kb_pipe_segments_roots()
self.ground_areas_df = self.db.get_kb_ground_areas()
self.proj = Projections()
self.create_area_geoms()
def store_word(self, db, title, text, url):
url_index = db.store_url(url, title)
if text != 'None' :
seggen = jieba.cut_for_search(title+text)
seglist = list(seggen)
for word in seglist:
db.store_word(word, url_index)
db.commit()
def __init__(self, xval='date_float', yval='mep_uit'):
self.xval = xval
self.yval = yval
self.rootNumber = 0
self.stof = DB()
self.roots_df = self.stof.get_kb_roots()
root_id = self.get_root_id(self.rootNumber)
self.area_df = self.stof.get_kb_oodi_dd(root_id)
NavigationToolbar2.forward = self.next_button_area
NavigationToolbar2.back = self.back_button_area
NavigationToolbar2.home = self.home_button
class TestDb(unittest.TestCase):
def setUp(self) -> None:
self.db = DB('../players.db')
def test_create_dict_from_db_query(self):
result = create_dict_from_db_query([('David De Gea Quintana', '€205K'),
('Paul Pogba', '€250K')],
['Name', 'Wage'])
self.assertEqual(
result, {
'Name': ['David De Gea Quintana', 'Paul Pogba'],
'Wage': ['€205K', '€250K']
})
def test_create_db_object_fails_if_not_file_exists(self):
self.assertRaises(EnvironmentError, DB, 'no/such/database.db')
def test_select__simple_select(self):
result = self.db.select(['Name'])
self.assertEqual(len(result['Name']), 18547)
self.assertTrue('Lionel Messi' in result['Name'])
def test_select__non_dict_return_type(self):
result = self.db.select(['Name'],
where='Club="Manchester United"',
dict_result=False)
self.assertTrue(isinstance(result, list))
def test_select__with_where(self):
result = self.db.select(['Name'], where='Club="Manchester United"')
self.assertEqual(len(result['Name']), 33)
self.assertTrue('Paul Pogba' in result['Name'])
def test_validate_team__team_in_dataset(self):
self.assertTrue(self.db.validate_team('Liverpool'))
def test_validate_team__team_not_in_dataset(self):
self.assertFalse(self.db.validate_team('ASL FC'))
def __init__(self, xval='date_float'):
self.xval = xval
self.rootNumber = 0
self.mpNumber = 0
self.stof = DB()
self.roots_df = self.stof.get_kb_roots()
root_id = self.get_root_id(self.rootNumber)
self.area_df = self.stof.get_kb_oodi_dd(root_id)
self.meas_df = self.area_df[self.area_df.measurepoint_id ==
self.get_mp_ids()[0]]
NavigationToolbar2.forward = self.next_button_area
NavigationToolbar2.back = self.back_button_area
NavigationToolbar2.forward_mp = self.next_button_mp
NavigationToolbar2.back_mp = self.back_button_mp
NavigationToolbar2.toolitems = NavigationToolbar2.toolitems + (
('Back mp', 'Back to previous mp', 'back', 'back_mp'),
('Forward mp', 'Forward to next mp', 'forward', 'forward_mp'))
async def get_conso(filiere: str = None,
region: str = None,
dptmt: str = None,
commune: str = None,
operateur: str = None):
return DB.find_conso(filiere=filiere,
region=region,
dptmt=dptmt,
commune=commune,
operateur=operateur)
async def update_doc(recordid: str,
filiere: str = None,
secteur: str = None,
operateur: str = None,
conso: float = None):
return DB.update_item(recordid=recordid,
filiere=filiere,
secteur=secteur,
operateur=operateur,
conso=conso)
async def get_dropdown(filiere: str = None,
region: str = None,
dptmt: str = None,
commune: str = None,
operateur: str = None):
args = locals()
remove = [k for k, v in args.items() if v == None]
for k in remove:
del args[k]
return DB.find_ctgr(args)
def __init__(self, xval='date_float', yval='mep_uit'):
self.xval = xval
self.yval = yval
self.rootNumber = 0
self.mpNumber = 0
self.stof = DB()
self.roots_df = self.stof.get_kb_roots()
root_id = self.get_root_id(self.rootNumber)
self.area_df = self.stof.get_kb_oodi_dd(root_id)
self.polyInt = PolyInterpolation(precision=3)
self.area_df = self.stof.get_kb_oodi_dd(root_id)
self.meas_df = self.area_df[self.area_df.measurepoint_id == self.get_mp_ids()[0]]
self.meas_df = self.prepare_meas_df(self.meas_df)
NavigationToolbar2.forward = self.next_button_area
NavigationToolbar2.back = self.back_button_area
NavigationToolbar2.forward_mp = self.next_button_mp
NavigationToolbar2.back_mp = self.back_button_mp
NavigationToolbar2.toolitems = NavigationToolbar2.toolitems + (
('Back mp', 'Back to previous mp', 'back', 'back_mp'),
('Forward mp', 'Forward to next mp', 'forward', 'forward_mp')
)
plt.figure()
plt.show()
async def get_infos(filiere: str = None,
region: str = None,
recordid: str = None,
commune: str = None,
dptmt: str = None,
operateur: str = None,
complete: bool = False):
return DB.find_infos(filiere=filiere,
region=region,
recordid=recordid,
commune=commune,
dptmt=dptmt,
operateur=operateur,
complete=complete)
def main():
while True:
# init data soures
dm = DB()
# make the screen object
pygame.init()
md = MD()
# make map
mm = MM()
# instantiate hero--> TODO: Make this a DB lookup or a new Game
hero = Hero('Grognak', mm.hero_start)
# make monsters for level
cm = CM(hero)
gm = GM.graphics_manager(md.base_surface, mm, hero, cm)
tm = TM(md.base_surface, hero, cm)
gm.update_game()
tm.update()
ic = IC(gm, tm, cm, hero)
pygame.display.update()
def test_multiple_db(self):
db = self.db
db = DB(directory=TEST_WRITE_ENV)
db.open('test_multiple_1')
db.wipe()
# Has only 1
db.put('cake', 'moo')
self.assertEqual(len(tuple(db.iter())), 1)
db2 = DB(directory=TEST_WRITE_ENV)
db2.open('test_multiple_2')
db2.wipe()
# only wipe the seperate db.
self.assertEqual(len(tuple(db.iter())), 1)
self.assertEqual(len(tuple(db2.iter())), 0)
db.put('dog', 'blue')
db2.put('cat', 'blue')
# Did not wipe the first key from the first db
self.assertCountEqual(tuple(db.iter()), (('dog', 'blue',), ('cake', 'moo') ))
self.assertCountEqual(tuple(db2.iter()), (('cat', 'blue',), ))
from database.area_model import AreaModel from database.db import DB db = DB() model = AreaModel() roots_df = db.get_kb_pipe_segments_roots() area_pip_roots = roots_df[roots_df.area == 'Almelo Tusveld'].pip_id.values area_segments_df = db.get_kb_pipe_segments(area_pip_roots) print roots_df[roots_df['area'] == 'Denekamp 2'] print area_segments_df
async def check_exist(recordid: str = None):
return {'exist': DB.check_ex(recordid)}
# if __name__ == '__main__':
# uvicorn.run('main:app', reload=True)
class StaticFeatures(object):
def __init__(self, file_path=None):
self.file_path = file_path
if file_path is None:
self.db = DB()
self.roots_df = self.db.get_kb_pipe_segments_roots()
self.ground_areas_df = self.db.get_kb_ground_areas()
self.proj = Projections()
self.create_area_geoms()
# gets the areas from the db and creates the geom objects for intersection.
# todo: takes long, maybe serialize and save
def create_area_geoms(self):
"""
Creates a dictionary with ground area geoms
:return:
"""
self.ground_geoms = dict()
for idx, row in self.ground_areas_df.iterrows():
geom_string = row['geom']
self.ground_geoms[row['id']] = self.proj.load_and_project(geom_string)
def create_feature_map(self):
"""
Creates a map to save the features in
:return: an empty feature map
:rtype: FeatureMap
"""
fm = FeatureMap([
FeatureLabels.acid,
FeatureLabels.ground_water,
FeatureLabels.stability,
FeatureLabels.ground_type,
FeatureLabels.pipe
])
return fm
def extract_static_features(self):
"""
Extracts the features for the pipes attached to a mp from the database, intersected with the areas
:return: the static features
:rtype: list
"""
if self.file_path is not None:
return self._extract_static_features_from_file()
features = []
for area_name in tqdm(self.roots_df.area.unique(), desc="Intersecting"):
area_segments_df = self.get_area_segments(area_name)
for pipe_root_id in area_segments_df.root.unique():
pipes_df = self.get_pipes_df(pipe_root_id, area_segments_df)
if np.sum(pipes_df.length.values) == 0: continue
fm = self.create_feature_map()
for idx, pipe_row in pipes_df.iterrows():
pipe_geom = self.proj.load_and_project(pipe_row['geom'])
area_intersects = self.intersect_pipe_with_area(pipe_geom)
self.set_area_features(fm, area_intersects)
self.set_pipe_features(fm, pipes_df)
mp_id = self.get_segment_mp_id(pipe_root_id, area_segments_df)
feature_row = np.concatenate(([area_name], [mp_id], fm.get_feature_list()[0]))
features.append(feature_row)
return features
def extract_static_features_to_dict(self, features):
"""
Creates a dict from the feature list. Key: (area_name, mp_id) Value: list(feature1, feature2, ...)
:param features: the features list (2d)
:return: feature dictionary
:rtype: dict
"""
f_dict = dict()
for idx, row in enumerate(features):
if self.file_path is not None: row = list(row) # todo: dirty hack, fix
f_dict[(row[0], row[1])] = row[2:]
return f_dict
def _extract_static_features_from_file(self):
"""
Gets the static features from a file (csv)
:return: the static features
:rtype: list
"""
features = np.genfromtxt(self.file_path, delimiter=',', skip_header=1, dtype=None)
return features
def set_area_features(self, fm, area_intersects):
"""
Matches and sets the lengths a pipe is in a ground area with its features.
:param fm: the Feature map
:param area_intersects: the intersecting areas
:return:
"""
for area_id, length in area_intersects:
area_row = self.ground_areas_df[self.ground_areas_df['id'] == area_id]
for group in FeatureLabels.groups: # todo: define this list somewhere
label = self.pd_serie_to_str(area_row[group])
new_val = fm.get_feature(group, label) + length
fm.set_feature(group, label, new_val)
def set_pipe_features(self, fm, segment_df):
"""
Sets the features of the pipes in this segments (coating, length, etc)
:param fm: the FeatureMap
:param segment_df: the pipe segment DataFrame
:return:
"""
group_name = FeatureLabels.pipe[0]
coating_name = FeatureLabels.pipe[1][0]
length_name = FeatureLabels.pipe[1][1] # todo: refactor feature label names, unclear
coating_val, _, _ = self.segment_coating(segment_df)
length_val = self.segment_length(segment_df)
fm.set_feature(group_name, coating_name, coating_val)
fm.set_feature(group_name, length_name, length_val)
def get_segment_mp_id(self, pipe_root_id, area_segments_df):
"""
Gets the measuring point id for this pipe segment
:param pipe_root_id: root pipe id (attatched to the mp)
:param area_segments_df: the DataFrame with all segments in the area
:return: the id of the measuring point
:rtype: int
"""
root_df = area_segments_df[area_segments_df.id == pipe_root_id]
mp_id = int(root_df.mp_id.iloc[0])
return mp_id
def get_area_segments(self, area_name):
"""
Gets the pipe segments for an area
:param area_name: name of the area
:return: the DataFrame with the area segments
:rtype: pandas.Dataframe
"""
area_pip_roots = self.roots_df[self.roots_df.area == area_name].pip_id.values
area_segments_df = self.db.get_kb_pipe_segments(area_pip_roots)
return area_segments_df
def get_pipes_df(self, pipe_root_id, area_segments_df):
"""
Gets the dataframe containing the pipes in the segment branching from the root pipe
:param pipe_root_id: the root pipe id (attached to the measure point)
:param area_segments_df: the DataFrame containing the segments in an area
:return: the DataFrame with the pipes
:rtype: pandas.DataFrame
"""
pipes_df = area_segments_df[area_segments_df.root == pipe_root_id]
pipes_df = pipes_df[pipes_df.length != 0] # drop pipes with length 0
return pipes_df.dropna(subset=['length', 'geom', 'coating'])
#
def pd_serie_to_str(self, pd_serie):
"""
Filthy function to get the python string from a dataframe
:param pd_serie:
:return: the string
:rtype: str
"""
pd_str = unicodedata.normalize('NFKD', pd_serie.to_string()).encode('ascii', 'ignore')
return ''.join([i for i in pd_str if i.isalpha()])
# Gets the coating from a pipe segment
def segment_coating(self, segment_df):
"""
Calulates the percentage of coating in a segemnt (pce vs bitumen)
:param segment_df: the DataFrame with the pipes
:return: percentage pce, sum of pce, total length
:rtype: float, float, float
"""
bit_df = segment_df[segment_df.coating == 9] # coating bitumen
pce_df = segment_df[segment_df.coating != 9] # coating pce
bit_length = np.array(bit_df.length.values) # get length of bitumen pipes
pce_length = np.array(pce_df.length.values) # get length of pce pipesddd
sum_bit = np.sum(bit_length) # sum the length
sum_pce = np.sum(pce_length)
total_length = np.sum(sum_bit) + np.sum(sum_pce) # total length of both types
return sum_pce / total_length, sum_pce, total_length
def segment_length(self, segment_df):
return np.sum(np.array(segment_df.length.values))
# finds the areas this pipe intersects with, should always return atleast 1 area
def intersect_pipe_with_area(self, pipe_geom):
"""
gets the water level for the given pipe
:param pipe_geom: LineString the pipe geom
:return: areas this pipe is in
:rtype : list
"""
areas = [] # tuple list of area_id and persentage length in that area
for area_id, area_geom in self.ground_geoms.iteritems(): # todo: check if search space can be narrowed
if not area_geom.intersects(pipe_geom): continue
diff_geom = pipe_geom.intersection(area_geom)
if diff_geom.length == pipe_geom.length: return [(area_id, np.round(pipe_geom.length, 2))]
areas.append((area_id, np.round(diff_geom.length, 2)))
return areas
class KB_plotter(object):
def __init__(self, xval='date_float'):
self.xval = xval
self.rootNumber = 0
self.mpNumber = 0
self.stof = DB()
self.roots_df = self.stof.get_kb_roots()
root_id = self.get_root_id(self.rootNumber)
self.area_df = self.stof.get_kb_oodi_dd(root_id)
self.meas_df = self.area_df[self.area_df.measurepoint_id ==
self.get_mp_ids()[0]]
NavigationToolbar2.forward = self.next_button_area
NavigationToolbar2.back = self.back_button_area
NavigationToolbar2.forward_mp = self.next_button_mp
NavigationToolbar2.back_mp = self.back_button_mp
NavigationToolbar2.toolitems = NavigationToolbar2.toolitems + (
('Back mp', 'Back to previous mp', 'back', 'back_mp'),
('Forward mp', 'Forward to next mp', 'forward', 'forward_mp'))
def next_button_area(self, *args, **kwargs):
if self.rootNumber != self.roots_df.shape[0] - 1:
self.rootNumber = self.rootNumber + 1
root_id = self.get_root_id(self.rootNumber)
self.area_df = self.stof.get_kb_oodi_dd(root_id)
self.meas_df = self.area_df[self.area_df.measurepoint_id ==
self.get_mp_ids()[0]]
self.mpNumber = 0
self.show_plot()
def back_button_area(self, *args, **kwargs):
if self.rootNumber != 0:
self.rootNumber = self.rootNumber - 1
root_id = self.get_root_id(self.rootNumber)
self.area_df = self.stof.get_kb_oodi_dd(root_id)
self.meas_df = self.area_df[self.area_df.measurepoint_id ==
self.get_mp_ids()[0]]
self.mpNumber = 0
self.show_plot()
def next_button_mp(self, *args, **kwargs):
if self.mpNumber != self.get_mp_ids().shape[0] - 1:
self.mpNumber = self.mpNumber + 1
mpid = self.get_mp_ids()[self.mpNumber]
self.meas_df = self.area_df[self.area_df.measurepoint_id == mpid]
self.show_plot()
def back_button_mp(self, *args, **kwargs):
if self.mpNumber != 0:
self.mpNumber = self.mpNumber - 1
mpid = self.get_mp_ids()[self.mpNumber]
self.meas_df = self.area_df[self.area_df.measurepoint_id == mpid]
self.show_plot()
def get_root_id(self, index):
root = self.roots_df.iloc[index]
return root['id']
def get_mp_ids(self):
return self.area_df.measurepoint_id.unique()
def get_area_name(self, index):
root = self.roots_df.iloc[index]
return root['area_name']
def show_plot(self):
plt.clf()
print self.meas_df['date_float'] - np.min(self.meas_df['date_float'])
print self.meas_df['mep_uit']
area_name = self.get_area_name(self.rootNumber)
mp_name = str(self.get_mp_ids()[self.mpNumber])
plt.suptitle(area_name + " - " + mp_name)
self.sub_plot(221, "MEP-aan", self.meas_df[[self.xval, 'mep_aan']],
True, True, False, True)
self.sub_plot(222, "MEP-uit", self.meas_df[[self.xval, 'mep_uit']],
True, True, False, True)
self.sub_plot(223, "Diff", self.meas_df[[self.xval, 'difference']],
True, True, False, True)
self.sub_plot(224, "I-Flens", self.meas_df[[self.xval, 'iflens']],
True, True, False, True)
plt.show()
def sub_plot(self,
loc,
name,
mdf,
mean=False,
std=False,
sin=False,
lin=False):
mdf.columns = ['x', 'y']
if (mdf['y'].isnull().all()):
return
(m, s) = stats.mean_std(mdf['y'])
mdf = mdf[mdf.y < m + 2 * s]
mdf = mdf[mdf.y > m - 2 * s]
(m, s) = stats.mean_std(mdf['y'])
plt.subplot(loc)
plt.title(name)
plt.scatter(mdf['x'], mdf['y'])
if mean: plt.plot([np.min(mdf['x']), np.max(mdf['x'])], [m, m], 'g--')
if std:
plt.plot([np.min(mdf['x']), np.max(mdf['x'])], [m + s, m + s],
'r:')
plt.plot([np.min(mdf['x']), np.max(mdf['x'])], [m - s, m - s],
'r:')
if sin:
ls = np.linspace(np.min(mdf['x']), np.max(mdf['x']), 100)
sin_pred = reg.fit_sin(mdf['x'], mdf['y'])
plt.plot(ls, sin_pred['fitfunc'](ls))
if lin:
try:
pred = reg.fit_lin(mdf[['x']], mdf['y'])
plt.plot(mdf['x'], pred)
except:
return
async def delete_id(recordid: str = None):
return DB.delete_id(recordid=recordid)
def __init__(self, fast5_path):
self.fast5_path = fast5_path
self.db = DB()
self.running = True
self.t1 = Thread(target=self.fast5_loop)
self.t1.start()
class TestDB(unittest.TestCase):
def setUp(self):
self.db = DB(directory=TEST_WRITE_ENV)
def tearDown(self):
if hasattr(self.db, 'env'):
try:
self.db.env.close()
except Exception as e:
print('Error for tearDown close', e)
if os.path.isdir(TEST_WRITE_ENV):
try:
shutil.rmtree(TEST_WRITE_ENV)
except PermissionError:
print('PermissionError whilst deleting test database- Using db.wipe(True)...')
time.sleep(1)
self.db.wipe(True)
shutil.rmtree(TEST_WRITE_ENV)
self.assertFalse(os.path.exists(TEST_WRITE_ENV), 'tearDown did not Delete Test Directory.')
def test_open(self):
"""Open a new DB, asserting the new directory data"""
self.db.open('test')
self.assertEqual(self.db.env.path(), TEST_WRITE_ENV)
self.assertTrue(os.path.exists(self.db.env.path()))
@patch('lmdb.open')
def test_make_directory(self, lmdb_open):
"""Open a new DB, asseting the correct library methods are used"""
self.db.open('test')
args = self.db._open_args(map_size=MAX_BYTES)
lmdb_open.assert_called_with(TEST_WRITE_ENV, **args)
def test_destroy(self):
db = self.db
db.open('test_wipe')
db.put('doggy', (1,2,3))
self.assertTupleEqual(db.get('doggy'), (1,2,3))
db.close()
db.open('test_wipe')
self.assertTupleEqual(db.get('doggy'), (1,2,3))
db.wipe()
db.open('test_wipe')
self.assertEqual(db.get('doggy'), None)
db.put('doggy', (1,2,3))
db.close()
db.open('test_wipe')
self.assertTupleEqual(db.get('doggy'), (1,2,3))
db.start = Mock(side_effect=db.start)
db.wipe(True)
db.start.assert_called()
self.assertEqual(db.get('doggy'), None)
def test_put_raise_error(self):
"""Add a key value and assert its existence"""
self.db.open('test_put')
key, value = b'my_test_key', b'my test value'
success = self.db.put(key, value)
self.assertTrue(success)
result = self.db.get(key)
self.assertEqual(result, value)
with self.assertRaises(TypeError):
self.db.put('apple', 'green', encode_key=False)
def test_put_without_save(self):
self.db.open('test_put')
key, value = b'my_test_key', b'my test value'
self.db._transaction_success = Mock()
success = self.db.put(key, value, save=False)
self.assertTrue(success)
self.assertEqual(self.db._transaction_success.call_count, 0)
def test_decode_with_decode_func(self):
class Foo:
def __str__(self):
return 'cake'
self.assertEqual(self.db.decode(Foo()), 'cake')
class Two:
def decode(self, encode):
return 1
def __str__(self):
return 'cake'
self.assertEqual(self.db.decode(Foo()), 'cake')
self.assertEqual(self.db.decode(Two()), 1)
self.assertEqual(self.db.decode(b'convert string'), 'convert string')
def test_translate(self):
key = b'window'
self.assertEqual(self.db.translate(key), 'window')
self.assertEqual(self.db.translate(b'!:str!:dog'), [b'str', b'dog'])
def test_delete(self):
"""Add a key with put(), then delete. More deletes should yield false."""
# Use put for an input.
self.db.open('test_put')
key, value = b'my_test_key', b'my test value'
self.db.put(key, value)
success = self.db.delete(key)
self.assertTrue(success)
success = self.db.delete(key)
# not thing to delete the second time
self.assertFalse(success)
success = self.db.delete(key)
self.assertFalse(success)
def test_keys(self):
return_value = [1,2,3]
self.db.iter = Mock(return_value=return_value)
result = self.db.keys(cake=True)
self.db.iter.assert_called_with(values=False,
dups=False,
convert=False,
cake=True)
self.assertCountEqual(result, return_value)
def test_values(self):
return_value = [1,2,3]
self.db.iter = Mock(return_value=return_value)
result = self.db.values(cake=True)
self.db.iter.assert_called_with(keys=False,
cake=True)
self.assertCountEqual(result, return_value)
def test_count(self):
db = self.db
db.open('test_count')
db.put('a', 'b')
db.put('c', 'c')
db.put('c', 'd')
self.assertEqual(db.count('a'), 1)
self.assertEqual(db.count('c'), 2)
def test_delete(self):
db = self.db
db.open('test_delete')
db.wipe()
db.put('a', 'b')
db.put('a', 'e')
db.put('b', 'c')
db.put('c', 'd')
db.put('c', 'e')
db.put('d', 'f')
keys = tuple(db.iter())
expected = (
('a', 'b'),
('a', 'e'),
('b', 'c'),
('c', 'd'),
('c', 'e'),
('d', 'f')
)
self.assertTupleEqual(keys, expected)
result = db.delete('a')
self.assertTrue(result)
keys = tuple(db.iter())
expected = (
('b', 'c'),
('c', 'd'),
('c', 'e'),
('d', 'f')
)
self.assertTupleEqual(keys, expected)
result = db.delete('c', value='d')
keys = tuple(db.iter())
expected = (
('b', 'c'),
('c', 'e'),
('d', 'f')
)
self.assertTupleEqual(keys, expected)
def test_get_last(self):
db = self.db
db.open('test_delete')
db.wipe()
db.put('a', 'b')
db.put('a', 'e')
db.put('b', 'c')
db.put('c', 'd')
db.put('c', 'e')
self.assertEqual(db.get('a'), 'b')
self.assertEqual(db.get('a', last=True), 'e')
self.assertEqual(db.get('c'), 'd')
self.assertEqual(db.get('c', last=True), 'e')
self.assertEqual(db.get('b'), 'c')
self.assertEqual(db.get('b', last=True), 'c')
self.assertNotEqual(db.get('b', last=True), 'a')
self.assertNotEqual(db.get('b'), 'a')
def test_pop(self):
self.db._cursor = Mock()
self.db.pop('apple')
self.db._cursor.pop.assert_called_with(b'apple')
def test_multiple_db(self):
db = self.db
db = DB(directory=TEST_WRITE_ENV)
db.open('test_multiple_1')
db.wipe()
# Has only 1
db.put('cake', 'moo')
self.assertEqual(len(tuple(db.iter())), 1)
db2 = DB(directory=TEST_WRITE_ENV)
db2.open('test_multiple_2')
db2.wipe()
# only wipe the seperate db.
self.assertEqual(len(tuple(db.iter())), 1)
self.assertEqual(len(tuple(db2.iter())), 0)
db.put('dog', 'blue')
db2.put('cat', 'blue')
# Did not wipe the first key from the first db
self.assertCountEqual(tuple(db.iter()), (('dog', 'blue',), ('cake', 'moo') ))
self.assertCountEqual(tuple(db2.iter()), (('cat', 'blue',), ))
@patch('lmdb.open')
def test_create_transaction_using(self, lmdb_open):
item = Mock()
begin_mock = Mock()
# self.db.open = Mock()
self.db.env = begin_mock
self.db._cursor = 'foo'
kw = dict(using=item, write=False)
self.db.create_transaction(**kw)
begin_mock.begin.assert_called_with(db=item, write=False)
self.assertEqual(self.db._cursor, None)
def test_commit(self):
"""put() a key and commit
"""
# First we assert the death of non persistent put() delete
self.db.open('test_commit')
key, value = b'commit_key', b'my test value'
success = self.db.put(key, value, save=True)
self.assertTrue(success)
success = self.db.delete(key, commit=False)
self.assertTrue(success)
result = self.db.get(key)
self.assertEqual(result, None)
# close and open-
self.db.close()
self.setUp()
self.db.open('test_commit')
# Should exist due to persistent write
result = self.db.get(key)
self.assertEqual(result, value)
# Now delete.
success = self.db.delete(key, commit=False)
self.assertTrue(success)
# store the change
self.db.commit()
# Delete and remake -
self.tearDown()
self.setUp()
self.db.open('test_commit')
# Should not exist due to commit()
result = self.db.get(key)
self.assertEqual(result, None)
def test_in_out_conversion(self):
self.db.open('test_commit')
result = self.db.put('apples', '[1,24,54,6,56]')
self.assertTrue(result)
result = self.db.get('apples')
self.db.put('apples', [1,24,54,6,56])
result = self.db.get('apples')
self.assertEqual(result, [1,24,54,6,56])
self.assertIsInstance(result, list)
def test_tuple_store(self):
self.db.open('test_commit')
self.db.put('nums', (1,2,3,4,5,6,7,8,))
self.assertIsInstance(self.db.get('nums'), tuple)
def test_list_store(self):
self.db.open('test_commit')
self.db.put('list', [1,2,3,4,5])
res = self.db.get('list')
self.assertIsInstance(self.db.get('list'), list)
def test_collect(self):
self.db.open('test_collect')
self.db.delete('collect')
self.db.put('collect', (1,2,3,4,5,))
self.db.put('collect', (5,4,3,2,1,))
result = self.db.collect('collect')
self.assertEqual(len(list(result)), 2)
def test_dict_store(self):
self.db.open('test_commit')
de = {'foo': 1, 'bar': 'two', 'three': True}
self.db.put('dict', de)
res = self.db.get('dict')
expected = {'foo': 1, 'bar': 'two', 'three': True}
self.assertDictEqual(res, expected)
def test___repr__(self):
db = DB(directory=TEST_WRITE_ENV, name='dave')
s= '<database.db.DB "dave">'
self.assertEqual(str(db), s)
def setupDB(replication_class='SimpleStrategy', replication_factor=1):
DB.setupDB(AuthDB.keyspace, replication_class=replication_class,
replication_factor=replication_factor)
def setUp(self) -> None:
self.db = DB('../players.db')
def setUp(self):
self.db = DB(directory=TEST_WRITE_ENV)
class KB_plotter_avg_n(object):
def __init__(self, xval='date_float', yval='mep_uit'):
self.xval = xval
self.yval = yval
self.rootNumber = 0
self.stof = DB()
self.roots_df = self.stof.get_kb_roots()
root_id = self.get_root_id(self.rootNumber)
self.area_df = self.stof.get_kb_oodi_dd(root_id)
NavigationToolbar2.forward = self.next_button_area
NavigationToolbar2.back = self.back_button_area
NavigationToolbar2.home = self.home_button
def next_button_area(self, *args, **kwargs):
if self.rootNumber != self.roots_df.shape[0] - 1:
self.rootNumber = self.rootNumber + 1
root_id = self.get_root_id(self.rootNumber)
self.area_df = self.stof.get_kb_oodi_dd(root_id)
#plt.clf()
def back_button_area(self, *args, **kwargs):
if self.rootNumber != 0:
self.rootNumber = self.rootNumber - 1
root_id = self.get_root_id(self.rootNumber)
self.area_df = self.stof.get_kb_oodi_dd(root_id)
#plt.clf()
def home_button(self, *args, **kwargs):
self.show_plot()
def get_root_id(self, index):
root = self.roots_df.iloc[index]
return root['id']
def get_mp_ids(self):
return self.area_df.measurepoint_id.unique()
def get_area_name(self, index):
root = self.roots_df.iloc[index]
return root['area_name']
def save_errors(self):
for rootIdx in range(0, self.roots_df.shape[0]):
print "area: " + str(rootIdx) + "/" + str(self.roots_df.shape[0])
rootId = self.get_root_id(rootIdx)
self.area_df = self.stof.get_kb_oodi_dd(rootId)
area_name = self.get_area_name(rootIdx)
max_n = 50
mpLen = len(self.get_mp_ids())
e = np.zeros([mpLen, max_n])
polyInt = PolyInterpolation(precision=2)
for idx, mpid in enumerate(self.get_mp_ids()):
meas_df = self.area_df[self.area_df.measurepoint_id == mpid]
meas_df = meas_df[[self.xval, 'mep_uit']]
meas_df = meas_df.dropna()
meas_df.columns = ['x', 'y']
if (meas_df.shape[0] <= 1):
continue
(m, s) = stats.mean_std(meas_df['y'])
meas_df = meas_df[meas_df.y < m + 2 * s]
meas_df = meas_df[meas_df.y > m - 2 * s]
polyInt.set_t(meas_df['x'])
polyInt.set_y(meas_df['y'])
for n in range(0, max_n):
coefs = polyInt.find_coefs(n)
e[idx, n] = polyInt.avg_dist_reg(coefs[0:n], precision=2)
#e[idx, :] = (e[idx, :] - np.min(e[idx, :])) / (np.max(e[idx, :]) - np.min(e[idx, :]))
np.savetxt(area_name + '.csv', e, delimiter=',')
def save_errors_LOOCV(self):
polyInt = PolyInterpolation(precision=2)
for rootIdx in range(20, self.roots_df.shape[0]):
print
rootId = self.get_root_id(rootIdx)
self.area_df = self.stof.get_kb_oodi_dd(rootId)
area_name = self.get_area_name(rootIdx)
max_n = 50
mpLen = len(self.get_mp_ids())
e = np.zeros([mpLen, max_n])
for mpointIdx, mpid in enumerate(self.get_mp_ids()):
meas_df = self.area_df[self.area_df.measurepoint_id == mpid]
meas_df = meas_df[[self.xval, self.yval]]
meas_df = meas_df.dropna()
meas_df.columns = ['x', 'y']
if (meas_df.shape[0] <= 2):
print area_name + ": " + str(rootIdx + 1) + "/" + str(
self.roots_df.shape[0]
) + ", mp: " + str(mpointIdx + 1) + "/" + str(
mpLen) + " - ommited, has too little measurements"
continue
(m, s) = stats.mean_std(meas_df['y'])
meas_df = meas_df[meas_df.y < m + 2 * s]
meas_df = meas_df[meas_df.y > m - 2 * s]
if (meas_df.shape[0] <= 2):
print area_name + ": " + str(rootIdx + 1) + "/" + str(
self.roots_df.shape[0]
) + ", mp: " + str(mpointIdx + 1) + "/" + str(
mpLen) + " - ommited, has too little measurements"
continue
print area_name + ": " + str(rootIdx + 1) + "/" + str(
self.roots_df.shape[0]) + ", mp: " + str(
mpointIdx + 1) + "/" + str(mpLen)
measIdx = 0
for test_row in meas_df.iterrows():
polyInt.set_t(np.array(meas_df['x']))
polyInt.set_y(np.array(meas_df['y']))
testT = polyInt.t[measIdx]
testY = np.array(polyInt.y)[measIdx]
polyInt.t = np.delete(polyInt.t, measIdx)
polyInt.y = np.delete(polyInt.y, measIdx)
for n in range(0, max_n):
coefs = polyInt.find_coefs(n)
p = float(10.**-2)
yHat = polyInt.get_y_hat(coefs, precision=2)
yFit = np.interp(testT, np.arange(-1.0, 1. + p, p),
yHat)
e[mpointIdx, n] += np.abs(testY - yFit)
measIdx += 1
e[mpointIdx, :] = e[mpointIdx, :] / float(meas_df.shape[0])
np.savetxt(area_name + '.csv', e, delimiter=',')
def show_plot(self):
#plt.clf()
area_name = self.get_area_name(self.rootNumber)
max_n = 25
mpLen = len(self.get_mp_ids())
e = np.zeros([mpLen, max_n])
polyInt = PolyInterpolation(precision=2)
for idx, mpid in enumerate(self.get_mp_ids()):
print str(idx) + "/" + str(mpLen)
meas_df = self.area_df[self.area_df.measurepoint_id == mpid]
meas_df = meas_df[[self.xval, 'mep_uit']]
meas_df = meas_df.dropna()
meas_df.columns = ['x', 'y']
if (meas_df.shape[0] <= 1):
continue
(m, s) = stats.mean_std(meas_df['y'])
meas_df = meas_df[meas_df.y < m + 2 * s]
meas_df = meas_df[meas_df.y > m - 2 * s]
polyInt.set_t(meas_df['x'])
polyInt.set_y(meas_df['y'])
coefs = polyInt.find_coefs(max_n)
for n in range(0, max_n):
e[idx, n] = polyInt.avg_dist_reg(coefs[0:n], precision=2)
e[idx, :] = (e[idx, :] - np.min(e[idx, :])) / (np.max(e[idx, :]) -
np.min(e[idx, :]))
#plt.plot(range(0, max_n), e[idx,:])
plt.plot(range(0, max_n), np.mean(e, axis=0))
plt.title(area_name)
plt.show()
from model.student import Student
from model.exam import Exam
from model.user import User
from model.class_model import Class
from model.workbook import Workbook
import datetime
from datetime import timedelta
from controller.exam_holder import Holder
from database.db import DB
DB.start_connection()
DB.drop_db()
DB.generate_database_schema()
session = DB.Session_factory()
user1 = User("Mohammad", "Siavashi", "*****@*****.**", "4490347022",
"MS95", "2223858", True, [], [], [])
session.add(user1)
student1 = Student(
first_name="Ahmad",
last_name="siavash",
expire_date=datetime.datetime.now(),
advisor="ahmad",
national_id="44903232527",
student_id="9332045",
user_id=session.query(User).filter(User.username == "MS95").first().id,
classes=[],
exams=[],
workbooks=[])
def test___repr__(self):
db = DB(directory=TEST_WRITE_ENV, name='dave')
s= '<database.db.DB "dave">'
self.assertEqual(str(db), s)
def __init__(self, db_name):
self.db = DB(db_name)
self.N = self.db.len()
class CoefsPlotter(object):
def __init__(self, xval='date_float', yval='mep_uit'):
self.xval = xval
self.yval = yval
self.rootNumber = 0
self.mpNumber = 0
self.stof = DB()
self.roots_df = self.stof.get_kb_roots()
root_id = self.get_root_id(self.rootNumber)
self.area_df = self.stof.get_kb_oodi_dd(root_id)
self.polyInt = PolyInterpolation(precision=3)
self.area_df = self.stof.get_kb_oodi_dd(root_id)
self.meas_df = self.area_df[self.area_df.measurepoint_id == self.get_mp_ids()[0]]
self.meas_df = self.prepare_meas_df(self.meas_df)
NavigationToolbar2.forward = self.next_button_area
NavigationToolbar2.back = self.back_button_area
NavigationToolbar2.forward_mp = self.next_button_mp
NavigationToolbar2.back_mp = self.back_button_mp
NavigationToolbar2.toolitems = NavigationToolbar2.toolitems + (
('Back mp', 'Back to previous mp', 'back', 'back_mp'),
('Forward mp', 'Forward to next mp', 'forward', 'forward_mp')
)
plt.figure()
plt.show()
def next_button_area(self, *args, **kwargs):
if self.rootNumber != self.roots_df.shape[0] - 1:
self.rootNumber = self.rootNumber + 1
root_id = self.get_root_id(self.rootNumber)
self.area_df = self.stof.get_kb_oodi_dd(root_id)
self.meas_df = self.area_df[self.area_df.measurepoint_id == self.get_mp_ids()[0]]
self.meas_df = self.prepare_meas_df(self.meas_df)
self.mpNumber = 0
self.show_plot()
def back_button_area(self, *args, **kwargs):
if self.rootNumber != 0:
self.rootNumber = self.rootNumber - 1
root_id = self.get_root_id(self.rootNumber)
self.area_df = self.stof.get_kb_oodi_dd(root_id)
self.meas_df = self.area_df[self.area_df.measurepoint_id == self.get_mp_ids()[0]]
self.meas_df = self.prepare_meas_df(self.meas_df)
self.mpNumber = 0
self.show_plot()
def next_button_mp(self, *args, **kwargs):
if self.mpNumber != self.get_mp_ids().shape[0] - 1:
self.mpNumber = self.mpNumber + 1
mpid = self.get_mp_ids()[self.mpNumber]
self.meas_df = self.area_df[self.area_df.measurepoint_id == mpid]
self.meas_df = self.prepare_meas_df(self.meas_df)
self.show_plot()
def back_button_mp(self, *args, **kwargs):
if self.mpNumber != 0:
self.mpNumber = self.mpNumber - 1
mpid = self.get_mp_ids()[self.mpNumber]
self.meas_df = self.area_df[self.area_df.measurepoint_id == mpid]
self.meas_df = self.prepare_meas_df(self.meas_df)
self.show_plot()
def get_root_id(self, index):
root = self.roots_df.iloc[index]
return root['id']
def get_mp_ids(self):
return self.area_df.measurepoint_id.unique()
def get_area_name(self, index):
root = self.roots_df.iloc[index]
return root['area_name']
def get_number_of_mps(self):
return self.get_mp_ids().shape[0]
def get_number_of_areas(self):
return self.roots_df.shape[0]
def prepare_meas_df(self, meas_df):
meas_df = meas_df[[self.xval, self.yval]]
meas_df = meas_df.dropna()
meas_df.columns = ['x', 'y']
if (meas_df.shape[0] <= 2):
return None
(m, s) = stats.mean_std(meas_df['y'])
meas_df = meas_df[meas_df.y < m + 2 * s]
meas_df = meas_df[meas_df.y > m - 2 * s]
if (meas_df.shape[0] <= 2):
return None
return meas_df
def show_plot(self):
plt.clf()
if self.meas_df is None:
print 'Not enough points'
return
area_name = self.get_area_name(self.rootNumber)
self.polyInt.set_t(np.array(self.meas_df['x']))
self.polyInt.set_y(np.array(self.meas_df['y']))
coefs = self.polyInt.find_coefs(10)
yHat = self.polyInt.get_y_hat_for_range(coefs, np.arange(-1, 1, 0.04))
tHat = np.arange(-1, 1, 0.04)
t = self.polyInt.t
y = self.polyInt.y
t_sorted, y_sorted = zip(*sorted(zip(t, y), key=lambda x: x[0]))
plt.plot(t_sorted, y_sorted)
plt.plot(tHat, yHat)
area_title = area_name + ": " + str(self.rootNumber + 1) + "/" + str(self.get_number_of_areas())
mp_title = "mp: " + str(self.mpNumber + 1) + "/" + str(self.get_number_of_mps())
plt.title(area_title + " " + mp_title)
plt.draw()