def dropfields(input_path, output_path, todrop):
input_file = tables.open_file(input_path, mode="r")
input_root = input_file.root
output_file = tables.open_file(output_path, mode="w")
output_globals = output_file.create_group("/", "globals", "Globals")
print(" * copying globals ...", end=' ')
copy_table(input_root.globals.periodic, output_globals)
print("done.")
output_entities = output_file.create_group("/", "entities", "Entities")
for table in input_file.iterNodes(input_root.entities):
table_fields = get_fields(table)
table_fields = [(fname, ftype) for fname, ftype in table_fields
if fname not in todrop]
size = (len(table) * table.dtype.itemsize) / 1024.0 / 1024.0
#noinspection PyProtectedMember
print(" * copying table %s (%.2f Mb) ..." % (table._v_name, size),
end=' ')
copy_table(table, output_entities, table_fields)
print("done.")
input_file.close()
output_file.close()
def dropfields(input_path, output_path, todrop):
input_file = tables.open_file(input_path, mode="r")
input_root = input_file.root
output_file = tables.open_file(output_path, mode="w")
output_globals = output_file.create_group("/", "globals", "Globals")
print(" * copying globals ...", end=' ')
copy_table(input_root.globals.periodic, output_globals)
print("done.")
output_entities = output_file.create_group("/", "entities", "Entities")
for table in input_file.iterNodes(input_root.entities):
table_fields = get_fields(table)
output_dtype = np.dtype([(fname, ftype)
for fname, ftype in table_fields
if fname not in todrop])
size = (len(table) * table.dtype.itemsize) / 1024.0 / 1024.0
# noinspection PyProtectedMember
print(" * copying table %s (%.2f Mb) ..." % (table._v_name, size),
end=' ')
copy_table(table, output_entities, output_dtype)
print("done.")
input_file.close()
output_file.close()
def search():
app.logger.debug('search')
#abort(make_response('Not implemented yet ;)', 501))
if request.method == 'POST':
ID = request.form['ID']
if ID:
upvote(int(ID))
response = render_template('latest.html',
images=IMAGES,
themes=THEMES,
buttons=BUTTONS)
if request.method == 'GET':
sw = request.args.get('pattern')
r = request.args.get('regexp')
RES = []
if sw:
if r == "on": # RECHERCHE PAR THEME
for img in IMAGES:
if sw in get_fields(img['id']):
RES.append(img)
else: # RECHERCHE PAR MOTS CLES
for img in IMAGES:
if sw.capitalize() in img['title'].capitalize():
RES.append(img)
response = render_template('search.html',
images=RES,
themes=THEMES,
sw=sw,
r=r,
buttons=BUTTONS)
else:
response = render_template('search.html')
return response
def listPage():
if request.method == 'GET':
db=load()
srch=True
techniques=get_techniques(db)
projects=search(db, sort_by='start_date',sort_order='desc',techniques=None,search=None,search_fields=None)
field=get_fields(db)
return render_template('list.html',srch=srch,projects=projects, field=field, techniques=techniques)
def list_page():
"""List Page"""
db = data.init()
fields = data.get_fields(db)
#If user is searhing (POST)
if request.method == 'POST':
search_list = []
for x in data.get_fields(db):
#only add to list if x exists otherwise pass
try:
search_list.append(request.form[x])
except:
pass
if search_list == []:
search_list = None
#Selects how it should sort the list
sort_order = request.form["sort_order"]
db = data.search(db, search=request.form["search"],sort_order=sort_order, search_fields=search_list)
return render_template("list.html",dataB = db, _fields = fields)
def dropfields(input_path, output_path, todrop):
input_file = tables.openFile(input_path, mode="r")
input_root = input_file.root
output_file = tables.openFile(output_path, mode="w")
output_globals = output_file.createGroup("/", "globals", "Globals")
print " * copying globals ...",
copyTable(input_root.globals.periodic, output_file, output_globals)
print "done."
output_entities = output_file.createGroup("/", "entities", "Entities")
for table in input_file.iterNodes(input_root.entities):
table_fields = get_fields(table)
table_fields = [(fname, ftype) for fname, ftype in table_fields
if fname not in todrop]
print " * copying table %s (%.2f Mb) ..." % (table._v_name,
table_size(table)),
copyTable(table, output_file, output_entities,
table_fields)
print "done."
input_file.close()
output_file.close()
def from_table(cls, table):
return Entity(table.name, get_fields(table), missing_fields=[],
links={}, macro_strings={}, process_strings={})
def get_group_fields(node):
if node is None:
return {}
# noinspection PyProtectedMember
return {table._v_name: get_fields(table) for table in node._f_iter_nodes()}
def get_h5_fields(input_file):
#noinspection PyProtectedMember
return dict((table._v_name, get_fields(table))
for table in input_file.iterNodes(input_file.root.entities))
def from_table(cls, table):
return Entity(table.name,
get_fields(table),
links={},
macro_strings={},
process_strings={})
def __init__(self,
name,
fields=None,
links=None,
macro_strings=None,
process_strings=None,
array=None):
"""
Parameters
----------
name
fields : list of tuple (name, type)
links : {name: links.Link}
macro_strings
process_strings
array
"""
self.name = name
# we should have exactly one of either array or fields defined
assert ((fields is None and array is not None)
or (fields is not None and array is None))
if array is not None:
if fields is None:
fields = get_fields(array)
array_period = np.min(array['period'])
else:
array_period = None
if not isinstance(fields, FieldCollection):
def fdef2field(name, fielddef):
initialdata = True
output = True
default_value = None
if isinstance(fielddef, Field):
return fielddef
elif isinstance(fielddef, (dict, str)):
if isinstance(fielddef, dict):
strtype = fielddef['type']
initialdata = fielddef.get('initialdata', True)
output = fielddef.get('output', True)
default_value = fielddef.get(
'default', default_value_by_strtype[strtype])
elif isinstance(fielddef, str):
strtype = fielddef
default_value = default_value_by_strtype[strtype]
else:
raise Exception('invalid field definition')
dtype = field_str_to_type(strtype, "field '%s'" % name)
else:
assert isinstance(fielddef, type)
dtype = normalize_type(fielddef)
return Field(name, dtype, initialdata, output, default_value)
fields = FieldCollection(
fdef2field(name, fdef) for name, fdef in fields)
duplicate_names = [
name for name, num in count_occurrences(fields.names) if num > 1
]
if duplicate_names:
raise Exception("duplicate fields in entity '%s': %s" %
(self.name, ', '.join(duplicate_names)))
fnames = set(fields.names)
if 'id' not in fnames:
fields.insert(0, Field('id', int))
if 'period' not in fnames:
fields.insert(0, Field('period', int))
self.fields = fields
self.links = links
if macro_strings is None:
macro_strings = {}
self.macro_strings = macro_strings
self.process_strings = process_strings
self.processes = None
self.expectedrows = tables.parameters.EXPECTED_ROWS_TABLE
self.table = None
self.input_table = None
self.indexed_input_table = None
self.indexed_output_table = None
self.input_rows = {}
# TODO: it is unnecessary to keep periods which have already been
# simulated, because (currently) when we go back in time, we always go
# back using the output table... but periods before the start_period
# are only present in input_index
self.input_index = {}
self.output_rows = {}
self.output_index = {}
self.output_index_node = None
self.base_period = None
# we need a separate field, instead of using array['period'] to be able
# to get the period even when the array is empty.
self.array_period = array_period
self.array = array
self.lag_fields = []
self.array_lag = None
self.num_tmp = 0
self.temp_variables = {}
self.id_to_rownum = None
if array is not None:
rows_per_period, index_per_period = index_table(array)
self.input_rows = rows_per_period
self.output_rows = rows_per_period
self.input_index = index_per_period
self.output_index = index_per_period
self.id_to_rownum = index_per_period[array_period]
self._variables = None
self._methods = None
def __init__(self, name, fields=None, links=None, macro_strings=None,
process_strings=None, array=None):
self.name = name
# we should have exactly one of either array or fields defined
assert ((fields is None and array is not None) or
(fields is not None and array is None))
if array is not None:
if fields is None:
fields = get_fields(array)
array_period = np.min(array['period'])
else:
array_period = None
if not isinstance(fields, FieldCollection):
def fdef2field(name, fielddef):
initialdata = True
output = True
default_value = None
if isinstance(fielddef, Field):
return fielddef
elif isinstance(fielddef, (dict, str)):
if isinstance(fielddef, dict):
strtype = fielddef['type']
initialdata = fielddef.get('initialdata', True)
output = fielddef.get('output', True)
default_value = fielddef.get('default', default_value_by_strtype[strtype])
elif isinstance(fielddef, str):
strtype = fielddef
default_value = default_value_by_strtype[strtype]
else:
raise Exception('invalid field definition')
dtype = field_str_to_type(strtype, "field '%s'" % name)
else:
assert isinstance(fielddef, type)
dtype = normalize_type(fielddef)
return Field(name, dtype, initialdata, output, default_value)
fields = FieldCollection(fdef2field(name, fdef)
for name, fdef in fields)
duplicate_names = [name
for name, num
in count_occurrences(fields.names)
if num > 1]
if duplicate_names:
raise Exception("duplicate fields in entity '%s': %s"
% (self.name, ', '.join(duplicate_names)))
fnames = set(fields.names)
if 'id' not in fnames:
fields.insert(0, Field('id', int))
if 'period' not in fnames:
fields.insert(0, Field('period', int))
self.fields = fields
self.links = links
if macro_strings is None:
macro_strings = {}
self.macro_strings = macro_strings
self.process_strings = process_strings
self.processes = None
self.expectedrows = tables.parameters.EXPECTED_ROWS_TABLE
self.table = None
self.input_table = None
self.indexed_input_table = None
self.indexed_output_table = None
self.input_rows = {}
# TODO: it is unnecessary to keep periods which have already been
# simulated, because (currently) when we go back in time, we always go
# back using the output table... but periods before the start_period
# are only present in input_index
self.input_index = {}
self.output_rows = {}
self.output_index = {}
self.output_index_node = None
self.base_period = None
# we need a separate field, instead of using array['period'] to be able
# to get the period even when the array is empty.
self.array_period = array_period
self.array = array
self.lag_fields = []
self.array_lag = None
self.num_tmp = 0
self.temp_variables = {}
self.id_to_rownum = None
if array is not None:
rows_per_period, index_per_period = index_table(array)
self.input_rows = rows_per_period
self.output_rows = rows_per_period
self.input_index = index_per_period
self.output_index = index_per_period
self.id_to_rownum = index_per_period[array_period]
self._variables = None
self._methods = None
def diff_array(array1, array2, numdiff=10, raiseondiff=False):
if len(array1) != len(array2):
print("length is different: %d vs %d" % (len(array1),
len(array2)))
ids1 = array1['id']
ids2 = array2['id']
all_ids = np.union1d(ids1, ids2)
notin1 = np.setdiff1d(ids1, all_ids)
notin2 = np.setdiff1d(ids2, all_ids)
if notin1:
print("the following ids are not present in file 1:", \
notin1)
elif notin2:
print("the following ids are not present in file 2:", \
notin2)
else:
# some ids must be duplicated
if len(ids1) > len(all_ids):
print("file 1 contain duplicate ids:", end=' ')
uniques, dupes = unique_dupes(ids1)
print(dupes)
array1 = array1[uniques]
if len(ids2) > len(all_ids):
print("file 2 contain duplicate ids:", end=' ')
uniques, dupes = unique_dupes(ids2)
print(dupes)
array2 = array2[uniques]
fields1 = get_fields(array1)
fields2 = get_fields(array2)
fnames1 = set(array1.dtype.names)
fnames2 = set(array2.dtype.names)
# use merge_items instead of fnames1 | fnames2 to preserve ordering
for fname, _ in merge_items(fields1, fields2):
print(" - %s:" % fname, end=' ')
if fname not in fnames1:
print("missing in file 1")
continue
elif fname not in fnames2:
print("missing in file 2")
continue
col1, col2 = array1[fname], array2[fname]
if issubclass(col1.dtype.type, np.inexact):
if len(col1) == len(col2):
both_nan = np.isnan(col1) & np.isnan(col2)
eq = np.all(both_nan | (col1 == col2))
else:
eq = False
else:
eq = np.array_equal(col1, col2)
if eq:
print("ok")
else:
print("different", end=' ')
if len(col1) != len(col2):
print("(length)")
else:
diff = (col1 != col2).nonzero()[0]
print("(%d differences)" % len(diff))
ids = array1['id']
if len(diff) > numdiff:
diff = diff[:numdiff]
print(PrettyTable([['id',
fname + ' (file1)',
fname + ' (file2)']] +
[[ids[idx], col1[idx], col2[idx]]
for idx in diff]))
if raiseondiff:
raise Exception('different')
def listSearch():
db=load()
srch=True
field=get_fields(db)
techniques=get_techniques(db)
Search=request.form['Search']
techniqs=request.form.getlist('filters')
sorting=request.form.getlist('Sort')
fields=request.form.getlist('search_fields')
if Search== '':
Search=None
if sorting[0] == 'date':
date=True
else:
date=False
if sorting[0] == 'etad':
etad=True
else:
etad=False
if sorting[0] == 'abcd':
abcd=True
else:
abcd=False
if sorting[0] == 'project_no':
count=True
else:
count=False
Stext=' '
projects=[]
if fields==[]:
if techniqs==[]:
if Search == None:
if count==True:
projects=search(db, sort_by='project_no',sort_order='desc',techniques=None,search=None,search_fields=None)
elif etad==True:
projects=search(db, sort_by='start_date',sort_order='asc',techniques=None,search=None,search_fields=None)
elif abcd==True:
projects=search(db, sort_by='project_name',sort_order='desc',techniques=None,search=None,search_fields=None)
else:
projects=search(db, sort_by='start_date',sort_order='desc',techniques=None,search=None,search_fields=None)
if search(db, sort_by='start_date',sort_order='desc',techniques=None,search=Search,search_fields=None) == 'error':
Stext='Couldn\'t find what you were looking for, please try again.'
srch=False
else:
if count==True:
projects=search(db, sort_by='project_no',sort_order='desc',techniques=None,search=Search,search_fields=None)
elif etad==True:
projects=search(db, sort_by='start_date',sort_order='asc',techniques=None,search=Search,search_fields=None)
elif abcd==True:
projects=search(db, sort_by='project_name',sort_order='desc',techniques=None,search=Search,search_fields=None)
else:
projects=search(db, sort_by='start_date',sort_order='desc',techniques=None,search=Search,search_fields=None)
return render_template('list.html',srch=srch, projects=projects, Stext=Stext, field=field, techniques=techniques)
else:
if Search == None:
if count==True:
projects=search(db, sort_by='project_no',sort_order='desc',techniques=techniqs,search=None,search_fields=None)
elif etad==True:
projects=search(db, sort_by='start_date',sort_order='asc',techniques=techniqs,search=None,search_fields=None)
elif abcd==True:
projects=search(db, sort_by='project_name',sort_order='desc',techniques=techniqs,search=None,search_fields=None)
else:
projects=search(db, sort_by='start_date',sort_order='desc',techniques=techniqs,search=None,search_fields=None)
if search(db, sort_by='start_date',sort_order='desc',techniques=techniqs,search=Search,search_fields=None) == 'error':
Stext='Couldn\'t find what you were looking for, please try again.'
srch=False
else:
if count==True:
projects=search(db, sort_by='project_no',sort_order='desc',techniques=techniqs,search=Search,search_fields=None)
elif etad==True:
projects=search(db, sort_by='start_date',sort_order='asc',techniques=techniqs,search=Search,search_fields=None)
elif abcd==True:
projects=search(db, sort_by='project_name',sort_order='desc',techniques=techniqs,search=Search,search_fields=None)
else:
projects=search(db, sort_by='start_date',sort_order='desc',techniques=techniqs,search=Search,search_fields=None)
return render_template('list.html',srch=srch, projects=projects, Stext=Stext, field=field, techniques=techniques)
else:
if techniqs==[]:
if Search == None:
if count==True:
projects=search(db, sort_by='project_no',sort_order='desc',techniques=None,search=None,search_fields=fields)
elif etad==True:
projects=search(db, sort_by='start_date',sort_order='asc',techniques=None,search=None,search_fields=fields)
elif abcd==True:
projects=search(db, sort_by='project_name',sort_order='desc',techniques=None,search=None,search_fields=fields)
else:
projects=search(db, sort_by='start_date',sort_order='desc',techniques=None,search=None,search_fields=fields)
if search(db, sort_by='start_date',sort_order='desc',techniques=None,search=Search,search_fields=fields) == 'error':
Stext='Couldn\'t find what you were looking for, please try again.'
srch=False
else:
if count==True:
projects=search(db, sort_by='project_no',sort_order='desc',techniques=None,search=Search,search_fields=fields)
elif etad==True:
projects=search(db, sort_by='start_date',sort_order='asc',techniques=None,search=Search,search_fields=fields)
elif abcd==True:
projects=search(db, sort_by='project_name',sort_order='desc',techniques=None,search=Search,search_fields=fields)
else:
projects=search(db, sort_by='start_date',sort_order='desc',techniques=None,search=Search,search_fields=fields)
return render_template('list.html',srch=srch, projects=projects, Stext=Stext, field=field, techniques=techniques)
else:
if Search == None:
if count==True:
projects=search(db, sort_by='project_no',sort_order='desc',techniques=techniqs,search=None,search_fields=fields)
elif etad==True:
projects=search(db, sort_by='start_date',sort_order='asc',techniques=techniqs,search=None,search_fields=fields)
elif abcd==True:
projects=search(db, sort_by='project_name',sort_order='desc',techniques=techniqs,search=None,search_fields=fields)
else:
projects=search(db, sort_by='start_date',sort_order='desc',techniques=techniqs,search=None,search_fields=fields)
if search(db, sort_by='start_date',sort_order='desc',techniques=techniqs,search=Search,search_fields=fields) == 'error':
Stext='Couldn\'t find what you were looking for, please try again.'
srch=False
else:
if count==True:
projects=search(db, sort_by='project_no',sort_order='desc',techniques=techniqs,search=Search,search_fields=fields)
elif etad==True:
projects=search(db, sort_by='start_date',sort_order='asc',techniques=techniqs,search=Search,search_fields=fields)
elif abcd==True:
projects=search(db, sort_by='project_name',sort_order='desc',techniques=techniqs,search=Search,search_fields=fields)
else:
projects=search(db, sort_by='start_date',sort_order='desc',techniques=techniqs,search=Search,search_fields=fields)
return render_template('list.html',srch=srch, projects=projects, Stext=Stext, field=field, techniques=techniques)
def __init__(self, name, fields=None, missing_fields=None, links=None,
macro_strings=None, process_strings=None,
array=None):
self.name = name
# we should have exactly one of either array or fields defined
assert ((fields is None and array is not None) or
(fields is not None and array is None))
if array is not None:
if fields is None:
fields = get_fields(array)
array_period = np.min(array['period'])
else:
array_period = None
duplicate_names = [name
for name, num
in count_occurrences(fname for fname, _ in fields)
if num > 1]
if duplicate_names:
raise Exception("duplicate fields in entity '%s': %s"
% (self.name, ', '.join(duplicate_names)))
fnames = [name for name, _ in fields]
if 'id' not in fnames:
fields.insert(0, ('id', int))
if 'period' not in fnames:
fields.insert(0, ('period', int))
self.fields = fields
# only used in data (to check that all "required" fields are present
# in the input file)
# one potential solution would be to split the fields argument and
# attribute in input_fields and output_fields (regardless of whether
# it is split in the simulation/yaml file).
# however that might be just a temporary solution as we will soon need
# more arguments to fields (default values, ranges, etc...)
# another solution is to use a Field class
# seems like the better long term solution
self.missing_fields = missing_fields
self.stored_fields = set(name for name, _ in fields)
self.links = links
self.macro_strings = macro_strings
self.process_strings = process_strings
self.processes = None
self.expectedrows = tables.parameters.EXPECTED_ROWS_TABLE
self.table = None
self.input_table = None
self.indexed_input_table = None
self.indexed_output_table = None
self.input_rows = {}
#TODO: it is unnecessary to keep periods which have already been
# simulated, because (currently) when we go back in time, we always go
# back using the output table.
self.input_index = {}
self.output_rows = {}
self.output_index = {}
self.base_period = None
# we need a separate field, instead of using array['period'] to be able
# to get the period even when the array is empty.
self.array_period = array_period
self.array = None
self.lag_fields = []
self.array_lag = None
self.num_tmp = 0
self.temp_variables = {}
self.id_to_rownum = None
self._variables = None
self._methods = None
def get_h5_fields(input_file):
return dict((table._v_name, get_fields(table))
for table in input_file.iterNodes(input_file.root.entities))
def diff_array(array1, array2, showdiffs=10, raiseondiff=False):
if len(array1) != len(array2):
print("length is different: %d vs %d" % (len(array1), len(array2)))
ids1 = array1['id']
ids2 = array2['id']
all_ids = np.union1d(ids1, ids2)
notin1 = np.setdiff1d(ids1, all_ids)
notin2 = np.setdiff1d(ids2, all_ids)
if notin1:
print("the following ids are not present in file 1:", notin1)
elif notin2:
print("the following ids are not present in file 2:", notin2)
else:
# some ids must be duplicated
if len(ids1) > len(all_ids):
print("file 1 contain duplicate ids:", end=' ')
uniques, dupes = unique_dupes(ids1)
print(dupes)
array1 = array1[uniques]
if len(ids2) > len(all_ids):
print("file 2 contain duplicate ids:", end=' ')
uniques, dupes = unique_dupes(ids2)
print(dupes)
array2 = array2[uniques]
fields1 = get_fields(array1)
fields2 = get_fields(array2)
fnames1 = set(array1.dtype.names)
fnames2 = set(array2.dtype.names)
# use merge_items instead of fnames1 | fnames2 to preserve ordering
for fname, _ in merge_items(fields1, fields2):
print(" - %s:" % fname, end=' ')
if fname not in fnames1:
print("missing in file 1")
continue
elif fname not in fnames2:
print("missing in file 2")
continue
col1, col2 = array1[fname], array2[fname]
if np.issubdtype(col1.dtype, np.inexact):
if len(col1) == len(col2):
both_nan = np.isnan(col1) & np.isnan(col2)
eq = np.all(both_nan | (col1 == col2))
else:
eq = False
else:
eq = np.array_equal(col1, col2)
if eq:
print("ok")
else:
print("different", end=' ')
if len(col1) != len(col2):
print("(length)")
else:
diff = (col1 != col2).nonzero()[0]
print("(%d differences)" % len(diff))
ids = array1['id']
if len(diff) > showdiffs:
diff = diff[:showdiffs]
print(
PrettyTable(
[['id', fname + ' (file1)', fname + ' (file2)']] +
[[ids[idx], col1[idx], col2[idx]] for idx in diff]))
if raiseondiff:
raise Exception('different')
