def dropfields(input_path, output_path, todrop): input_file = tables.open_file(input_path, mode="r") input_root = input_file.root output_file = tables.open_file(output_path, mode="w") output_globals = output_file.create_group("/", "globals", "Globals") print(" * copying globals ...", end=' ') copy_table(input_root.globals.periodic, output_globals) print("done.") output_entities = output_file.create_group("/", "entities", "Entities") for table in input_file.iterNodes(input_root.entities): table_fields = get_fields(table) table_fields = [(fname, ftype) for fname, ftype in table_fields if fname not in todrop] size = (len(table) * table.dtype.itemsize) / 1024.0 / 1024.0 #noinspection PyProtectedMember print(" * copying table %s (%.2f Mb) ..." % (table._v_name, size), end=' ') copy_table(table, output_entities, table_fields) print("done.") input_file.close() output_file.close()
def dropfields(input_path, output_path, todrop): input_file = tables.open_file(input_path, mode="r") input_root = input_file.root output_file = tables.open_file(output_path, mode="w") output_globals = output_file.create_group("/", "globals", "Globals") print(" * copying globals ...", end=' ') copy_table(input_root.globals.periodic, output_globals) print("done.") output_entities = output_file.create_group("/", "entities", "Entities") for table in input_file.iterNodes(input_root.entities): table_fields = get_fields(table) output_dtype = np.dtype([(fname, ftype) for fname, ftype in table_fields if fname not in todrop]) size = (len(table) * table.dtype.itemsize) / 1024.0 / 1024.0 # noinspection PyProtectedMember print(" * copying table %s (%.2f Mb) ..." % (table._v_name, size), end=' ') copy_table(table, output_entities, output_dtype) print("done.") input_file.close() output_file.close()
def search(): app.logger.debug('search') #abort(make_response('Not implemented yet ;)', 501)) if request.method == 'POST': ID = request.form['ID'] if ID: upvote(int(ID)) response = render_template('latest.html', images=IMAGES, themes=THEMES, buttons=BUTTONS) if request.method == 'GET': sw = request.args.get('pattern') r = request.args.get('regexp') RES = [] if sw: if r == "on": # RECHERCHE PAR THEME for img in IMAGES: if sw in get_fields(img['id']): RES.append(img) else: # RECHERCHE PAR MOTS CLES for img in IMAGES: if sw.capitalize() in img['title'].capitalize(): RES.append(img) response = render_template('search.html', images=RES, themes=THEMES, sw=sw, r=r, buttons=BUTTONS) else: response = render_template('search.html') return response
def listPage(): if request.method == 'GET': db=load() srch=True techniques=get_techniques(db) projects=search(db, sort_by='start_date',sort_order='desc',techniques=None,search=None,search_fields=None) field=get_fields(db) return render_template('list.html',srch=srch,projects=projects, field=field, techniques=techniques)
def list_page(): """List Page""" db = data.init() fields = data.get_fields(db) #If user is searhing (POST) if request.method == 'POST': search_list = [] for x in data.get_fields(db): #only add to list if x exists otherwise pass try: search_list.append(request.form[x]) except: pass if search_list == []: search_list = None #Selects how it should sort the list sort_order = request.form["sort_order"] db = data.search(db, search=request.form["search"],sort_order=sort_order, search_fields=search_list) return render_template("list.html",dataB = db, _fields = fields)
def dropfields(input_path, output_path, todrop): input_file = tables.openFile(input_path, mode="r") input_root = input_file.root output_file = tables.openFile(output_path, mode="w") output_globals = output_file.createGroup("/", "globals", "Globals") print " * copying globals ...", copyTable(input_root.globals.periodic, output_file, output_globals) print "done." output_entities = output_file.createGroup("/", "entities", "Entities") for table in input_file.iterNodes(input_root.entities): table_fields = get_fields(table) table_fields = [(fname, ftype) for fname, ftype in table_fields if fname not in todrop] print " * copying table %s (%.2f Mb) ..." % (table._v_name, table_size(table)), copyTable(table, output_file, output_entities, table_fields) print "done." input_file.close() output_file.close()
def from_table(cls, table): return Entity(table.name, get_fields(table), missing_fields=[], links={}, macro_strings={}, process_strings={})
def get_group_fields(node): if node is None: return {} # noinspection PyProtectedMember return {table._v_name: get_fields(table) for table in node._f_iter_nodes()}
def get_h5_fields(input_file): #noinspection PyProtectedMember return dict((table._v_name, get_fields(table)) for table in input_file.iterNodes(input_file.root.entities))
def from_table(cls, table): return Entity(table.name, get_fields(table), links={}, macro_strings={}, process_strings={})
def __init__(self, name, fields=None, links=None, macro_strings=None, process_strings=None, array=None): """ Parameters ---------- name fields : list of tuple (name, type) links : {name: links.Link} macro_strings process_strings array """ self.name = name # we should have exactly one of either array or fields defined assert ((fields is None and array is not None) or (fields is not None and array is None)) if array is not None: if fields is None: fields = get_fields(array) array_period = np.min(array['period']) else: array_period = None if not isinstance(fields, FieldCollection): def fdef2field(name, fielddef): initialdata = True output = True default_value = None if isinstance(fielddef, Field): return fielddef elif isinstance(fielddef, (dict, str)): if isinstance(fielddef, dict): strtype = fielddef['type'] initialdata = fielddef.get('initialdata', True) output = fielddef.get('output', True) default_value = fielddef.get( 'default', default_value_by_strtype[strtype]) elif isinstance(fielddef, str): strtype = fielddef default_value = default_value_by_strtype[strtype] else: raise Exception('invalid field definition') dtype = field_str_to_type(strtype, "field '%s'" % name) else: assert isinstance(fielddef, type) dtype = normalize_type(fielddef) return Field(name, dtype, initialdata, output, default_value) fields = FieldCollection( fdef2field(name, fdef) for name, fdef in fields) duplicate_names = [ name for name, num in count_occurrences(fields.names) if num > 1 ] if duplicate_names: raise Exception("duplicate fields in entity '%s': %s" % (self.name, ', '.join(duplicate_names))) fnames = set(fields.names) if 'id' not in fnames: fields.insert(0, Field('id', int)) if 'period' not in fnames: fields.insert(0, Field('period', int)) self.fields = fields self.links = links if macro_strings is None: macro_strings = {} self.macro_strings = macro_strings self.process_strings = process_strings self.processes = None self.expectedrows = tables.parameters.EXPECTED_ROWS_TABLE self.table = None self.input_table = None self.indexed_input_table = None self.indexed_output_table = None self.input_rows = {} # TODO: it is unnecessary to keep periods which have already been # simulated, because (currently) when we go back in time, we always go # back using the output table... but periods before the start_period # are only present in input_index self.input_index = {} self.output_rows = {} self.output_index = {} self.output_index_node = None self.base_period = None # we need a separate field, instead of using array['period'] to be able # to get the period even when the array is empty. self.array_period = array_period self.array = array self.lag_fields = [] self.array_lag = None self.num_tmp = 0 self.temp_variables = {} self.id_to_rownum = None if array is not None: rows_per_period, index_per_period = index_table(array) self.input_rows = rows_per_period self.output_rows = rows_per_period self.input_index = index_per_period self.output_index = index_per_period self.id_to_rownum = index_per_period[array_period] self._variables = None self._methods = None
def __init__(self, name, fields=None, links=None, macro_strings=None, process_strings=None, array=None): self.name = name # we should have exactly one of either array or fields defined assert ((fields is None and array is not None) or (fields is not None and array is None)) if array is not None: if fields is None: fields = get_fields(array) array_period = np.min(array['period']) else: array_period = None if not isinstance(fields, FieldCollection): def fdef2field(name, fielddef): initialdata = True output = True default_value = None if isinstance(fielddef, Field): return fielddef elif isinstance(fielddef, (dict, str)): if isinstance(fielddef, dict): strtype = fielddef['type'] initialdata = fielddef.get('initialdata', True) output = fielddef.get('output', True) default_value = fielddef.get('default', default_value_by_strtype[strtype]) elif isinstance(fielddef, str): strtype = fielddef default_value = default_value_by_strtype[strtype] else: raise Exception('invalid field definition') dtype = field_str_to_type(strtype, "field '%s'" % name) else: assert isinstance(fielddef, type) dtype = normalize_type(fielddef) return Field(name, dtype, initialdata, output, default_value) fields = FieldCollection(fdef2field(name, fdef) for name, fdef in fields) duplicate_names = [name for name, num in count_occurrences(fields.names) if num > 1] if duplicate_names: raise Exception("duplicate fields in entity '%s': %s" % (self.name, ', '.join(duplicate_names))) fnames = set(fields.names) if 'id' not in fnames: fields.insert(0, Field('id', int)) if 'period' not in fnames: fields.insert(0, Field('period', int)) self.fields = fields self.links = links if macro_strings is None: macro_strings = {} self.macro_strings = macro_strings self.process_strings = process_strings self.processes = None self.expectedrows = tables.parameters.EXPECTED_ROWS_TABLE self.table = None self.input_table = None self.indexed_input_table = None self.indexed_output_table = None self.input_rows = {} # TODO: it is unnecessary to keep periods which have already been # simulated, because (currently) when we go back in time, we always go # back using the output table... but periods before the start_period # are only present in input_index self.input_index = {} self.output_rows = {} self.output_index = {} self.output_index_node = None self.base_period = None # we need a separate field, instead of using array['period'] to be able # to get the period even when the array is empty. self.array_period = array_period self.array = array self.lag_fields = [] self.array_lag = None self.num_tmp = 0 self.temp_variables = {} self.id_to_rownum = None if array is not None: rows_per_period, index_per_period = index_table(array) self.input_rows = rows_per_period self.output_rows = rows_per_period self.input_index = index_per_period self.output_index = index_per_period self.id_to_rownum = index_per_period[array_period] self._variables = None self._methods = None
def diff_array(array1, array2, numdiff=10, raiseondiff=False): if len(array1) != len(array2): print("length is different: %d vs %d" % (len(array1), len(array2))) ids1 = array1['id'] ids2 = array2['id'] all_ids = np.union1d(ids1, ids2) notin1 = np.setdiff1d(ids1, all_ids) notin2 = np.setdiff1d(ids2, all_ids) if notin1: print("the following ids are not present in file 1:", \ notin1) elif notin2: print("the following ids are not present in file 2:", \ notin2) else: # some ids must be duplicated if len(ids1) > len(all_ids): print("file 1 contain duplicate ids:", end=' ') uniques, dupes = unique_dupes(ids1) print(dupes) array1 = array1[uniques] if len(ids2) > len(all_ids): print("file 2 contain duplicate ids:", end=' ') uniques, dupes = unique_dupes(ids2) print(dupes) array2 = array2[uniques] fields1 = get_fields(array1) fields2 = get_fields(array2) fnames1 = set(array1.dtype.names) fnames2 = set(array2.dtype.names) # use merge_items instead of fnames1 | fnames2 to preserve ordering for fname, _ in merge_items(fields1, fields2): print(" - %s:" % fname, end=' ') if fname not in fnames1: print("missing in file 1") continue elif fname not in fnames2: print("missing in file 2") continue col1, col2 = array1[fname], array2[fname] if issubclass(col1.dtype.type, np.inexact): if len(col1) == len(col2): both_nan = np.isnan(col1) & np.isnan(col2) eq = np.all(both_nan | (col1 == col2)) else: eq = False else: eq = np.array_equal(col1, col2) if eq: print("ok") else: print("different", end=' ') if len(col1) != len(col2): print("(length)") else: diff = (col1 != col2).nonzero()[0] print("(%d differences)" % len(diff)) ids = array1['id'] if len(diff) > numdiff: diff = diff[:numdiff] print(PrettyTable([['id', fname + ' (file1)', fname + ' (file2)']] + [[ids[idx], col1[idx], col2[idx]] for idx in diff])) if raiseondiff: raise Exception('different')
def listSearch(): db=load() srch=True field=get_fields(db) techniques=get_techniques(db) Search=request.form['Search'] techniqs=request.form.getlist('filters') sorting=request.form.getlist('Sort') fields=request.form.getlist('search_fields') if Search== '': Search=None if sorting[0] == 'date': date=True else: date=False if sorting[0] == 'etad': etad=True else: etad=False if sorting[0] == 'abcd': abcd=True else: abcd=False if sorting[0] == 'project_no': count=True else: count=False Stext=' ' projects=[] if fields==[]: if techniqs==[]: if Search == None: if count==True: projects=search(db, sort_by='project_no',sort_order='desc',techniques=None,search=None,search_fields=None) elif etad==True: projects=search(db, sort_by='start_date',sort_order='asc',techniques=None,search=None,search_fields=None) elif abcd==True: projects=search(db, sort_by='project_name',sort_order='desc',techniques=None,search=None,search_fields=None) else: projects=search(db, sort_by='start_date',sort_order='desc',techniques=None,search=None,search_fields=None) if search(db, sort_by='start_date',sort_order='desc',techniques=None,search=Search,search_fields=None) == 'error': Stext='Couldn\'t find what you were looking for, please try again.' srch=False else: if count==True: projects=search(db, sort_by='project_no',sort_order='desc',techniques=None,search=Search,search_fields=None) elif etad==True: projects=search(db, sort_by='start_date',sort_order='asc',techniques=None,search=Search,search_fields=None) elif abcd==True: projects=search(db, sort_by='project_name',sort_order='desc',techniques=None,search=Search,search_fields=None) else: projects=search(db, sort_by='start_date',sort_order='desc',techniques=None,search=Search,search_fields=None) return render_template('list.html',srch=srch, projects=projects, Stext=Stext, field=field, techniques=techniques) else: if Search == None: if count==True: projects=search(db, sort_by='project_no',sort_order='desc',techniques=techniqs,search=None,search_fields=None) elif etad==True: projects=search(db, sort_by='start_date',sort_order='asc',techniques=techniqs,search=None,search_fields=None) elif abcd==True: projects=search(db, sort_by='project_name',sort_order='desc',techniques=techniqs,search=None,search_fields=None) else: projects=search(db, sort_by='start_date',sort_order='desc',techniques=techniqs,search=None,search_fields=None) if search(db, sort_by='start_date',sort_order='desc',techniques=techniqs,search=Search,search_fields=None) == 'error': Stext='Couldn\'t find what you were looking for, please try again.' srch=False else: if count==True: projects=search(db, sort_by='project_no',sort_order='desc',techniques=techniqs,search=Search,search_fields=None) elif etad==True: projects=search(db, sort_by='start_date',sort_order='asc',techniques=techniqs,search=Search,search_fields=None) elif abcd==True: projects=search(db, sort_by='project_name',sort_order='desc',techniques=techniqs,search=Search,search_fields=None) else: projects=search(db, sort_by='start_date',sort_order='desc',techniques=techniqs,search=Search,search_fields=None) return render_template('list.html',srch=srch, projects=projects, Stext=Stext, field=field, techniques=techniques) else: if techniqs==[]: if Search == None: if count==True: projects=search(db, sort_by='project_no',sort_order='desc',techniques=None,search=None,search_fields=fields) elif etad==True: projects=search(db, sort_by='start_date',sort_order='asc',techniques=None,search=None,search_fields=fields) elif abcd==True: projects=search(db, sort_by='project_name',sort_order='desc',techniques=None,search=None,search_fields=fields) else: projects=search(db, sort_by='start_date',sort_order='desc',techniques=None,search=None,search_fields=fields) if search(db, sort_by='start_date',sort_order='desc',techniques=None,search=Search,search_fields=fields) == 'error': Stext='Couldn\'t find what you were looking for, please try again.' srch=False else: if count==True: projects=search(db, sort_by='project_no',sort_order='desc',techniques=None,search=Search,search_fields=fields) elif etad==True: projects=search(db, sort_by='start_date',sort_order='asc',techniques=None,search=Search,search_fields=fields) elif abcd==True: projects=search(db, sort_by='project_name',sort_order='desc',techniques=None,search=Search,search_fields=fields) else: projects=search(db, sort_by='start_date',sort_order='desc',techniques=None,search=Search,search_fields=fields) return render_template('list.html',srch=srch, projects=projects, Stext=Stext, field=field, techniques=techniques) else: if Search == None: if count==True: projects=search(db, sort_by='project_no',sort_order='desc',techniques=techniqs,search=None,search_fields=fields) elif etad==True: projects=search(db, sort_by='start_date',sort_order='asc',techniques=techniqs,search=None,search_fields=fields) elif abcd==True: projects=search(db, sort_by='project_name',sort_order='desc',techniques=techniqs,search=None,search_fields=fields) else: projects=search(db, sort_by='start_date',sort_order='desc',techniques=techniqs,search=None,search_fields=fields) if search(db, sort_by='start_date',sort_order='desc',techniques=techniqs,search=Search,search_fields=fields) == 'error': Stext='Couldn\'t find what you were looking for, please try again.' srch=False else: if count==True: projects=search(db, sort_by='project_no',sort_order='desc',techniques=techniqs,search=Search,search_fields=fields) elif etad==True: projects=search(db, sort_by='start_date',sort_order='asc',techniques=techniqs,search=Search,search_fields=fields) elif abcd==True: projects=search(db, sort_by='project_name',sort_order='desc',techniques=techniqs,search=Search,search_fields=fields) else: projects=search(db, sort_by='start_date',sort_order='desc',techniques=techniqs,search=Search,search_fields=fields) return render_template('list.html',srch=srch, projects=projects, Stext=Stext, field=field, techniques=techniques)
def __init__(self, name, fields=None, missing_fields=None, links=None, macro_strings=None, process_strings=None, array=None): self.name = name # we should have exactly one of either array or fields defined assert ((fields is None and array is not None) or (fields is not None and array is None)) if array is not None: if fields is None: fields = get_fields(array) array_period = np.min(array['period']) else: array_period = None duplicate_names = [name for name, num in count_occurrences(fname for fname, _ in fields) if num > 1] if duplicate_names: raise Exception("duplicate fields in entity '%s': %s" % (self.name, ', '.join(duplicate_names))) fnames = [name for name, _ in fields] if 'id' not in fnames: fields.insert(0, ('id', int)) if 'period' not in fnames: fields.insert(0, ('period', int)) self.fields = fields # only used in data (to check that all "required" fields are present # in the input file) # one potential solution would be to split the fields argument and # attribute in input_fields and output_fields (regardless of whether # it is split in the simulation/yaml file). # however that might be just a temporary solution as we will soon need # more arguments to fields (default values, ranges, etc...) # another solution is to use a Field class # seems like the better long term solution self.missing_fields = missing_fields self.stored_fields = set(name for name, _ in fields) self.links = links self.macro_strings = macro_strings self.process_strings = process_strings self.processes = None self.expectedrows = tables.parameters.EXPECTED_ROWS_TABLE self.table = None self.input_table = None self.indexed_input_table = None self.indexed_output_table = None self.input_rows = {} #TODO: it is unnecessary to keep periods which have already been # simulated, because (currently) when we go back in time, we always go # back using the output table. self.input_index = {} self.output_rows = {} self.output_index = {} self.base_period = None # we need a separate field, instead of using array['period'] to be able # to get the period even when the array is empty. self.array_period = array_period self.array = None self.lag_fields = [] self.array_lag = None self.num_tmp = 0 self.temp_variables = {} self.id_to_rownum = None self._variables = None self._methods = None
def get_h5_fields(input_file): return dict((table._v_name, get_fields(table)) for table in input_file.iterNodes(input_file.root.entities))
def diff_array(array1, array2, showdiffs=10, raiseondiff=False): if len(array1) != len(array2): print("length is different: %d vs %d" % (len(array1), len(array2))) ids1 = array1['id'] ids2 = array2['id'] all_ids = np.union1d(ids1, ids2) notin1 = np.setdiff1d(ids1, all_ids) notin2 = np.setdiff1d(ids2, all_ids) if notin1: print("the following ids are not present in file 1:", notin1) elif notin2: print("the following ids are not present in file 2:", notin2) else: # some ids must be duplicated if len(ids1) > len(all_ids): print("file 1 contain duplicate ids:", end=' ') uniques, dupes = unique_dupes(ids1) print(dupes) array1 = array1[uniques] if len(ids2) > len(all_ids): print("file 2 contain duplicate ids:", end=' ') uniques, dupes = unique_dupes(ids2) print(dupes) array2 = array2[uniques] fields1 = get_fields(array1) fields2 = get_fields(array2) fnames1 = set(array1.dtype.names) fnames2 = set(array2.dtype.names) # use merge_items instead of fnames1 | fnames2 to preserve ordering for fname, _ in merge_items(fields1, fields2): print(" - %s:" % fname, end=' ') if fname not in fnames1: print("missing in file 1") continue elif fname not in fnames2: print("missing in file 2") continue col1, col2 = array1[fname], array2[fname] if np.issubdtype(col1.dtype, np.inexact): if len(col1) == len(col2): both_nan = np.isnan(col1) & np.isnan(col2) eq = np.all(both_nan | (col1 == col2)) else: eq = False else: eq = np.array_equal(col1, col2) if eq: print("ok") else: print("different", end=' ') if len(col1) != len(col2): print("(length)") else: diff = (col1 != col2).nonzero()[0] print("(%d differences)" % len(diff)) ids = array1['id'] if len(diff) > showdiffs: diff = diff[:showdiffs] print( PrettyTable( [['id', fname + ' (file1)', fname + ' (file2)']] + [[ids[idx], col1[idx], col2[idx]] for idx in diff])) if raiseondiff: raise Exception('different')