def _load_paramtable(self, file_name=LocalData.default_data_file): # Load parameter table if self._metadata is None: self._metadata = LocalData(file_name) elif not self._metadata.is_db_path_correct(file_name): self._metadata.close() self._metadata = LocalData(file_name) self._param_table = self._metadata.get_data( "params", "ems_id = %d" % self._ems_id)
def _load_paramtable(self, file_name=None): if self._metadata is None: self._metadata = LocalData(file_name) else: if (file_name is None) and (self._metadata.file_loc() != os.path.abspath(file_name)): self._metadata.close() self._metadata = LocalData(file_name) self._param_table = self._metadata.get_data( "params", "ems_id = %d" % self._ems_id)
def save_tree(self, file_name=None): from shutil import copyfile ld = self._metadata if (file_name is not None) and (ld.file_loc() != os.path.abspath(file_name)): # A new file location is given, copy all the data in the current file with new file name, # and save the currently loaded tree data into the new file too. copyfile(self._metadata.file_loc(), file_name) self._metadata.close() self._metadata = LocalData(file_name) self.__save_fieldtree() self.__save_dbtree() self.__save_kvmaps()
def load_tree(self, file_name=None): if self._metadata is None: self._metadata = LocalData(file_name) else: if (file_name is not None) and (self._metadata.file_loc() != os.path.abspath(file_name)): self._metadata.close() self._metadata = LocalData(file_name) self._trees = { 'fieldtree': self.__get_fieldtree(), 'dbtree': self.__get_dbtree(), 'kvmaps': self.__get_kvmaps() }
def save_tree(self, file_name=LocalData.default_data_file): """ Method to save trees to a defined database file Parameters ---------- file_name: str path to database file Returns ------- None """ from shutil import copyfile ld = self._metadata if not ld.is_db_path_correct(file_name): # A different location is given, if a previous file exists copy it to the new location. if ld.file_loc() is not None and file_name is not None: copyfile(ld.file_loc(), os.path.abspath(file_name)) self._metadata.close() self._metadata = LocalData(file_name) self.__save_fieldtree() self.__save_dbtree() self.__save_kvmaps()
def load_tree(self, file_name=LocalData.default_data_file): """ Load trees into the object Parameters ---------- file_name: str path to database file Returns ------- None """ if self._metadata is None: self._metadata = LocalData(file_name) elif not self._metadata.is_db_path_correct(file_name): self._metadata.close() self._metadata = LocalData(file_name) self._trees = { 'fieldtree': self.__get_fieldtree(), 'dbtree': self.__get_dbtree(), 'kvmaps': self.__get_kvmaps() }
class Flight: temp_exclude = [ 'Download Information', 'Download Review', 'Processing', 'Profile 16 Extra Data', 'Flight Review', 'Data Information', 'Operational Information', 'Operational Information (ODW2)', 'Weather Information', 'Profiles', 'Profile' ] def __init__(self, conn, ems_id, data_file=None): self._conn = conn self._ems_id = ems_id self._db_id = None self._metadata = None self._trees = {'fieldtree': None, 'dbtree': None, 'kvmaps': None} self._fields = [] self.__cntr = 0 # Retreive the field tree data from local storage. If it doesn't exist, generate a new # default one. self.load_tree(data_file) # Set default database # self.set_database('fdw flight') def load_tree(self, file_name=None): if self._metadata is None: self._metadata = LocalData(file_name) else: if (file_name is not None) and (self._metadata.file_loc() != os.path.abspath(file_name)): self._metadata.close() self._metadata = LocalData(file_name) self._trees = { 'fieldtree': self.__get_fieldtree(), 'dbtree': self.__get_dbtree(), 'kvmaps': self.__get_kvmaps() } def save_tree(self, file_name=None): from shutil import copyfile ld = self._metadata if (file_name is not None) and (ld.file_loc() != os.path.abspath(file_name)): # A new file location is given, copy all the data in the current file with new file name, # and save the currently loaded tree data into the new file too. copyfile(self._metadata.file_loc(), file_name) self._metadata.close() self._metadata = LocalData(file_name) self.__save_fieldtree() self.__save_dbtree() self.__save_kvmaps() def __get_fieldtree(self): if self._db_id is None: return pd.DataFrame(columns=self._metadata.table_info['fieldtree']) else: return self._metadata.get_data( "fieldtree", "ems_id = %d and db_id = '%s'" % (self._ems_id, self._db_id)) def __save_fieldtree(self): if len(self._trees['fieldtree']) > 0: self._metadata.delete_data( "fieldtree", "ems_id = %d and db_id = '%s'" % (self._ems_id, self._db_id)) self._metadata.append_data("fieldtree", self._trees['fieldtree']) def __get_dbtree(self): T = self._metadata.get_data("dbtree", "ems_id = %d" % self._ems_id) if len(T) < 1: dbroot = { 'ems_id': self._ems_id, 'id': "[-hub-][entity-type-group][[--][internal-type-group][root]]", 'name': "<root>", 'nodetype': "root", 'parent_id': None } self._trees['dbtree'] = pd.DataFrame([dbroot]) self.__update_children(dbroot, treetype="dbtree") self.update_tree("fdw", treetype="dbtree", exclude_tree=["APM Events"]) self.__save_dbtree() T = self._trees['dbtree'] return T def __save_dbtree(self): if len(self._trees['dbtree']) > 0: self._metadata.delete_data("dbtree", "ems_id = %d" % self._ems_id) self._metadata.append_data("dbtree", self._trees['dbtree']) def __get_kvmaps(self): T = self._metadata.get_data("kvmaps", "ems_id = %d" % self._ems_id) return T def __save_kvmaps(self): if len(self._trees['kvmaps']) > 0: self._metadata.delete_data("kvmaps", "ems_id = %d" % self._ems_id) self._metadata.append_data("kvmaps", self._trees['kvmaps']) def set_database(self, name): tr = self._trees['dbtree'] self._db_id = tr[(tr.nodetype == 'database') & tr.name.str.contains( treat_spchar(name), case=False)]['id'].values[0] self._trees['fieldtree'] = self.__get_fieldtree() if self._trees['fieldtree'].empty: self.__update_children(self.get_database(), treetype="fieldtree") print("Using database '%s'." % self.get_database()['name']) def get_database(self): tr = self._trees['dbtree'] return tr[(tr.nodetype == "database") & (tr.id == self._db_id)].iloc[0].to_dict() def __db_request(self, parent): body = None if parent['nodetype'] == "database_group": body = {'groupId': parent['id']} resp_h, d = self._conn.request(uri_keys=('database', 'group'), uri_args=self._ems_id, body=body) d1 = [] if len(d['databases']) > 0: d1 = map( lambda x: { 'ems_id': parent['ems_id'], 'id': x['id'], 'nodetype': 'database', 'name': x['pluralName'], 'parent_id': parent['id'] }, d['databases']) d2 = [] if len(d['groups']) > 0: d2 = map( lambda x: { 'ems_id': parent['ems_id'], 'id': x['id'], 'nodetype': 'database_group', 'name': x['name'], 'parent_id': parent['id'] }, d['groups']) return d1, d2 def __fl_request(self, parent): body = None if parent['nodetype'] == "field_group": body = {'groupId': parent['id']} resp_h, d = self._conn.request(uri_keys=('database', 'field_group'), uri_args=(self._ems_id, self._db_id), body=body) d1 = [] if len(d['fields']) > 0: d1 = map( lambda x: { 'ems_id': parent['ems_id'], 'db_id': self._db_id, 'id': x['id'], 'nodetype': 'field', 'type': x['type'], 'name': x['name'], 'parent_id': parent['id'] }, d['fields']) d2 = [] if len(d['groups']) > 0: d2 = map( lambda x: { 'ems_id': parent['ems_id'], 'db_id': self._db_id, 'id': x['id'], 'nodetype': 'field_group', 'type': None, 'name': x['name'], 'parent_id': parent['id'] }, d['groups']) return d1, d2 def __add_subtree(self, parent, exclude_tree=[], treetype='fieldtree'): print("On " + parent['name'] + "(" + parent['nodetype'] + ")" + "...") if treetype == "dbtree": searchtype = 'database' d1, d2 = self.__db_request(parent) else: searchtype = "field" d1, d2 = self.__fl_request(parent) if len(d1) > 0: self._trees[treetype] = self._trees[treetype].append( d1, ignore_index=True) plural = "s" if len(d1) > 1 else "" print("-- Added %d %s%s" % (len(d1), searchtype, plural)) for x in d2: self._trees[treetype] = self._trees[treetype].append( x, ignore_index=True) if len(exclude_tree) > 0: if all([y not in x['name'] for y in exclude_tree]): self.__add_subtree(x, exclude_tree, treetype) else: self.__add_subtree(x, exclude_tree, treetype) def __get_children(self, parent_id, treetype='fieldtree'): tr = self._trees[treetype] # tr = tr[tr.nodetype != ('field' if treetype=='fieldtree' else 'dbtree')] if isinstance(parent_id, (list, tuple, pd.Series)): return tr[tr.parent_id.isin(parent_id)] return tr[tr.parent_id == parent_id] def __remove_subtree(self, parent, treetype='fieldtree'): tr = self._trees[treetype] chld = tr[tr.parent_id == parent['id']] # Update the instance tree by deleting children self._trees[treetype] = tr[tr.parent_id != parent['id']] # Iterate and do recursive removal of children of children leaftype = 'field' if treetype == 'fieldtree' else 'database' for i, x in chld[chld.nodetype != leaftype].iterrows(): self.__remove_subtree(x, treetype=treetype) # def __remove_subtree(self, parent, rm_parent=True, treetype='fieldtree'): # rm_list = list() # if rm_parent: # rm_list.append(parent['id']) # parent_id = [parent['id']] # cntr = 0 # while len(parent_id) > 0: # child_id = self.__get_children(parent_id, treetype=treetype)['id'].tolist() # rm_list += child_id # parent_id = child_id # cntr += 1 # if cntr > 1e4: # sys.exit("Something's wrong. Subtree removal went over 10,000 iterations.") # if len(rm_list) > 0: # tr = self._trees[treetype] # self._trees[treetype] = tr[~tr.id.isin(rm_list)] # print("Removed the subtree of %s (%s) with total of %d nodes (fields/databases/groups)." % ( # parent['name'], parent['nodetype'], len(rm_list))) def __update_children(self, parent, treetype='fieldtree'): ''' This function updates the direct children of a parent node. ''' print("On " + parent['name'] + "(" + parent['nodetype'] + ")" + "...") if treetype == "dbtree": searchtype = 'database' d1, d2 = self.__db_request(parent) else: searchtype = "field" d1, d2 = self.__fl_request(parent) T = self._trees[treetype] self._trees[treetype] = T[~((T.nodetype == searchtype) & (T.parent_id == parent['id']))] if len(d1) > 0: self._trees[treetype] = self._trees[treetype].append( d1, ignore_index=True) plural = "s" if len(d1) > 1 else "" print("-- Added %d %s%s" % (len(d1), searchtype, plural)) # If there is an array of groups as children add any that appeared new and remove who does not. old_groups = T[(T.nodetype == '%s_group' % searchtype) & (T.parent_id == parent['id'])] old_ones = old_groups["id"].tolist() new_ones = [x['id'] for x in d2] rm_id = listdiff(old_ones, new_ones) if len(rm_id) > 0: [ self.__remove_subtree(x.to_dict(), treetype=treetype) for i, x in old_groups.iterrows() if x['id'] in rm_id ] add_id = listdiff(new_ones, old_ones) if len(add_id) > 0: self._trees[treetype] = self._trees[treetype].append( [x for x in d2 if x['id'] in add_id]) def update_tree(self, *args, **kwargs): ''' Optional arguments ------------------ treetype : "fieldtree" or "dbtree" exclude_tree: Exact name strings (case sensitive) of the field groups you don't want to search through Ex. ['Profiles', 'Weather Information'] ''' treetype = kwargs.get("treetype", "fieldtree") exclude_tree = kwargs.get("exclude_tree", []) searchtype = "field" if treetype == "fieldtree" else "database" if treetype not in ("fieldtree", "dbtree"): raise ValueError("treetype = '%s': there is no such data table." % treetype) fld_path = [s.lower() for s in args] for i, p in enumerate(fld_path): p = treat_spchar(p) if i == 0: T = self._trees[treetype] parent = T[T.name.str.contains(p, case=False)] else: self.__update_children(parent, treetype=treetype) chld_df = self.__get_children(parent['id'], treetype=treetype) child = chld_df[chld_df.name.str.contains(p, case=False)] parent = child if len(parent) == 0: raise ValueError( "Search keyword '%s' did not return any %s group." % (p, searchtype)) ptype = "%s_group" % searchtype parent = parent[parent.nodetype == ptype] parent = get_shortest(parent)
class Analytic(object): """ Analytic class """ def __init__(self, conn, ems_id, data_file=LocalData.default_data_file, searchtype='contain'): """ Analytic class initialization Parameters ---------- conn: emspy.connection.Connection connection object ems_id: int EMS system id data_file: str path to local DB file searchtype: str search type to perform: contain: uses the Pandas string method 'contains' match: uses the Pandas string method 'match' """ self._conn = conn self._ems_id = ems_id self._metadata = None self._load_paramtable(data_file) self.searchtype = searchtype def _load_paramtable(self, file_name=LocalData.default_data_file): # Load parameter table if self._metadata is None: self._metadata = LocalData(file_name) elif not self._metadata.is_db_path_correct(file_name): self._metadata.close() self._metadata = LocalData(file_name) self._param_table = self._metadata.get_data( "params", "ems_id = %d" % self._ems_id) def _save_paramtable(self): # Save parameter table if len(self._param_table) > 0: self._metadata.delete_data("params", "ems_id = %d" % self._ems_id) self._metadata.append_data("params", self._param_table) def get_param_details(self, analytic_id): """ Method to get parameter details from the API Parameters ---------- analytic_id: str unique analytic id Returns ------- content: list response from API """ _, content = self._conn.request(rtype="POST", uri_keys=('analytic', 'search'), uri_args=self._ems_id, jsondata={'id': analytic_id}) return content def search_param(self, keyword, in_df=False): """ Method to search a parameter in EMS and return the following fields: - id: EMS id - name: parameter name - description: parameter description - units: parameter units - ems_id: ems system id Parameters ---------- keyword: str parameter to search in_df: bool selector for output type, if True returns a dataframe, if False returns a list Returns ------- pd.DataFrame or list parameter information (id, name, description, units, ems_id) """ print('Searching for params with keyword "%s" from EMS ...' % keyword, end=' ') # EMS API Call _, content = self._conn.request(uri_keys=('analytic', 'search'), uri_args=self._ems_id, body={'text': keyword}) if len(content) == 0: sys.exit("No parameter found with search keyword %s." % keyword) elif len(content) == 1: res = content else: word_len = [len(x['name']) for x in content] idx = np.argsort(word_len).tolist() res = [content[i] for i in idx] print("done.") for i in range(len(res)): res[i]['ems_id'] = self._ems_id if in_df: return pd.DataFrame(res) return res def get_param(self, keyword, unique=True): """ Get parameter information Parameters ---------- keyword: str parameter to fetch unique: bool indicates whether to return the parameter with the shortest name (if True) Returns ------- dict fetched parameter information (uri_root, ems_id, id, name, description, units) """ # if the param table is empty, just return an empty param dict. if self._param_table.empty: return dict(ems_id="", id="", name="", description="", units="") # If the param table is not empty, do search by keyword if self.searchtype == 'contain': bool_idx = self._param_table['name'].str.contains(keyword, case=False, regex=False) elif self.searchtype == 'match': bool_idx = self._param_table['name'].str.match(keyword, case=False) df = self._param_table[bool_idx] # If the search result is empty, return empty param dict if df.empty: return dict(ems_id="", id="", name="", description="", units="") # If not empty, return the one with shortest name if df.shape[0] > 1: idx = df['name'].map(lambda x: len(x)).sort_values().index df = df.loc[idx, :] # When unique = True if unique: return df.iloc[0, :].to_dict() # When unique = False return df.to_dict('records')
class Flight(object): temp_exclude = [ 'Download Information', 'Download Review', 'Processing', 'Profile 16 Extra Data', 'Flight Review', 'Data Information', 'Operational Information', 'Operational Information (ODW2)', 'Weather Information', 'Profiles', 'Profile' ] def __init__(self, conn, ems_id, data_file=LocalData.default_data_file, searchtype='contain'): """ Flight object initialization Parameters ---------- conn: emspy.connection.Connection connection object ems_id: int EMS system id data_file: str path to database file searchtype: str search type to perform: contain: uses the Pandas string method 'contains' match: uses the Pandas string method 'match' """ self.searchtype = searchtype self._conn = conn self._ems_id = ems_id self._db_id = None self._metadata = None self._trees = {'fieldtree': None, 'dbtree': None, 'kvmaps': None} self._fields = [] self.__cntr = 0 self._uri_root = conn._uri_root # Retrieve the field tree data from local storage. If it doesn't exist, generate a new # default one. self.load_tree(data_file) # Set default database # self.set_database('fdw flight') def load_tree(self, file_name=LocalData.default_data_file): """ Load trees into the object Parameters ---------- file_name: str path to database file Returns ------- None """ if self._metadata is None: self._metadata = LocalData(file_name) elif not self._metadata.is_db_path_correct(file_name): self._metadata.close() self._metadata = LocalData(file_name) self._trees = { 'fieldtree': self.__get_fieldtree(), 'dbtree': self.__get_dbtree(), 'kvmaps': self.__get_kvmaps() } def save_tree(self, file_name=LocalData.default_data_file): """ Method to save trees to a defined database file Parameters ---------- file_name: str path to database file Returns ------- None """ from shutil import copyfile ld = self._metadata if not ld.is_db_path_correct(file_name): # A different location is given, if a previous file exists copy it to the new location. if ld.file_loc() is not None and file_name is not None: copyfile(ld.file_loc(), os.path.abspath(file_name)) self._metadata.close() self._metadata = LocalData(file_name) self.__save_fieldtree() self.__save_dbtree() self.__save_kvmaps() def __get_fieldtree(self): if self._db_id is None: return pd.DataFrame(columns=self._metadata.table_info['fieldtree']) else: return self._metadata.get_data( "fieldtree", "ems_id = %d and db_id = '%s'" % (self._ems_id, self._db_id) ) def __save_fieldtree(self): if len(self._trees['fieldtree']) > 0: self._metadata.delete_data( "fieldtree", "ems_id = %d and db_id = '%s'" % (self._ems_id, self._db_id) ) self._metadata.append_data("fieldtree", self._trees['fieldtree']) def __get_dbtree(self): tree = self._metadata.get_data("dbtree", self.__get_filter('dbtree')) if len(tree) < 1: dbroot = { 'uri_root': self._uri_root, 'ems_id': self._ems_id, 'id': "[-hub-][entity-type-group][[--][internal-type-group][root]]", 'name': "<root>", 'nodetype': "root", 'parent_id': None } self._trees['dbtree'] = pd.DataFrame([dbroot]) self.__update_children(dbroot, treetype="dbtree") self.update_tree("fdw", treetype="dbtree", exclude_tree=["APM Events"]) self.__save_dbtree() tree = self._trees['dbtree'] return tree def __save_dbtree(self): if len(self._trees['dbtree']) > 0: self._metadata.delete_data("dbtree", self.__get_filter('dbtree')) self._metadata.append_data("dbtree", self._trees['dbtree']) def __get_kvmaps(self): tree = self._metadata.get_data("kvmaps", self.__get_filter('kvmaps')) return tree def __save_kvmaps(self): if len(self._trees['kvmaps']) > 0: self._metadata.delete_data("kvmaps", self.__get_filter('kvmaps')) self._metadata.append_data("kvmaps", self._trees['kvmaps']) def set_database(self, name): """ Method to select a database from the available databases in dbtree Parameters ---------- name: str database name Returns ------- None """ tree = self._trees['dbtree'] if self.searchtype == 'contain': dbs = tree[(tree.nodetype == 'database') & tree.name.str.contains(_treat_spchar(name), case=False)] # Return database with shortest name self._db_id = dbs.loc[dbs['name'].str.len().idxmin(), 'id']
class Analytic: def __init__(self, conn, ems_id, data_file=None): self._conn = conn self._ems_id = ems_id self._metadata = None self._load_paramtable(data_file) def _load_paramtable(self, file_name=None): if self._metadata is None: self._metadata = LocalData(file_name) else: if (file_name is None) and (self._metadata.file_loc() != os.path.abspath(file_name)): self._metadata.close() self._metadata = LocalData(file_name) self._param_table = self._metadata.get_data( "params", "ems_id = %d" % self._ems_id) def _save_paramtable(self): if len(self._param_table) > 0: self._metadata.delete_data("params", "ems_id = %d" % self._ems_id) self._metadata.append_data("params", self._param_table) def search_param(self, keyword, in_df=False): print 'Searching for params with keyword "%s" from EMS ...' % keyword, # EMS API Call resp_h, content = self._conn.request(uri_keys=('analytic', 'search'), uri_args=self._ems_id, body={'text': keyword}) if len(content) == 0: sys.exit("No parameter found with search keyword %s." % keyword) elif len(content) == 1: res = content else: word_len = [len(x['name']) for x in content] idx = np.argsort(word_len).tolist() res = [content[i] for i in idx] print "done." for i in range(len(res)): res[i]['ems_id'] = self._ems_id if in_df: return pd.DataFrame(res) return res def get_param(self, keyword, unique=True): # if the param table is empty, just return an empty param dict. if self._param_table.empty: return dict(ems_id="", id="", name="", description="", units="") # If the param table is not empty, do search by keyword bool_idx = self._param_table['name'].str.contains(keyword, case=False, regex=False) df = self._param_table[bool_idx] # If the search result is empty, return empty param dict if df.empty: return dict(ems_id="", id="", name="", description="", units="") # If not empty, return the one with shortest name if df.shape[0] > 1: idx = df['name'].map(lambda x: len(x)).sort_values().index df = df.loc[idx, :] # When unique = True if unique: return df.iloc[0, :].to_dict() # When unique = False return df.to_dict('records')