def _load_paramtable(self, file_name=LocalData.default_data_file):
# Load parameter table
if self._metadata is None:
self._metadata = LocalData(file_name)
elif not self._metadata.is_db_path_correct(file_name):
self._metadata.close()
self._metadata = LocalData(file_name)
self._param_table = self._metadata.get_data(
"params", "ems_id = %d" % self._ems_id)
def _load_paramtable(self, file_name=None):
if self._metadata is None:
self._metadata = LocalData(file_name)
else:
if (file_name is None) and (self._metadata.file_loc() !=
os.path.abspath(file_name)):
self._metadata.close()
self._metadata = LocalData(file_name)
self._param_table = self._metadata.get_data(
"params", "ems_id = %d" % self._ems_id)
def save_tree(self, file_name=None):
from shutil import copyfile
ld = self._metadata
if (file_name
is not None) and (ld.file_loc() != os.path.abspath(file_name)):
# A new file location is given, copy all the data in the current file with new file name,
# and save the currently loaded tree data into the new file too.
copyfile(self._metadata.file_loc(), file_name)
self._metadata.close()
self._metadata = LocalData(file_name)
self.__save_fieldtree()
self.__save_dbtree()
self.__save_kvmaps()
def load_tree(self, file_name=None):
if self._metadata is None:
self._metadata = LocalData(file_name)
else:
if (file_name is not None) and (self._metadata.file_loc() !=
os.path.abspath(file_name)):
self._metadata.close()
self._metadata = LocalData(file_name)
self._trees = {
'fieldtree': self.__get_fieldtree(),
'dbtree': self.__get_dbtree(),
'kvmaps': self.__get_kvmaps()
}
def save_tree(self, file_name=LocalData.default_data_file):
"""
Method to save trees to a defined database file
Parameters
----------
file_name: str
path to database file
Returns
-------
None
"""
from shutil import copyfile
ld = self._metadata
if not ld.is_db_path_correct(file_name):
# A different location is given, if a previous file exists copy it to the new location.
if ld.file_loc() is not None and file_name is not None:
copyfile(ld.file_loc(), os.path.abspath(file_name))
self._metadata.close()
self._metadata = LocalData(file_name)
self.__save_fieldtree()
self.__save_dbtree()
self.__save_kvmaps()
def load_tree(self, file_name=LocalData.default_data_file):
"""
Load trees into the object
Parameters
----------
file_name: str
path to database file
Returns
-------
None
"""
if self._metadata is None:
self._metadata = LocalData(file_name)
elif not self._metadata.is_db_path_correct(file_name):
self._metadata.close()
self._metadata = LocalData(file_name)
self._trees = {
'fieldtree': self.__get_fieldtree(),
'dbtree': self.__get_dbtree(),
'kvmaps': self.__get_kvmaps()
}
class Flight:
temp_exclude = [
'Download Information', 'Download Review', 'Processing',
'Profile 16 Extra Data', 'Flight Review', 'Data Information',
'Operational Information', 'Operational Information (ODW2)',
'Weather Information', 'Profiles', 'Profile'
]
def __init__(self, conn, ems_id, data_file=None):
self._conn = conn
self._ems_id = ems_id
self._db_id = None
self._metadata = None
self._trees = {'fieldtree': None, 'dbtree': None, 'kvmaps': None}
self._fields = []
self.__cntr = 0
# Retreive the field tree data from local storage. If it doesn't exist, generate a new
# default one.
self.load_tree(data_file)
# Set default database
# self.set_database('fdw flight')
def load_tree(self, file_name=None):
if self._metadata is None:
self._metadata = LocalData(file_name)
else:
if (file_name is not None) and (self._metadata.file_loc() !=
os.path.abspath(file_name)):
self._metadata.close()
self._metadata = LocalData(file_name)
self._trees = {
'fieldtree': self.__get_fieldtree(),
'dbtree': self.__get_dbtree(),
'kvmaps': self.__get_kvmaps()
}
def save_tree(self, file_name=None):
from shutil import copyfile
ld = self._metadata
if (file_name
is not None) and (ld.file_loc() != os.path.abspath(file_name)):
# A new file location is given, copy all the data in the current file with new file name,
# and save the currently loaded tree data into the new file too.
copyfile(self._metadata.file_loc(), file_name)
self._metadata.close()
self._metadata = LocalData(file_name)
self.__save_fieldtree()
self.__save_dbtree()
self.__save_kvmaps()
def __get_fieldtree(self):
if self._db_id is None:
return pd.DataFrame(columns=self._metadata.table_info['fieldtree'])
else:
return self._metadata.get_data(
"fieldtree",
"ems_id = %d and db_id = '%s'" % (self._ems_id, self._db_id))
def __save_fieldtree(self):
if len(self._trees['fieldtree']) > 0:
self._metadata.delete_data(
"fieldtree",
"ems_id = %d and db_id = '%s'" % (self._ems_id, self._db_id))
self._metadata.append_data("fieldtree", self._trees['fieldtree'])
def __get_dbtree(self):
T = self._metadata.get_data("dbtree", "ems_id = %d" % self._ems_id)
if len(T) < 1:
dbroot = {
'ems_id': self._ems_id,
'id':
"[-hub-][entity-type-group][[--][internal-type-group][root]]",
'name': "<root>",
'nodetype': "root",
'parent_id': None
}
self._trees['dbtree'] = pd.DataFrame([dbroot])
self.__update_children(dbroot, treetype="dbtree")
self.update_tree("fdw",
treetype="dbtree",
exclude_tree=["APM Events"])
self.__save_dbtree()
T = self._trees['dbtree']
return T
def __save_dbtree(self):
if len(self._trees['dbtree']) > 0:
self._metadata.delete_data("dbtree", "ems_id = %d" % self._ems_id)
self._metadata.append_data("dbtree", self._trees['dbtree'])
def __get_kvmaps(self):
T = self._metadata.get_data("kvmaps", "ems_id = %d" % self._ems_id)
return T
def __save_kvmaps(self):
if len(self._trees['kvmaps']) > 0:
self._metadata.delete_data("kvmaps", "ems_id = %d" % self._ems_id)
self._metadata.append_data("kvmaps", self._trees['kvmaps'])
def set_database(self, name):
tr = self._trees['dbtree']
self._db_id = tr[(tr.nodetype == 'database') & tr.name.str.contains(
treat_spchar(name), case=False)]['id'].values[0]
self._trees['fieldtree'] = self.__get_fieldtree()
if self._trees['fieldtree'].empty:
self.__update_children(self.get_database(), treetype="fieldtree")
print("Using database '%s'." % self.get_database()['name'])
def get_database(self):
tr = self._trees['dbtree']
return tr[(tr.nodetype == "database")
& (tr.id == self._db_id)].iloc[0].to_dict()
def __db_request(self, parent):
body = None
if parent['nodetype'] == "database_group":
body = {'groupId': parent['id']}
resp_h, d = self._conn.request(uri_keys=('database', 'group'),
uri_args=self._ems_id,
body=body)
d1 = []
if len(d['databases']) > 0:
d1 = map(
lambda x: {
'ems_id': parent['ems_id'],
'id': x['id'],
'nodetype': 'database',
'name': x['pluralName'],
'parent_id': parent['id']
}, d['databases'])
d2 = []
if len(d['groups']) > 0:
d2 = map(
lambda x: {
'ems_id': parent['ems_id'],
'id': x['id'],
'nodetype': 'database_group',
'name': x['name'],
'parent_id': parent['id']
}, d['groups'])
return d1, d2
def __fl_request(self, parent):
body = None
if parent['nodetype'] == "field_group":
body = {'groupId': parent['id']}
resp_h, d = self._conn.request(uri_keys=('database', 'field_group'),
uri_args=(self._ems_id, self._db_id),
body=body)
d1 = []
if len(d['fields']) > 0:
d1 = map(
lambda x: {
'ems_id': parent['ems_id'],
'db_id': self._db_id,
'id': x['id'],
'nodetype': 'field',
'type': x['type'],
'name': x['name'],
'parent_id': parent['id']
}, d['fields'])
d2 = []
if len(d['groups']) > 0:
d2 = map(
lambda x: {
'ems_id': parent['ems_id'],
'db_id': self._db_id,
'id': x['id'],
'nodetype': 'field_group',
'type': None,
'name': x['name'],
'parent_id': parent['id']
}, d['groups'])
return d1, d2
def __add_subtree(self, parent, exclude_tree=[], treetype='fieldtree'):
print("On " + parent['name'] + "(" + parent['nodetype'] + ")" + "...")
if treetype == "dbtree":
searchtype = 'database'
d1, d2 = self.__db_request(parent)
else:
searchtype = "field"
d1, d2 = self.__fl_request(parent)
if len(d1) > 0:
self._trees[treetype] = self._trees[treetype].append(
d1, ignore_index=True)
plural = "s" if len(d1) > 1 else ""
print("-- Added %d %s%s" % (len(d1), searchtype, plural))
for x in d2:
self._trees[treetype] = self._trees[treetype].append(
x, ignore_index=True)
if len(exclude_tree) > 0:
if all([y not in x['name'] for y in exclude_tree]):
self.__add_subtree(x, exclude_tree, treetype)
else:
self.__add_subtree(x, exclude_tree, treetype)
def __get_children(self, parent_id, treetype='fieldtree'):
tr = self._trees[treetype]
# tr = tr[tr.nodetype != ('field' if treetype=='fieldtree' else 'dbtree')]
if isinstance(parent_id, (list, tuple, pd.Series)):
return tr[tr.parent_id.isin(parent_id)]
return tr[tr.parent_id == parent_id]
def __remove_subtree(self, parent, treetype='fieldtree'):
tr = self._trees[treetype]
chld = tr[tr.parent_id == parent['id']]
# Update the instance tree by deleting children
self._trees[treetype] = tr[tr.parent_id != parent['id']]
# Iterate and do recursive removal of children of children
leaftype = 'field' if treetype == 'fieldtree' else 'database'
for i, x in chld[chld.nodetype != leaftype].iterrows():
self.__remove_subtree(x, treetype=treetype)
# def __remove_subtree(self, parent, rm_parent=True, treetype='fieldtree'):
# rm_list = list()
# if rm_parent:
# rm_list.append(parent['id'])
# parent_id = [parent['id']]
# cntr = 0
# while len(parent_id) > 0:
# child_id = self.__get_children(parent_id, treetype=treetype)['id'].tolist()
# rm_list += child_id
# parent_id = child_id
# cntr += 1
# if cntr > 1e4:
# sys.exit("Something's wrong. Subtree removal went over 10,000 iterations.")
# if len(rm_list) > 0:
# tr = self._trees[treetype]
# self._trees[treetype] = tr[~tr.id.isin(rm_list)]
# print("Removed the subtree of %s (%s) with total of %d nodes (fields/databases/groups)." % (
# parent['name'], parent['nodetype'], len(rm_list)))
def __update_children(self, parent, treetype='fieldtree'):
'''
This function updates the direct children of a parent node.
'''
print("On " + parent['name'] + "(" + parent['nodetype'] + ")" + "...")
if treetype == "dbtree":
searchtype = 'database'
d1, d2 = self.__db_request(parent)
else:
searchtype = "field"
d1, d2 = self.__fl_request(parent)
T = self._trees[treetype]
self._trees[treetype] = T[~((T.nodetype == searchtype) &
(T.parent_id == parent['id']))]
if len(d1) > 0:
self._trees[treetype] = self._trees[treetype].append(
d1, ignore_index=True)
plural = "s" if len(d1) > 1 else ""
print("-- Added %d %s%s" % (len(d1), searchtype, plural))
# If there is an array of groups as children add any that appeared new and remove who does not.
old_groups = T[(T.nodetype == '%s_group' % searchtype)
& (T.parent_id == parent['id'])]
old_ones = old_groups["id"].tolist()
new_ones = [x['id'] for x in d2]
rm_id = listdiff(old_ones, new_ones)
if len(rm_id) > 0:
[
self.__remove_subtree(x.to_dict(), treetype=treetype)
for i, x in old_groups.iterrows() if x['id'] in rm_id
]
add_id = listdiff(new_ones, old_ones)
if len(add_id) > 0:
self._trees[treetype] = self._trees[treetype].append(
[x for x in d2 if x['id'] in add_id])
def update_tree(self, *args, **kwargs):
'''
Optional arguments
------------------
treetype :
"fieldtree" or "dbtree"
exclude_tree:
Exact name strings (case sensitive) of the field groups you don't want to search through
Ex. ['Profiles', 'Weather Information']
'''
treetype = kwargs.get("treetype", "fieldtree")
exclude_tree = kwargs.get("exclude_tree", [])
searchtype = "field" if treetype == "fieldtree" else "database"
if treetype not in ("fieldtree", "dbtree"):
raise ValueError("treetype = '%s': there is no such data table." %
treetype)
fld_path = [s.lower() for s in args]
for i, p in enumerate(fld_path):
p = treat_spchar(p)
if i == 0:
T = self._trees[treetype]
parent = T[T.name.str.contains(p, case=False)]
else:
self.__update_children(parent, treetype=treetype)
chld_df = self.__get_children(parent['id'], treetype=treetype)
child = chld_df[chld_df.name.str.contains(p, case=False)]
parent = child
if len(parent) == 0:
raise ValueError(
"Search keyword '%s' did not return any %s group." %
(p, searchtype))
ptype = "%s_group" % searchtype
parent = parent[parent.nodetype == ptype]
parent = get_shortest(parent)
class Analytic(object):
"""
Analytic class
"""
def __init__(self,
conn,
ems_id,
data_file=LocalData.default_data_file,
searchtype='contain'):
"""
Analytic class initialization
Parameters
----------
conn: emspy.connection.Connection
connection object
ems_id: int
EMS system id
data_file: str
path to local DB file
searchtype: str
search type to perform:
contain: uses the Pandas string method 'contains'
match: uses the Pandas string method 'match'
"""
self._conn = conn
self._ems_id = ems_id
self._metadata = None
self._load_paramtable(data_file)
self.searchtype = searchtype
def _load_paramtable(self, file_name=LocalData.default_data_file):
# Load parameter table
if self._metadata is None:
self._metadata = LocalData(file_name)
elif not self._metadata.is_db_path_correct(file_name):
self._metadata.close()
self._metadata = LocalData(file_name)
self._param_table = self._metadata.get_data(
"params", "ems_id = %d" % self._ems_id)
def _save_paramtable(self):
# Save parameter table
if len(self._param_table) > 0:
self._metadata.delete_data("params", "ems_id = %d" % self._ems_id)
self._metadata.append_data("params", self._param_table)
def get_param_details(self, analytic_id):
"""
Method to get parameter details from the API
Parameters
----------
analytic_id: str
unique analytic id
Returns
-------
content: list
response from API
"""
_, content = self._conn.request(rtype="POST",
uri_keys=('analytic', 'search'),
uri_args=self._ems_id,
jsondata={'id': analytic_id})
return content
def search_param(self, keyword, in_df=False):
"""
Method to search a parameter in EMS and return the following fields:
- id: EMS id
- name: parameter name
- description: parameter description
- units: parameter units
- ems_id: ems system id
Parameters
----------
keyword: str
parameter to search
in_df: bool
selector for output type, if True returns a dataframe,
if False returns a list
Returns
-------
pd.DataFrame or list
parameter information (id, name, description, units, ems_id)
"""
print('Searching for params with keyword "%s" from EMS ...' % keyword,
end=' ')
# EMS API Call
_, content = self._conn.request(uri_keys=('analytic', 'search'),
uri_args=self._ems_id,
body={'text': keyword})
if len(content) == 0:
sys.exit("No parameter found with search keyword %s." % keyword)
elif len(content) == 1:
res = content
else:
word_len = [len(x['name']) for x in content]
idx = np.argsort(word_len).tolist()
res = [content[i] for i in idx]
print("done.")
for i in range(len(res)):
res[i]['ems_id'] = self._ems_id
if in_df:
return pd.DataFrame(res)
return res
def get_param(self, keyword, unique=True):
"""
Get parameter information
Parameters
----------
keyword: str
parameter to fetch
unique: bool
indicates whether to return the parameter with the shortest name (if True)
Returns
-------
dict
fetched parameter information (uri_root, ems_id, id, name, description, units)
"""
# if the param table is empty, just return an empty param dict.
if self._param_table.empty:
return dict(ems_id="", id="", name="", description="", units="")
# If the param table is not empty, do search by keyword
if self.searchtype == 'contain':
bool_idx = self._param_table['name'].str.contains(keyword,
case=False,
regex=False)
elif self.searchtype == 'match':
bool_idx = self._param_table['name'].str.match(keyword, case=False)
df = self._param_table[bool_idx]
# If the search result is empty, return empty param dict
if df.empty:
return dict(ems_id="", id="", name="", description="", units="")
# If not empty, return the one with shortest name
if df.shape[0] > 1:
idx = df['name'].map(lambda x: len(x)).sort_values().index
df = df.loc[idx, :]
# When unique = True
if unique:
return df.iloc[0, :].to_dict()
# When unique = False
return df.to_dict('records')
class Flight(object):
temp_exclude = [
'Download Information',
'Download Review',
'Processing',
'Profile 16 Extra Data',
'Flight Review',
'Data Information',
'Operational Information',
'Operational Information (ODW2)',
'Weather Information',
'Profiles',
'Profile'
]
def __init__(self, conn, ems_id, data_file=LocalData.default_data_file, searchtype='contain'):
"""
Flight object initialization
Parameters
----------
conn: emspy.connection.Connection
connection object
ems_id: int
EMS system id
data_file: str
path to database file
searchtype: str
search type to perform:
contain: uses the Pandas string method 'contains'
match: uses the Pandas string method 'match'
"""
self.searchtype = searchtype
self._conn = conn
self._ems_id = ems_id
self._db_id = None
self._metadata = None
self._trees = {'fieldtree': None, 'dbtree': None, 'kvmaps': None}
self._fields = []
self.__cntr = 0
self._uri_root = conn._uri_root
# Retrieve the field tree data from local storage. If it doesn't exist, generate a new
# default one.
self.load_tree(data_file)
# Set default database
# self.set_database('fdw flight')
def load_tree(self, file_name=LocalData.default_data_file):
"""
Load trees into the object
Parameters
----------
file_name: str
path to database file
Returns
-------
None
"""
if self._metadata is None:
self._metadata = LocalData(file_name)
elif not self._metadata.is_db_path_correct(file_name):
self._metadata.close()
self._metadata = LocalData(file_name)
self._trees = {
'fieldtree': self.__get_fieldtree(),
'dbtree': self.__get_dbtree(),
'kvmaps': self.__get_kvmaps()
}
def save_tree(self, file_name=LocalData.default_data_file):
"""
Method to save trees to a defined database file
Parameters
----------
file_name: str
path to database file
Returns
-------
None
"""
from shutil import copyfile
ld = self._metadata
if not ld.is_db_path_correct(file_name):
# A different location is given, if a previous file exists copy it to the new location.
if ld.file_loc() is not None and file_name is not None:
copyfile(ld.file_loc(), os.path.abspath(file_name))
self._metadata.close()
self._metadata = LocalData(file_name)
self.__save_fieldtree()
self.__save_dbtree()
self.__save_kvmaps()
def __get_fieldtree(self):
if self._db_id is None:
return pd.DataFrame(columns=self._metadata.table_info['fieldtree'])
else:
return self._metadata.get_data(
"fieldtree",
"ems_id = %d and db_id = '%s'" % (self._ems_id, self._db_id)
)
def __save_fieldtree(self):
if len(self._trees['fieldtree']) > 0:
self._metadata.delete_data(
"fieldtree",
"ems_id = %d and db_id = '%s'" % (self._ems_id, self._db_id)
)
self._metadata.append_data("fieldtree", self._trees['fieldtree'])
def __get_dbtree(self):
tree = self._metadata.get_data("dbtree", self.__get_filter('dbtree'))
if len(tree) < 1:
dbroot = {
'uri_root': self._uri_root,
'ems_id': self._ems_id,
'id': "[-hub-][entity-type-group][[--][internal-type-group][root]]",
'name': "<root>",
'nodetype': "root",
'parent_id': None
}
self._trees['dbtree'] = pd.DataFrame([dbroot])
self.__update_children(dbroot, treetype="dbtree")
self.update_tree("fdw", treetype="dbtree", exclude_tree=["APM Events"])
self.__save_dbtree()
tree = self._trees['dbtree']
return tree
def __save_dbtree(self):
if len(self._trees['dbtree']) > 0:
self._metadata.delete_data("dbtree", self.__get_filter('dbtree'))
self._metadata.append_data("dbtree", self._trees['dbtree'])
def __get_kvmaps(self):
tree = self._metadata.get_data("kvmaps", self.__get_filter('kvmaps'))
return tree
def __save_kvmaps(self):
if len(self._trees['kvmaps']) > 0:
self._metadata.delete_data("kvmaps", self.__get_filter('kvmaps'))
self._metadata.append_data("kvmaps", self._trees['kvmaps'])
def set_database(self, name):
"""
Method to select a database from the available databases in dbtree
Parameters
----------
name: str
database name
Returns
-------
None
"""
tree = self._trees['dbtree']
if self.searchtype == 'contain':
dbs = tree[(tree.nodetype == 'database')
& tree.name.str.contains(_treat_spchar(name), case=False)]
# Return database with shortest name
self._db_id = dbs.loc[dbs['name'].str.len().idxmin(), 'id']
class Analytic:
def __init__(self, conn, ems_id, data_file=None):
self._conn = conn
self._ems_id = ems_id
self._metadata = None
self._load_paramtable(data_file)
def _load_paramtable(self, file_name=None):
if self._metadata is None:
self._metadata = LocalData(file_name)
else:
if (file_name is None) and (self._metadata.file_loc() !=
os.path.abspath(file_name)):
self._metadata.close()
self._metadata = LocalData(file_name)
self._param_table = self._metadata.get_data(
"params", "ems_id = %d" % self._ems_id)
def _save_paramtable(self):
if len(self._param_table) > 0:
self._metadata.delete_data("params", "ems_id = %d" % self._ems_id)
self._metadata.append_data("params", self._param_table)
def search_param(self, keyword, in_df=False):
print 'Searching for params with keyword "%s" from EMS ...' % keyword,
# EMS API Call
resp_h, content = self._conn.request(uri_keys=('analytic', 'search'),
uri_args=self._ems_id,
body={'text': keyword})
if len(content) == 0:
sys.exit("No parameter found with search keyword %s." % keyword)
elif len(content) == 1:
res = content
else:
word_len = [len(x['name']) for x in content]
idx = np.argsort(word_len).tolist()
res = [content[i] for i in idx]
print "done."
for i in range(len(res)):
res[i]['ems_id'] = self._ems_id
if in_df:
return pd.DataFrame(res)
return res
def get_param(self, keyword, unique=True):
# if the param table is empty, just return an empty param dict.
if self._param_table.empty:
return dict(ems_id="", id="", name="", description="", units="")
# If the param table is not empty, do search by keyword
bool_idx = self._param_table['name'].str.contains(keyword,
case=False,
regex=False)
df = self._param_table[bool_idx]
# If the search result is empty, return empty param dict
if df.empty:
return dict(ems_id="", id="", name="", description="", units="")
# If not empty, return the one with shortest name
if df.shape[0] > 1:
idx = df['name'].map(lambda x: len(x)).sort_values().index
df = df.loc[idx, :]
# When unique = True
if unique:
return df.iloc[0, :].to_dict()
# When unique = False
return df.to_dict('records')