def add_row_to_history(self, run_id, resources, status, run_name = None):
"""update the run history table to indicate changes to the state of this run history trail."""
self.update_environment_variables(run_resources = resources)
resources['run_id'] = run_id
pickled_resources = pickle.dumps(resources)
description = resources.get('description', 'No description')
if run_name is None:
run_name = description
values = {'run_id':run_id,
'run_name': run_name,
'run_description':'%s' % description,
'status':'%s' % status,
'processor_name':'%s' % get_host_name(),
'date_time':datetime.datetime.now(),
'resources':'%s' % pickled_resources,
'cache_directory': resources['cache_directory'],
'project_name': resources.get('project_name', None)
}
run_activity_table = self.services_db.get_table('run_activity')
if not 'project_name' in run_activity_table.c:
del values['project_name']
if self.has_run(run_id):
qry = run_activity_table.update(values = values,
whereclause = run_activity_table.c.run_id == run_id)
else:
qry = run_activity_table.insert(values = values)
self.services_db.execute(qry)
def clean_runs(self):
# TODO: refactor this into a call of self.get_runs() and remove get_run_info
for run_id, _, _, processor_name, resources in self.get_run_info(
resources=True):
if processor_name == get_host_name() and not os.path.exists(
resources['cache_directory']):
self.delete_everything_for_this_run(
run_id, cache_directory=resources['cache_directory'])
def add_computed_indicator(self, indicator_name, dataset_name, expression, run_id, data_path, project_name):
"""update the run history table to indicate changes to the state of this run history trail."""
values = {
'run_id':run_id,
'indicator_name':str(indicator_name),
'dataset_name':dataset_name,
'expression':expression,
'data_path':data_path,
'processor_name':get_host_name(),
'date_time':datetime.datetime.now(),
'project_name': project_name
}
computed_indicators_table = self.services_db.get_table('computed_indicators')
if not 'project_name' in computed_indicators_table.c:
del values['project_name']
qry = computed_indicators_table.insert(values = values)
self.services_db.execute(qry)
def add_computed_indicator(self, indicator_name, dataset_name, expression,
run_id, data_path, project_name):
"""update the run history table to indicate changes to the state of this run history trail."""
values = {
'run_id': run_id,
'indicator_name': str(indicator_name),
'dataset_name': dataset_name,
'expression': expression,
'data_path': data_path,
'processor_name': get_host_name(),
'date_time': datetime.datetime.now(),
'project_name': project_name
}
computed_indicators_table = self.services_db.get_table(
'computed_indicators')
if not 'project_name' in computed_indicators_table.c:
del values['project_name']
qry = computed_indicators_table.insert(values=values)
self.services_db.execute(qry)
def get_results(self):
host = get_host_name()
tbl = self.services_db.get_table('computed_indicators')
s = select([
tbl.c.run_id, tbl.c.indicator_name, tbl.c.dataset_name,
tbl.c.expression, tbl.c.data_path, tbl.c.date_time
],
whereclause=tbl.c.processor_name == host)
results = []
for run_id, indicator_name, dataset_name, expression, data_path, date_time \
in self.services_db.execute(s).fetchall():
res = {
'run_id': run_id,
'indicator_name': indicator_name,
'dataset_name': dataset_name,
'expression': expression,
'data_path': data_path,
'date_time': date_time,
}
results.append(res)
return results
def add_row_to_history(self, run_id, resources, status, run_name=None, scenario_name=""):
"""update the run history table to indicate changes to the state of this run history trail."""
self.update_environment_variables(run_resources=resources)
resources["run_id"] = run_id
if self.server_config.blob_compression:
pickled_resources = zlib.compress(pickle.dumps(resources), 9)
else:
pickled_resources = pickle.dumps(resources)
description = resources.get("description", "No description")
if run_name is None:
run_name = description
values = {
"run_id": run_id,
"run_name": run_name,
"run_description": "%s" % description,
"status": "%s" % status,
"processor_name": "%s" % get_host_name(),
"date_time": datetime.datetime.now(),
"resources": "%s" % pickled_resources,
"cache_directory": resources["cache_directory"],
"project_name": resources.get("project_name", None),
"scenario_name": scenario_name,
}
run_activity_table = self.services_db.get_table("run_activity")
if not "project_name" in run_activity_table.c:
del values["project_name"]
if (not "scenario_name" in run_activity_table.c) or not scenario_name:
del values["scenario_name"]
if self.has_run(run_id=run_id):
qry = run_activity_table.update(values=values, whereclause=run_activity_table.c.run_id == run_id)
else:
qry = run_activity_table.insert(values=values)
self.services_db.execute(qry)
def get_results(self):
host = get_host_name()
tbl = self.services_db.get_table('computed_indicators')
s = select([tbl.c.run_id,
tbl.c.indicator_name,
tbl.c.dataset_name,
tbl.c.expression,
tbl.c.data_path,
tbl.c.date_time],
whereclause = tbl.c.processor_name == host)
results = []
for run_id, indicator_name, dataset_name, expression, data_path, date_time \
in self.services_db.execute(s).fetchall():
res = {
'run_id':run_id,
'indicator_name':indicator_name,
'dataset_name':dataset_name,
'expression':expression,
'data_path':data_path,
'date_time':date_time,
}
results.append(res)
return results
def _sync_base_year_data(self, run_manager=None):
"""
synchronize base_year_data information in xml_configuration with
run_activity table. Information in xml_configuration takes
precedent, because we assume users don't directly modify data in
serivecs.run_activity table
"""
# TODO baseyear_dir is somewhat hard-coded; it would be better to read
# from xml_configuration instead, but there is no such node currently
run_name = 'base_year_data'
baseyear_dir = os.path.join(self.project.data_path(), run_name)
baseyear_dir = os.path.normpath(baseyear_dir)
if not os.path.exists(baseyear_dir):
MessageBox.warning(mainwindow = self.base_widget,
text="base_year_data directory %s doesn't exist. " % baseyear_dir
)
return
import glob
years = [int(os.path.basename(year_dir)) for year_dir in
glob.glob(os.path.join(baseyear_dir, '[0-9][0-9][0-9][0-9]'))]
if not years:
MessageBox.warning(mainwindow = self.base_widget,
text="base_year_data directory %s doesn't contain any year sub-directory. " % baseyear_dir
)
return
start_year = min(years)
end_year = max(years)
base_year = end_year # default to the last year in baseyear_dir
# and update it with information found in scenario_manager
scenario_manager_node = self.project.find('scenario_manager')
for base_year_node in scenario_manager_node.findall('.//base_year'):
try:
base_year = int(base_year_node.text.strip())
break
except (TypeError, ValueError):
continue
resources = {
'cache_directory': baseyear_dir,
'description': 'base year data',
'base_year': base_year,
'years': (start_year, end_year)
}
if run_manager is None: run_manager = get_run_manager()
base_year_data_db = run_manager.get_runs(run_name=run_name,
process_name=get_host_name())
if len(base_year_data_db) == 0:
run_id = run_manager._get_new_run_id()
elif len(base_year_data_db) >= 1:
for idx, row in enumerate(base_year_data_db):
if idx==0:
run_id = row[0]
else:
run_manager.delete_everything_for_this_run(row[0])
resources_db = run_manager.get_resources_for_run_id_from_history(run_id)
if resources_db.get('cache_directory', '') == baseyear_dir and \
resources_db.get('base_year', -1) == base_year:
#all good, we don't need to do anything
return
else:
resources_db.merge(resources)
resources = resources_db
run_manager.add_row_to_history(run_id = run_id,
resources = resources,
status = 'done',
run_name = run_name)
def is_localhost(self, hostname):
if (hostname == 'localhost') or (hostname == get_host_name()) or \
(hostname == socket.gethostname()):
return True
return False
def clean_runs(self):
# TODO: refactor this into a call of self.get_runs() and remove get_run_info
for run_id, _, _, processor_name, resources in self.get_run_info(resources = True):
if processor_name == get_host_name() and not os.path.exists(resources['cache_directory']):
self.delete_everything_for_this_run(run_id, cache_directory = resources['cache_directory'])
def is_localhost(self):
if (self.hostname == 'localhost') or (self.hostname == get_host_name()) or \
(self.hostname == socket.gethostname()):
return True
return False
def clean_runs(self):
for run_id, run_name, run_description, processor_name, resources in self.get_run_info(resources = True):
if processor_name == get_host_name() and not os.path.exists(resources['cache_directory']):
self.delete_everything_for_this_run(run_id, cache_directory = resources['cache_directory'])
