def generate_summary_files(processrunid):
flamingo_db_utils.generate_oasis_files_outputs(processrunid)
prog_oasis_location = \
flamingo_db_utils.get_prog_oasis_location(processrunid)
ts = values.get_timestamp()
process_dir = "ProcessRun_" + str(processrunid) + "_" + ts
input_location = str(prog_oasis_location) + "/" + str(process_dir)
if not os.path.isdir(input_location):
os.mkdir(input_location)
for i in ("items", "coverages", "fm_programme", "fm_policytc", "fm_xref",
"fm_profile"):
source_file = "{}/{}.csv".format(prog_oasis_location, i)
target_file = "{}/{}.csv".format(input_location, i)
if not IS_WINDOWS_HOST:
os.symlink(source_file, target_file)
else:
shutil.copy(source_file, target_file)
db.bcp("OasisGULSUMMARYXREF", input_location + "/gulsummaryxref_temp.csv")
db.bcp("OasisFMSUMMARYXREF", input_location + "/fmsummaryxref_temp.csv")
gulsummaryxref = input_location + "/gulsummaryxref.csv"
destination = open(gulsummaryxref, 'wb')
destination.write("coverage_id,summary_id,summaryset_id\n")
shutil.copyfileobj(open(input_location + "/gulsummaryxref_temp.csv", 'rb'),
destination)
destination.close()
os.remove(input_location + "/gulsummaryxref_temp.csv")
fmsummaryxref = input_location + "/fmsummaryxref.csv"
destination = open(fmsummaryxref, 'wb')
destination.write("output_id,summary_id,summaryset_id\n")
shutil.copyfileobj(open(input_location + "/fmsummaryxref_temp.csv", 'rb'),
destination)
destination.close()
os.remove(input_location + "/fmsummaryxref_temp.csv")
process_run_locationid = flamingo_db_utils.get_process_run_locationid(
prog_oasis_location, process_dir, processrunid)
flamingo_db_utils.generate_oasis_files_records_outputs(
processrunid, process_run_locationid)
return input_location
def do_run_prog_oasis(processrunid):
'''
Run a programme model combination.
'''
element_run_ids = list()
element_run_id = -1
flamingo_db_utils.update_process_run_status(processrunid, "In Progress")
try:
base_url = flamingo_db_utils.get_base_url(processrunid)
element_run_ids = \
flamingo_db_utils.get_element_run_ids(processrunid)
upload_directory = generate_summary_files(processrunid)
logging.getLogger().debug(
"Upload_directory: {}".format(upload_directory))
analysis_settings_json = get_analysis_settings_json(processrunid)
logging.getLogger().debug(analysis_settings_json)
if 'il_output' in analysis_settings_json['analysis_settings']:
create_il_bins = analysis_settings_json['analysis_settings'][
'il_output']
else:
create_il_bins = False
analysis_poll_interval_in_seconds = 5
client = OasisAPIClient(base_url, logging.getLogger())
element_run_id = element_run_ids[0][0]
input_location = client.upload_inputs_from_directory(
upload_directory,
do_il=create_il_bins,
do_build=True,
do_clean=True)
logging.getLogger().info("Input location: {}".format(input_location))
flamingo_db_utils.log_element_run_to_db(
element_run_id, "Success",
"Exposure files location: {}".format(input_location))
element_run_id = element_run_ids[1][0]
analysis_status_location = client.run_analysis(analysis_settings_json,
input_location)
flamingo_db_utils.log_element_run_to_db(element_run_id, "Success",
"Started analysis")
element_run_id = element_run_ids[2][0]
outputs_location = ""
while True:
logging.getLogger().debug("Polling analysis status for: {}".format(
analysis_status_location))
(status, outputs_location) = \
client.get_analysis_status(analysis_status_location)
flamingo_db_utils.log_element_run_to_db(element_run_id, status,
"In Progress")
if status == status_code.STATUS_SUCCESS:
if outputs_location is None:
raise Exception("Complete but no outputs location")
flamingo_db_utils.log_element_run_to_db(
element_run_id, status, "Analysis Completed")
break
elif status == status_code.STATUS_FAILURE:
error_message = "Analysis failed: {}".format(message)
logging.getLogger().error(error_message)
raise Exception(error_message)
time.sleep(analysis_poll_interval_in_seconds)
element_run_id = element_run_ids[3][0]
# download outputs and cleanup
outputs_file = os.path.join(upload_directory,
outputs_location + ".tar.gz")
client.download_outputs(outputs_location, outputs_file)
client.delete_exposure(input_location)
client.delete_outputs(outputs_location)
flamingo_db_utils.log_element_run_to_db(
element_run_id, 'Success', 'Downloaded output files successfully')
extract_tarball(
os.path.join(upload_directory, outputs_location + ".tar.gz"),
upload_directory)
output_file_list = ','.join(
map(str, os.listdir(os.path.join(upload_directory, "output"))))
logging.getLogger().debug(
"Output_file_list: {}".format(output_file_list))
db.execute(
"exec dbo.linkOutputFileToProcessRun @ProcessRunId = ?, @OutputFiles = ?",
processrunid, output_file_list)
flamingo_db_utils.update_process_run_status(processrunid, 'Completed')
# append summary id meanings to output files
output_file_details = flamingo_db_utils.get_output_file_details(
processrunid)
logging.getLogger().debug(
"output_file_details: {}".format(output_file_details))
columns = [
"FileName", "FileDesc", "PerspectiveName", "OutputID", "LECFlag",
"AnalysisFileNameStub", "SummaryLevelName"
]
df_output_file_details = pd.DataFrame(columns=columns)
recs = map(lambda tup: dict(zip(columns, list(tup))),
output_file_details)
df_output_file_details = df_output_file_details.append(recs)
logging.getLogger().debug(
"df_output_file_details:\n{}".format(df_output_file_details))
prog_oasis_location = flamingo_db_utils.get_prog_oasis_location(
processrunid)
itemdict = prog_oasis_location + '/ItemDict.csv'
fmdict = prog_oasis_location + '/FMDict.csv'
df_itemdict = pd.read_csv(itemdict)
df_fmdict = pd.read_csv(fmdict)
df_fmdict["policy_layer"] = df_fmdict["policy_name"].map(
str) + '--' + df_fmdict["layer_name"].map(str)
for index, row in df_output_file_details.iterrows():
output = upload_directory + '/output/' + row['FileName']
logging.getLogger().debug("FileName: {}".format(output))
output_tmp = output + '.tmp'
SummaryLevelName = row['SummaryLevelName']
df_output = pd.read_csv(output)
SummaryLevelId = SummaryLevelName.lower() + '_id'
SummaryLevelDesc = SummaryLevelName.lower() + '_desc'
logging.getLogger().debug(
"SummaryLevelName: {}".format(SummaryLevelName))
if SummaryLevelName != "Portfolio":
if SummaryLevelName == "Policy":
# join fmdict to file
df_summarydict = df_fmdict[['agg_id', 'policy_layer']]
logging.getLogger().debug(
"df_summarydict: {}".format(df_summarydict))
df_summarydict_distinct = df_summarydict.drop_duplicates()
df_output_temp = df_output.join(
df_summarydict_distinct.set_index('agg_id'),
on='summary_id')
else:
# join itemdict to file
df_summarydict = df_itemdict[[
SummaryLevelId, SummaryLevelDesc
]]
df_summarydict_distinct = df_summarydict.drop_duplicates()
df_output_temp = df_output.join(
df_summarydict_distinct.set_index(SummaryLevelId),
on='summary_id')
logging.getLogger().debug(
"df_summarydict_distinct: {}".format(
df_summarydict_distinct))
df_output_temp.to_csv(output, encoding='utf-8', index=False)
except Exception as e:
flamingo_db_utils.update_process_run_status(processrunid, "Failed")
if element_run_id != -1:
flamingo_db_utils.log_element_run_to_db(element_run_id, 'Failed: ',
str(e))
logging.getLogger().exception(
"Failed to run prog oasis: {}".format(processrunid))
def generate_summary_files(processrunid):
flamingo_db_utils.generate_oasis_files_outputs(processrunid)
prog_oasis_location = \
flamingo_db_utils.get_prog_oasis_location(processrunid)
ts = values.get_timestamp()
process_dir = "ProcessRun_" + str(processrunid) + "_" + ts
input_location = str(prog_oasis_location) + "/" + str(process_dir)
if not os.path.isdir(input_location):
os.mkdir(input_location)
for i in ("items", "coverages", "fm_programme", "fm_policytc", "fm_xref",
"fm_profile"):
source_file = "{}/{}.csv".format(prog_oasis_location, i)
target_file = "{}/{}.csv".format(input_location, i)
if not IS_WINDOWS_HOST:
os.symlink(source_file, target_file)
else:
shutil.copy(source_file, target_file)
input_dir_list = os.listdir(prog_oasis_location)
ri_dir_list = []
for dirs in input_dir_list:
if dirs.startswith('RI_'):
ri_dir_list.append(dirs)
for dirs in ri_dir_list:
ri_full_path = prog_oasis_location + '/' + dirs
ri_target_path = input_location + '/' + dirs
if not IS_WINDOWS_HOST:
os.symlink(ri_full_path, ri_target_path)
else:
shutil.copytree(ri_full_path, ri_target_path)
db.bcp("OasisGULSUMMARYXREF", input_location + "/gulsummaryxref_temp.csv")
db.bcp("OasisFMSUMMARYXREF", input_location + "/fmsummaryxref_temp.csv")
db.bcp("OasisRISUMMARYXREF", input_location + "/risummaryxref_temp.csv")
gulsummaryxref = input_location + "/gulsummaryxref.csv"
destination = open(gulsummaryxref, 'wb')
destination.write("coverage_id,summary_id,summaryset_id\n")
shutil.copyfileobj(open(input_location + "/gulsummaryxref_temp.csv", 'rb'),
destination)
destination.close()
os.remove(input_location + "/gulsummaryxref_temp.csv")
fmsummaryxref = input_location + "/fmsummaryxref.csv"
destination = open(fmsummaryxref, 'wb')
destination.write("output_id,summary_id,summaryset_id\n")
shutil.copyfileobj(open(input_location + "/fmsummaryxref_temp.csv", 'rb'),
destination)
destination.close()
os.remove(input_location + "/fmsummaryxref_temp.csv")
risummaryxref = input_location + "/risummaryxref.csv"
destination = open(risummaryxref, 'wb')
destination.write("output_id,summary_id,summaryset_id\n")
shutil.copyfileobj(open(input_location + "/risummaryxref_temp.csv", 'rb'),
destination)
destination.close()
os.remove(input_location + "/risummaryxref_temp.csv")
dirs = os.listdir(input_location)
ri_dirs = []
for dir in dirs:
if dir.startswith('RI_'):
ri_dirs.append(dir)
for dir in ri_dirs:
full_dir = os.path.join(input_location, dir)
shutil.copy(os.path.join(input_location, "risummaryxref.csv"),
os.path.join(full_dir, "fmsummaryxref.csv"))
process_run_locationid = flamingo_db_utils.get_process_run_locationid(
prog_oasis_location, process_dir, processrunid)
flamingo_db_utils.generate_oasis_files_records_outputs(
processrunid, process_run_locationid)
return input_location