def generate_summary_files(processrunid): flamingo_db_utils.generate_oasis_files_outputs(processrunid) prog_oasis_location = \ flamingo_db_utils.get_prog_oasis_location(processrunid) ts = values.get_timestamp() process_dir = "ProcessRun_" + str(processrunid) + "_" + ts input_location = str(prog_oasis_location) + "/" + str(process_dir) if not os.path.isdir(input_location): os.mkdir(input_location) for i in ("items", "coverages", "fm_programme", "fm_policytc", "fm_xref", "fm_profile"): source_file = "{}/{}.csv".format(prog_oasis_location, i) target_file = "{}/{}.csv".format(input_location, i) if not IS_WINDOWS_HOST: os.symlink(source_file, target_file) else: shutil.copy(source_file, target_file) db.bcp("OasisGULSUMMARYXREF", input_location + "/gulsummaryxref_temp.csv") db.bcp("OasisFMSUMMARYXREF", input_location + "/fmsummaryxref_temp.csv") gulsummaryxref = input_location + "/gulsummaryxref.csv" destination = open(gulsummaryxref, 'wb') destination.write("coverage_id,summary_id,summaryset_id\n") shutil.copyfileobj(open(input_location + "/gulsummaryxref_temp.csv", 'rb'), destination) destination.close() os.remove(input_location + "/gulsummaryxref_temp.csv") fmsummaryxref = input_location + "/fmsummaryxref.csv" destination = open(fmsummaryxref, 'wb') destination.write("output_id,summary_id,summaryset_id\n") shutil.copyfileobj(open(input_location + "/fmsummaryxref_temp.csv", 'rb'), destination) destination.close() os.remove(input_location + "/fmsummaryxref_temp.csv") process_run_locationid = flamingo_db_utils.get_process_run_locationid( prog_oasis_location, process_dir, processrunid) flamingo_db_utils.generate_oasis_files_records_outputs( processrunid, process_run_locationid) return input_location
def do_run_prog_oasis(processrunid): ''' Run a programme model combination. ''' element_run_ids = list() element_run_id = -1 flamingo_db_utils.update_process_run_status(processrunid, "In Progress") try: base_url = flamingo_db_utils.get_base_url(processrunid) element_run_ids = \ flamingo_db_utils.get_element_run_ids(processrunid) upload_directory = generate_summary_files(processrunid) logging.getLogger().debug( "Upload_directory: {}".format(upload_directory)) analysis_settings_json = get_analysis_settings_json(processrunid) logging.getLogger().debug(analysis_settings_json) if 'il_output' in analysis_settings_json['analysis_settings']: create_il_bins = analysis_settings_json['analysis_settings'][ 'il_output'] else: create_il_bins = False analysis_poll_interval_in_seconds = 5 client = OasisAPIClient(base_url, logging.getLogger()) element_run_id = element_run_ids[0][0] input_location = client.upload_inputs_from_directory( upload_directory, do_il=create_il_bins, do_build=True, do_clean=True) logging.getLogger().info("Input location: {}".format(input_location)) flamingo_db_utils.log_element_run_to_db( element_run_id, "Success", "Exposure files location: {}".format(input_location)) element_run_id = element_run_ids[1][0] analysis_status_location = client.run_analysis(analysis_settings_json, input_location) flamingo_db_utils.log_element_run_to_db(element_run_id, "Success", "Started analysis") element_run_id = element_run_ids[2][0] outputs_location = "" while True: logging.getLogger().debug("Polling analysis status for: {}".format( analysis_status_location)) (status, outputs_location) = \ client.get_analysis_status(analysis_status_location) flamingo_db_utils.log_element_run_to_db(element_run_id, status, "In Progress") if status == status_code.STATUS_SUCCESS: if outputs_location is None: raise Exception("Complete but no outputs location") flamingo_db_utils.log_element_run_to_db( element_run_id, status, "Analysis Completed") break elif status == status_code.STATUS_FAILURE: error_message = "Analysis failed: {}".format(message) logging.getLogger().error(error_message) raise Exception(error_message) time.sleep(analysis_poll_interval_in_seconds) element_run_id = element_run_ids[3][0] # download outputs and cleanup outputs_file = os.path.join(upload_directory, outputs_location + ".tar.gz") client.download_outputs(outputs_location, outputs_file) client.delete_exposure(input_location) client.delete_outputs(outputs_location) flamingo_db_utils.log_element_run_to_db( element_run_id, 'Success', 'Downloaded output files successfully') extract_tarball( os.path.join(upload_directory, outputs_location + ".tar.gz"), upload_directory) output_file_list = ','.join( map(str, os.listdir(os.path.join(upload_directory, "output")))) logging.getLogger().debug( "Output_file_list: {}".format(output_file_list)) db.execute( "exec dbo.linkOutputFileToProcessRun @ProcessRunId = ?, @OutputFiles = ?", processrunid, output_file_list) flamingo_db_utils.update_process_run_status(processrunid, 'Completed') # append summary id meanings to output files output_file_details = flamingo_db_utils.get_output_file_details( processrunid) logging.getLogger().debug( "output_file_details: {}".format(output_file_details)) columns = [ "FileName", "FileDesc", "PerspectiveName", "OutputID", "LECFlag", "AnalysisFileNameStub", "SummaryLevelName" ] df_output_file_details = pd.DataFrame(columns=columns) recs = map(lambda tup: dict(zip(columns, list(tup))), output_file_details) df_output_file_details = df_output_file_details.append(recs) logging.getLogger().debug( "df_output_file_details:\n{}".format(df_output_file_details)) prog_oasis_location = flamingo_db_utils.get_prog_oasis_location( processrunid) itemdict = prog_oasis_location + '/ItemDict.csv' fmdict = prog_oasis_location + '/FMDict.csv' df_itemdict = pd.read_csv(itemdict) df_fmdict = pd.read_csv(fmdict) df_fmdict["policy_layer"] = df_fmdict["policy_name"].map( str) + '--' + df_fmdict["layer_name"].map(str) for index, row in df_output_file_details.iterrows(): output = upload_directory + '/output/' + row['FileName'] logging.getLogger().debug("FileName: {}".format(output)) output_tmp = output + '.tmp' SummaryLevelName = row['SummaryLevelName'] df_output = pd.read_csv(output) SummaryLevelId = SummaryLevelName.lower() + '_id' SummaryLevelDesc = SummaryLevelName.lower() + '_desc' logging.getLogger().debug( "SummaryLevelName: {}".format(SummaryLevelName)) if SummaryLevelName != "Portfolio": if SummaryLevelName == "Policy": # join fmdict to file df_summarydict = df_fmdict[['agg_id', 'policy_layer']] logging.getLogger().debug( "df_summarydict: {}".format(df_summarydict)) df_summarydict_distinct = df_summarydict.drop_duplicates() df_output_temp = df_output.join( df_summarydict_distinct.set_index('agg_id'), on='summary_id') else: # join itemdict to file df_summarydict = df_itemdict[[ SummaryLevelId, SummaryLevelDesc ]] df_summarydict_distinct = df_summarydict.drop_duplicates() df_output_temp = df_output.join( df_summarydict_distinct.set_index(SummaryLevelId), on='summary_id') logging.getLogger().debug( "df_summarydict_distinct: {}".format( df_summarydict_distinct)) df_output_temp.to_csv(output, encoding='utf-8', index=False) except Exception as e: flamingo_db_utils.update_process_run_status(processrunid, "Failed") if element_run_id != -1: flamingo_db_utils.log_element_run_to_db(element_run_id, 'Failed: ', str(e)) logging.getLogger().exception( "Failed to run prog oasis: {}".format(processrunid))
def generate_summary_files(processrunid): flamingo_db_utils.generate_oasis_files_outputs(processrunid) prog_oasis_location = \ flamingo_db_utils.get_prog_oasis_location(processrunid) ts = values.get_timestamp() process_dir = "ProcessRun_" + str(processrunid) + "_" + ts input_location = str(prog_oasis_location) + "/" + str(process_dir) if not os.path.isdir(input_location): os.mkdir(input_location) for i in ("items", "coverages", "fm_programme", "fm_policytc", "fm_xref", "fm_profile"): source_file = "{}/{}.csv".format(prog_oasis_location, i) target_file = "{}/{}.csv".format(input_location, i) if not IS_WINDOWS_HOST: os.symlink(source_file, target_file) else: shutil.copy(source_file, target_file) input_dir_list = os.listdir(prog_oasis_location) ri_dir_list = [] for dirs in input_dir_list: if dirs.startswith('RI_'): ri_dir_list.append(dirs) for dirs in ri_dir_list: ri_full_path = prog_oasis_location + '/' + dirs ri_target_path = input_location + '/' + dirs if not IS_WINDOWS_HOST: os.symlink(ri_full_path, ri_target_path) else: shutil.copytree(ri_full_path, ri_target_path) db.bcp("OasisGULSUMMARYXREF", input_location + "/gulsummaryxref_temp.csv") db.bcp("OasisFMSUMMARYXREF", input_location + "/fmsummaryxref_temp.csv") db.bcp("OasisRISUMMARYXREF", input_location + "/risummaryxref_temp.csv") gulsummaryxref = input_location + "/gulsummaryxref.csv" destination = open(gulsummaryxref, 'wb') destination.write("coverage_id,summary_id,summaryset_id\n") shutil.copyfileobj(open(input_location + "/gulsummaryxref_temp.csv", 'rb'), destination) destination.close() os.remove(input_location + "/gulsummaryxref_temp.csv") fmsummaryxref = input_location + "/fmsummaryxref.csv" destination = open(fmsummaryxref, 'wb') destination.write("output_id,summary_id,summaryset_id\n") shutil.copyfileobj(open(input_location + "/fmsummaryxref_temp.csv", 'rb'), destination) destination.close() os.remove(input_location + "/fmsummaryxref_temp.csv") risummaryxref = input_location + "/risummaryxref.csv" destination = open(risummaryxref, 'wb') destination.write("output_id,summary_id,summaryset_id\n") shutil.copyfileobj(open(input_location + "/risummaryxref_temp.csv", 'rb'), destination) destination.close() os.remove(input_location + "/risummaryxref_temp.csv") dirs = os.listdir(input_location) ri_dirs = [] for dir in dirs: if dir.startswith('RI_'): ri_dirs.append(dir) for dir in ri_dirs: full_dir = os.path.join(input_location, dir) shutil.copy(os.path.join(input_location, "risummaryxref.csv"), os.path.join(full_dir, "fmsummaryxref.csv")) process_run_locationid = flamingo_db_utils.get_process_run_locationid( prog_oasis_location, process_dir, processrunid) flamingo_db_utils.generate_oasis_files_records_outputs( processrunid, process_run_locationid) return input_location