def log_element_run_to_db(element_run_id, status, description): db.execute( """ exec dbo.LogElementRun @ElementRunID = ?, @Status = ?, @Description = ? """, element_run_id, status, description)
def update_process_run_status(processrunid, status): db.execute( """ UPDATE dbo.ProcessRun SET ProcessRunStatus = ? WHERE ProcessRunID = ? """, status, processrunid)
def generate_oasis_files_records_outputs(processrunid, process_run_locationid): db.execute( """ exec dbo.generateOasisFilesRecordsOutputs @ProcessRunId = ?, @LocationID = ?, @GulSummaryXrefFileName = ?, @FMSummaryXrefFileName = ? """, processrunid, process_run_locationid, 'gulsummaryxref.csv', 'fmsummaryxref.csv')
def generate_oasis_file_records(progoasisid, location_id): db.execute( """ exec dbo.generateOasisFilesRecords @ProgOasisId = ?, @LocationID = ?, @ItemsFileName = 'items.csv', @CoveragesFileName = 'coverages.csv', @ItemDictFileName = 'ItemDict.csv', @FMProgrammeFileName = 'fm_programme.csv', @FMPolicyTCFileName = 'fm_policytc.csv', @FMProfileFileName = 'fm_profile.csv', @FMXRefFileName = 'fm_xref.csv', @FMDictFileName = 'FMDict.csv' """, progoasisid, location_id)
def do_run_prog_oasis(processrunid): ''' Run a programme model combination. ''' element_run_ids = list() element_run_id = -1 flamingo_db_utils.update_process_run_status(processrunid, "In Progress") try: base_url = flamingo_db_utils.get_base_url(processrunid) element_run_ids = \ flamingo_db_utils.get_element_run_ids(processrunid) upload_directory = generate_summary_files(processrunid) logging.getLogger().debug( "Upload_directory: {}".format(upload_directory)) analysis_settings_json = get_analysis_settings_json(processrunid) logging.getLogger().debug(analysis_settings_json) if 'il_output' in analysis_settings_json['analysis_settings']: create_il_bins = analysis_settings_json['analysis_settings'][ 'il_output'] else: create_il_bins = False analysis_poll_interval_in_seconds = 5 client = OasisAPIClient(base_url, logging.getLogger()) element_run_id = element_run_ids[0][0] input_location = client.upload_inputs_from_directory( upload_directory, do_il=create_il_bins, do_build=True, do_clean=True) logging.getLogger().info("Input location: {}".format(input_location)) flamingo_db_utils.log_element_run_to_db( element_run_id, "Success", "Exposure files location: {}".format(input_location)) element_run_id = element_run_ids[1][0] analysis_status_location = client.run_analysis(analysis_settings_json, input_location) flamingo_db_utils.log_element_run_to_db(element_run_id, "Success", "Started analysis") element_run_id = element_run_ids[2][0] outputs_location = "" while True: logging.getLogger().debug("Polling analysis status for: {}".format( analysis_status_location)) (status, outputs_location) = \ client.get_analysis_status(analysis_status_location) flamingo_db_utils.log_element_run_to_db(element_run_id, status, "In Progress") if status == status_code.STATUS_SUCCESS: if outputs_location is None: raise Exception("Complete but no outputs location") flamingo_db_utils.log_element_run_to_db( element_run_id, status, "Analysis Completed") break elif status == status_code.STATUS_FAILURE: error_message = "Analysis failed: {}".format(message) logging.getLogger().error(error_message) raise Exception(error_message) time.sleep(analysis_poll_interval_in_seconds) element_run_id = element_run_ids[3][0] # download outputs and cleanup outputs_file = os.path.join(upload_directory, outputs_location + ".tar.gz") client.download_outputs(outputs_location, outputs_file) client.delete_exposure(input_location) client.delete_outputs(outputs_location) flamingo_db_utils.log_element_run_to_db( element_run_id, 'Success', 'Downloaded output files successfully') extract_tarball( os.path.join(upload_directory, outputs_location + ".tar.gz"), upload_directory) output_file_list = ','.join( map(str, os.listdir(os.path.join(upload_directory, "output")))) logging.getLogger().debug( "Output_file_list: {}".format(output_file_list)) db.execute( "exec dbo.linkOutputFileToProcessRun @ProcessRunId = ?, @OutputFiles = ?", processrunid, output_file_list) flamingo_db_utils.update_process_run_status(processrunid, 'Completed') # append summary id meanings to output files output_file_details = flamingo_db_utils.get_output_file_details( processrunid) logging.getLogger().debug( "output_file_details: {}".format(output_file_details)) columns = [ "FileName", "FileDesc", "PerspectiveName", "OutputID", "LECFlag", "AnalysisFileNameStub", "SummaryLevelName" ] df_output_file_details = pd.DataFrame(columns=columns) recs = map(lambda tup: dict(zip(columns, list(tup))), output_file_details) df_output_file_details = df_output_file_details.append(recs) logging.getLogger().debug( "df_output_file_details:\n{}".format(df_output_file_details)) prog_oasis_location = flamingo_db_utils.get_prog_oasis_location( processrunid) itemdict = prog_oasis_location + '/ItemDict.csv' fmdict = prog_oasis_location + '/FMDict.csv' df_itemdict = pd.read_csv(itemdict) df_fmdict = pd.read_csv(fmdict) df_fmdict["policy_layer"] = df_fmdict["policy_name"].map( str) + '--' + df_fmdict["layer_name"].map(str) for index, row in df_output_file_details.iterrows(): output = upload_directory + '/output/' + row['FileName'] logging.getLogger().debug("FileName: {}".format(output)) output_tmp = output + '.tmp' SummaryLevelName = row['SummaryLevelName'] df_output = pd.read_csv(output) SummaryLevelId = SummaryLevelName.lower() + '_id' SummaryLevelDesc = SummaryLevelName.lower() + '_desc' logging.getLogger().debug( "SummaryLevelName: {}".format(SummaryLevelName)) if SummaryLevelName != "Portfolio": if SummaryLevelName == "Policy": # join fmdict to file df_summarydict = df_fmdict[['agg_id', 'policy_layer']] logging.getLogger().debug( "df_summarydict: {}".format(df_summarydict)) df_summarydict_distinct = df_summarydict.drop_duplicates() df_output_temp = df_output.join( df_summarydict_distinct.set_index('agg_id'), on='summary_id') else: # join itemdict to file df_summarydict = df_itemdict[[ SummaryLevelId, SummaryLevelDesc ]] df_summarydict_distinct = df_summarydict.drop_duplicates() df_output_temp = df_output.join( df_summarydict_distinct.set_index(SummaryLevelId), on='summary_id') logging.getLogger().debug( "df_summarydict_distinct: {}".format( df_summarydict_distinct)) df_output_temp.to_csv(output, encoding='utf-8', index=False) except Exception as e: flamingo_db_utils.update_process_run_status(processrunid, "Failed") if element_run_id != -1: flamingo_db_utils.log_element_run_to_db(element_run_id, 'Failed: ', str(e)) logging.getLogger().exception( "Failed to run prog oasis: {}".format(processrunid))
def generate_oasis_files_outputs(processrunid): db.execute( "exec dbo.generateOasisFilesOutputs ?", processrunid)
def load_programme_data(progid): db.execute( "exec dbo.LoadProgrammeData ?", progid)
def generate_canonical_model(progid): db.execute( "exec dbo.generateCanonicalModel ?", progid)
def generate_output_transform_file_records_for_progoasis(progoasisid, modelfilename): db.execute( "exec dbo.generateOutputTransformFileRecordsForProgOasos ?, ?", progoasisid, modelfilename)
def generate_output_transform_file_records_for_prog(progid, canlocfilename, canaccfilename): db.execute( "exec dbo.generateOutputTransformFileRecordsForProg ?, ?, ?", progid, canlocfilename, canaccfilename)
def create_api_error_file_record(filename, progoasisid): db.execute( "exec createAPIErrorFileRecord ?, ?", filename, progoasisid)
def update_progoasis_status(progoasisid, status): db.execute( "exec dbo.updateProgOasisStatus?, ?", progoasisid, status)
def get_api_return_data(progoasisid, filename, sessionid): db.execute( "exec getAPI1aReturnData ?, ?, ?", progoasisid, filename, sessionid)
def update_prog_status(progid, status): db.execute( "exec dbo.updateProgStatus?, ?", progid, status)