def test_no_occurrence_bin_exists___oasis_exception_is_raised(self): with TemporaryDirectory() as d: self.make_fake_bins(d) os.remove(os.path.join(d, 'static', 'occurrence.bin')) with self.assertRaises(OasisException): prepare_model_run_inputs({}, d)
def test_periods_bin_doesnt_not_exist_event_set_isnt_specified___bin_is_copied_from_static(self): with TemporaryDirectory() as d: self.make_fake_bins(d) with io.open(os.path.join(d, 'static', 'periods.bin'), 'w', encoding='utf-8') as periods_file: periods_file.write('periods bin') periods_file.flush() prepare_model_run_inputs({}, d) with io.open(os.path.join(d, 'input', 'periods.bin'), 'r', encoding='utf-8') as new_periods_file: self.assertEqual('periods bin', new_periods_file.read())
def test_periods_bin_already_exists___existing_bin_is_uncahnged(self): with TemporaryDirectory() as d: self.make_fake_bins(d) with io.open(os.path.join(d, 'input', 'periods.bin'), 'w', encoding='utf-8') as periods_file: periods_file.write('periods bin') periods_file.flush() prepare_model_run_inputs({}, d) with io.open(os.path.join(d, 'input', 'periods.bin'), 'r', encoding='utf-8') as new_periods_file: self.assertEqual('periods bin', new_periods_file.read())
def test_occurrence_bin_doesnt_not_exist_event_set_is_specified___event_occurrence_id_specific_bin_is_copied_from_static(self): with TemporaryDirectory() as d: self.make_fake_bins(d) with io.open(os.path.join(d, 'static', 'occurrence_occurrence_id.bin'), 'w', encoding='utf-8') as occurrence_file: occurrence_file.write('occurrence occurrence id bin') occurrence_file.flush() prepare_model_run_inputs({'model_settings': {'event_occurrence_id': 'occurrence id'}}, d) with io.open(os.path.join(d, 'input', 'occurrence.bin'), 'r', encoding='utf-8') as new_occurrence_file: self.assertEqual('occurrence occurrence id bin', new_occurrence_file.read())
def test_prepare_input_bin_raises___oasis_exception_is_raised(self): with patch('oasislmf.model_execution.bin._prepare_input_bin', Mock(side_effect=OSError('os error'))): with self.assertRaises(OasisException): prepare_model_run_inputs({}, 'some_dir')
def start_analysis(analysis_settings, input_location): ''' Run an analysis. Args: analysis_profile_json (string): The analysis settings. Returns: (string) The location of the outputs. ''' # Check that the input archive exists and is valid input_archive = os.path.join( settings.get('worker', 'INPUTS_DATA_DIRECTORY'), input_location + ARCHIVE_FILE_SUFFIX) if not os.path.exists(input_archive): raise MissingInputsException(input_archive) if not tarfile.is_tarfile(input_archive): raise InvalidInputsException(input_archive) source_tag = analysis_settings['analysis_settings']['source_tag'] analysis_tag = analysis_settings['analysis_settings']['analysis_tag'] logging.info("Source tag = {}; Analysis tag: {}".format( analysis_tag, source_tag)) module_supplier_id = analysis_settings['analysis_settings'][ 'module_supplier_id'] model_version_id = analysis_settings['analysis_settings'][ 'model_version_id'] logging.info("Model supplier - version = {} {}".format( module_supplier_id, model_version_id)) # Get the supplier module and call it use_default_model_runner = not Path( settings.get('worker', 'SUPPLIER_MODULE_DIRECTORY'), module_supplier_id).exists() model_data_path = os.path.join( settings.get('worker', 'MODEL_DATA_DIRECTORY'), module_supplier_id, model_version_id) if not os.path.exists(model_data_path): raise MissingModelDataException(model_data_path) logging.info("Setting up analysis working directory") directory_name = "{}_{}_{}".format(source_tag, analysis_tag, uuid.uuid4().hex) working_directory = os.path.join( settings.get('worker', 'WORKING_DIRECTORY'), directory_name) if 'ri_output' in analysis_settings['analysis_settings'].keys(): ri = analysis_settings['analysis_settings']['ri_output'] else: ri = False prepare_model_run_directory(working_directory, ri=ri, model_data_src_path=model_data_path, inputs_archive=input_archive) prepare_model_run_inputs(analysis_settings['analysis_settings'], working_directory, ri=ri) with setcwd(working_directory): logging.info("Working directory = {}".format(working_directory)) # Persist the analysis_settings with open("analysis_settings.json", "w") as json_file: json.dump(analysis_settings, json_file) if use_default_model_runner: model_runner_module = runner else: sys.path.append(settings.get('worker', 'SUPPLIER_MODULE_DIRECTORY')) model_runner_module = importlib.import_module( '{}.supplier_model_runner'.format(module_supplier_id)) ##! to add check that RI directories take the form of RI_{ID} amd ID is a monotonic index num_reinsurance_iterations = len(glob.glob('RI_[0-9]')) model_runner_module.run( analysis_settings['analysis_settings'], settings.getint('worker', 'KTOOLS_BATCH_COUNT'), num_reinsurance_iterations=num_reinsurance_iterations, ktools_mem_limit=settings.getboolean('worker', 'KTOOLS_MEMORY_LIMIT'), set_alloc_rule=settings.getint('worker', 'KTOOLS_ALLOC_RULE'), fifo_tmp_dir=False) output_location = uuid.uuid4().hex output_filepath = os.path.join( settings.get('worker', 'OUTPUTS_DATA_DIRECTORY'), output_location + ARCHIVE_FILE_SUFFIX) output_directory = os.path.join(working_directory, "output") with tarfile.open(output_filepath, "w:gz") as tar: tar.add(output_directory, arcname="output") if settings.getboolean('worker', 'DO_CLEAR_WORKING'): shutil.rmtree(working_directory, ignore_errors=True) logging.info("Output location = {}".format(output_location)) return output_location