def test_occurrence_bin_doesnt_not_exist_event_set_isnt_specified___bin_is_copied_from_static( self): with TemporaryDirectory() as d: self.make_fake_bins(d) with io.open(os.path.join(d, 'static', 'occurrence.bin'), 'w', encoding='utf-8') as occurrence_file: occurrence_file.write('occurrence bin') occurrence_file.flush() settings = {"gul_summaries": [{"lec_output": True}]} prepare_run_inputs(settings, d) with io.open(os.path.join(d, 'input', 'occurrence.bin'), 'r', encoding='utf-8') as new_occurrence_file: self.assertEqual('occurrence bin', new_occurrence_file.read())
def test_version_file_missing___correct_exception_raised(self, model): with TemporaryDirectory() as d: keys_path = os.path.join(d, 'keys') os.mkdir(keys_path) version_path = os.path.join(d, 'version.csv') module_path = os.path.join(d, '{}_lookup.py'.format(model)) self.write_py_module(model, module_path) with self.assertRaisesRegex(OasisException, r"The path .*/version.csv \(model_version_file_path\) is indicated as preexisting" r" but does not exist"): _, instance = olf.create( model_keys_data_path=keys_path, model_version_file_path=version_path, lookup_module_path=module_path )
def test_reinsurance_info_file_is_not_a_valid_format___response_is_400(self): with TemporaryDirectory() as d: with override_settings(MEDIA_ROOT=d): user = fake_user() portfolio = fake_portfolio() response = self.app.post( portfolio.get_absolute_reinsurance_info_file_url(), headers={ 'Authorization': 'Bearer {}'.format(AccessToken.for_user(user)) }, upload_files=( ('file', 'file.tar', b'content'), ), expect_errors=True, ) self.assertEqual(400, response.status_code)
def test_returnperiods_bin_already_exists___existing_bin_is_uncahnged( self): with TemporaryDirectory() as d: self.make_fake_bins(d) with io.open(os.path.join(d, 'input', 'returnperiods.bin'), 'w', encoding='utf-8') as returnperiods_file: returnperiods_file.write('returnperiods bin') returnperiods_file.flush() prepare_run_inputs({}, d) with io.open(os.path.join(d, 'input', 'returnperiods.bin'), 'r', encoding='utf-8') as new_returnperiods_file: self.assertEqual('returnperiods bin', new_returnperiods_file.read())
def test_produced_keys_are_passed_to_write_oasis_keys_file(self, data): with TemporaryDirectory() as d,\ patch('oasislmf.model_preparation.lookup.OasisLookupFactory.get_keys', Mock(return_value=(r for r in data))) as get_keys_mock,\ patch('oasislmf.model_preparation.lookup.OasisLookupFactory.write_oasis_keys_file') as write_oasis_keys_file_mock: keys_file_path = os.path.join(d, 'piwind-keys.csv') OasisLookupFactory.save_keys(lookup=self.create_fake_lookup(), keys_file_path=keys_file_path, model_exposure=json.dumps(data)) get_keys_mock.assert_called_once_with( lookup=self.lookup_instance, model_exposure=json.dumps(data), model_exposure_fp=None, success_only=True) write_oasis_keys_file_mock.assert_called_once_with(data, keys_file_path, id_col='id')
def test_should_calculate_median_class_weights_for_single_image_and_single_color( self): with TemporaryDirectory() as path: tfrecord_filename = os.path.join(path, 'data.tfrecord') get_logger().debug('writing to test tfrecord_filename: %s', tfrecord_filename) write_examples_to_tfrecord( tfrecord_filename, [dict_to_example({'image': encode_png([[COLOR_1, COLOR_2]])})]) class_weights_map = calculate_median_class_weights_for_tfrecord_paths_and_color_map( [tfrecord_filename], 'image', { 'color1': COLOR_1, 'color2': COLOR_2, 'color3': COLOR_3 }, channels=['color1', 'color2']) assert class_weights_map == {'color1': 0.5, 'color2': 0.5}
def test_records_are_given___records_are_written_to_json_keys_files_correctly(self, successes, nonsuccesses): with TemporaryDirectory() as d: keys_file_path = os.path.join(d, 'keys.json') keys_errors_file_path = os.path.join(d, 'keys-errors.json') _, successes_count = olf.write_json_keys_file(successes, keys_file_path) _, nonsuccesses_count = olf.write_json_keys_file(nonsuccesses, keys_errors_file_path) with io.open(keys_file_path, 'r', encoding='utf-8') as f1, io.open(keys_errors_file_path, 'r', encoding='utf-8') as f2: written_successes = json.load(f1) written_nonsuccesses = json.load(f2) self.assertEqual(successes_count, len(successes)) self.assertEqual(written_successes, successes) self.assertEqual(nonsuccesses_count, len(nonsuccesses)) self.assertEqual(written_nonsuccesses, nonsuccesses)
def test_check_gul_and_il_and_multiple_ri_directories_binaries_fail(self): with TemporaryDirectory() as d: for p in INPUT_FILES.values(): Path(os.path.join(d, p['name'] + '.csv')).touch() Path(os.path.join(d, p['name'] + '.bin')).touch() os.mkdir(os.path.join(d, "RI_1")) for p in INPUT_FILES.values(): Path(f=os.path.join(d, "RI_1", p['name'] + '.csv')).touch() Path(f=os.path.join(d, "RI_1", p['name'] + '.bin')).touch() os.mkdir(os.path.join(d, "RI_2")) for p in INPUT_FILES.values(): Path(os.path.join(d, "RI_2", p['name'] + '.bin')).touch() Path(os.path.join(d, "RI_2", p['name'] + '.bin')).touch() with self.assertRaises(OasisException): check_inputs_directory(d, il=True, ri=True, check_binaries=True)
def test_general_settings_file_contains_properties___properties_are_loaded_into_general_settings( self): with TemporaryDirectory() as d: self.create_fake_directory(d) with io.open(os.path.join(d, GENERAL_SETTINGS_FILE), 'w', encoding='utf-8') as f: f.writelines([ 'first,1,int\n', 'second,foo,str\n', 'third,2.2,float\n', ]) data = json.loads(create_analysis_settings_json(d)) self.assertEqual(data['first'], 1) self.assertEqual(data['second'], 'foo') self.assertEqual(data['third'], 2.2)
def __create_rf_input_generic(self, expected, subdir): items_file = os.path.join(TEST_INPUT_DIR, subdir, 'complex_items.csv') with open(items_file, 'r') as f: items_pd = pd.read_csv(f) coverages_file = os.path.join(TEST_INPUT_DIR, subdir, 'coverages.csv') with open(coverages_file, 'r') as f: coverages_pd = pd.read_csv(f) with TemporaryDirectory() as tmp_dir: sqlite_fp = os.path.join(tmp_dir, DEFAULT_DB) create_rf_input(items_pd, coverages_pd, sqlite_fp, TEST_MODEL_DATA_DIR) con = sqlite3.connect(os.path.join(tmp_dir, DEFAULT_DB)) cur = con.cursor() cur.execute("SELECT * from u_exposure;") rows = cur.fetchall() self.assertEqual(1, len(rows)) for i in range(0, len(expected)): self.assertEqual(expected[i], rows[0][i]) con.close()
def test_with_single_reinsurance_subfolder(self, targets): with TemporaryDirectory() as d: os.mkdir(os.path.join(d, 'RI_1')) for target in targets: with io.open(os.path.join(d, target), 'w', encoding='utf-8') as f: f.write(target) with io.open(os.path.join(d, 'RI_1', target), 'w', encoding='utf-8') as f: f.write(target) create_binary_tar_file(d) all_targets = copy(targets) for t in targets: # tarfile converts os-specific separators to forward slashes all_targets.append("RI_1/{}".format(t)) with tarfile.open(os.path.join(d, TAR_FILE), 'r:gz', encoding='utf-8') as tar: self.assertEqual(len(all_targets), len(tar.getnames())) self.assertEqual(set(all_targets), set(tar.getnames()))
def test_cache_engdb(engdb): """ Test cache creation Notes ----- The engdb fixture is called since the cache routine requires the live engineering db RESTful service. """ with TemporaryDirectory() as db_path: engdb_mock.cache_engdb(mnemonics=[GOOD_MNEMONIC], starttime=GOOD_STARTTIME, endtime=GOOD_ENDTIME, db_path=db_path) assert os.path.isfile(os.path.join(db_path, engdb_mock.META)) assert os.path.isfile( os.path.join(db_path, engdb_mock.mnemonic_data_fname(GOOD_MNEMONIC)))
def test_source_to_canonical(self, chunk_size): with TemporaryDirectory() as d: output_file = os.path.join(d, 'canonical.csv') translator = Translator(os.path.join(input_data_dir, 'source.csv'), output_file, os.path.join(input_data_dir, 'source_to_canonical.xslt'), os.path.join(input_data_dir, 'source_to_canonical.xsd'), chunk_size=chunk_size, append_row_nums=True) translator() diff = unified_diff(output_file, os.path.join(expected_data_dir, 'canonical.csv'), as_string=True) self.assertEqual(0, len(diff), diff)
def test_output_tracebackfile__and_status_are_updated(self, traceback): with TemporaryDirectory() as d: with override_settings(MEDIA_ROOT=d): initiator = fake_user() analysis = fake_analysis() record_run_analysis_failure(analysis.pk, initiator.pk, traceback) analysis.refresh_from_db() self.assertEqual(analysis.run_traceback_file.file.read(), traceback.encode()) self.assertEqual(analysis.run_traceback_file.content_type, 'text/plain') self.assertEqual(analysis.run_traceback_file.creator, initiator) self.assertEqual(analysis.status, analysis.status_choices.RUN_ERROR)
def test_get_custom_venv_choices(): bundled_venv = os.path.join(settings.BASE_VENV_PATH, 'ansible', '') assert sorted(common.get_custom_venv_choices()) == [bundled_venv] with TemporaryDirectory(dir=settings.BASE_VENV_PATH, prefix='tmp') as temp_dir: os.makedirs(os.path.join(temp_dir, 'bin', 'activate')) custom_venv_dir = os.path.join(temp_dir, 'custom') custom_venv_1 = os.path.join(custom_venv_dir, 'venv-1') custom_venv_awx = os.path.join(custom_venv_dir, 'custom', 'awx') os.makedirs(os.path.join(custom_venv_1, 'bin', 'activate')) os.makedirs(os.path.join(custom_venv_awx, 'bin', 'activate')) assert sorted(common.get_custom_venv_choices([custom_venv_dir])) == [ bundled_venv, os.path.join(temp_dir, ''), os.path.join(custom_venv_1, '') ]
def local_salt_call(fun, *args, **kwargs): """ Pythonic equivalent to `salt-call --local ...`. Args and kwargs are pased to the provided function `fun`, as though they were passed on the command line. """ # pull the grains out of kwargs if applicable grains = {} if 'grains' in kwargs: grains = kwargs['grains'] del kwargs['grains'] # establish cache dir with TemporaryDirectory('salt-test-cache') as cachedir: # build configuration around a non-existent config file for a 'local' run # NOTE: kwargs are reformatted as 'k=v' pairs as though they were passed on # the commandline opts = salt.config.minion_config('/dev/null') opts.update({ 'id': 'local', 'file_client': 'local', 'cachedir': cachedir, 'local': True, 'grains': grains, 'fileserver_backend': ['roots'], 'file_roots': { 'base': [os.getcwd()] }, 'fun': fun, 'arg': list(args) + ['{}={}'.format(k, v) for k, v in kwargs.iteritems()], 'retcode_passthrough': True, }) #print(json.dumps(opts, indent=4)) return BaseCaller(opts).call()
def test_keys_path_not_supplied___correct_instance_is_created_with_correct_model_info_and_keys_path(self, supplier, model, version): with TemporaryDirectory() as d: version_path = os.path.join(d, 'version.csv') self.write_version_file(supplier, model, version, version_path) module_path = os.path.join(d, '{}_lookup.py'.format(model)) self.write_py_module(model, module_path) _, instance = olf.create( model_version_file_path=version_path, lookup_module_path=module_path ) self.assertEqual(type(instance).__name__, '{}KeysLookup'.format(model)) self.assertEqual(instance.supplier, supplier) self.assertEqual(instance.model_name, model) self.assertEqual(instance.model_version, version) self.assertEqual(instance.keys_data_directory, os.path.join(os.sep, 'var', 'oasis', 'keys_data')) self.assertEqual(instance.complex_lookup_config_fp, None)
def test_output_file_and_status_are_updated(self, output_location): with TemporaryDirectory() as d: with override_settings(MEDIA_ROOT=d): Path(d, output_location).touch() initiator = fake_user() analysis = fake_analysis() run_analysis_success(output_location, analysis.pk, initiator.pk) analysis.refresh_from_db() self.assertEqual(analysis.output_file.file.name, output_location) self.assertEqual(analysis.output_file.content_type, 'application/gzip') self.assertEqual(analysis.output_file.creator, initiator) self.assertEqual(analysis.status, analysis.status_choices.RUN_COMPLETED)
def test_il_settings_contains_data___data_is_stored_in_il_summaries(self): with TemporaryDirectory() as d: self.create_fake_directory(d) with io.open(os.path.join(d, IL_SUMMARIES_FILE), 'w', encoding='utf-8') as f: f.writelines([ '1,leccalc_foo,TRUE\n', '1,leccalc_bar,FALSE\n', '1,another,FALSE\n', '2,leccalc_boo,FALSE\n', '2,leccalc_far,TRUE\n', '2,different,TRUE\n', ]) data = json.loads(create_analysis_settings_json(d)) self.assertEqual(data['il_summaries'], [ {'id': 1, 'another': False, 'leccalc': {'leccalc_foo': True, 'leccalc_bar': False}}, {'id': 2, 'different': True, 'leccalc': {'leccalc_boo': False, 'leccalc_far': True}}, ])
def test_tar_file_exists_in_specified_bin_dir___correct_file_is_posted(self): with TemporaryDirectory() as d, responses.RequestsMock() as rsps: bin_dir = os.path.join(d, 'alt_bin_dir') os.mkdir(bin_dir) Path(os.path.join(bin_dir, TAR_FILE)).touch() rsps.add(responses.POST, url='http://localhost:8001/exposure', body=json.dumps({'exposures': [{'location': 'exposure_location'}]}).encode()) client = OasisAPIClient('http://localhost:8001') client.upload_inputs_from_directory(d, bin_directory=bin_dir) self.assertEqual(1, len(rsps.calls)) request = rsps.calls[0].request self.assertEqual(request.url, 'http://localhost:8001/exposure') multipart_data = request.body.fields['file'] self.assertEqual('inputs.tar.gz', multipart_data[0]) self.assertEqual(os.path.join(bin_dir, TAR_FILE), multipart_data[1].name) self.assertEqual('text/plain', multipart_data[2])
def test_sidx_ordering(self, file_id, stream_type): test_filename = f"test_{file_id}_{stream_type}.csv" with TemporaryDirectory() as working_dir: file_fp = os.path.join(TEST_INPUT_DIR, test_filename) gul_csv_fp = recreate_csv_from_bin(working_dir, file_fp, stream_type) expected = pd.read_csv(gul_csv_fp) last_sid = -4 last_loc_batch = 0 for _, row in expected.iterrows(): current_loc_batch = row[ "coverage_id"] if "coverage_id" in row else row["item_id"] if last_loc_batch == 0: last_loc_batch = current_loc_batch sidx = row["sidx"] if last_loc_batch != current_loc_batch: last_sid = -4 self.assertLess(last_sid, sidx) last_sid = sidx
def test_input_generation_traceback_file_is_present___file_can_be_retrieved( self, file_content): with TemporaryDirectory() as d: with override_settings(MEDIA_ROOT=d): user = fake_user() analysis = fake_analysis( input_generation_traceback_file=fake_related_file( file=file_content, content_type='text/plain')) response = self.app.get( analysis.get_absolute_input_generation_traceback_file_url( ), headers={ 'Authorization': 'Bearer {}'.format(AccessToken.for_user(user)) }, ) self.assertEqual(response.body, file_content) self.assertEqual(response.content_type, 'text/plain')
def test_keys_path_is_supplied___correct_instance_is_created_with_correct_model_info_and_keys_path(self, supplier, model, version): with TemporaryDirectory() as d: keys_path = os.path.join(d, 'keys') version_path = os.path.join(d, 'version.csv') self.write_version_file(supplier, model, version, version_path) module_path = os.path.join(d, '{}_lookup.py'.format(model)) self.write_py_module(model, module_path) _, instance = OasisKeysLookupFactory.create( model_keys_data_path=keys_path, model_version_file_path=version_path, lookup_package_path=module_path, ) self.assertEqual(type(instance).__name__, '{}KeysLookup'.format(model)) self.assertEqual(instance.supplier, supplier) self.assertEqual(instance.model_name, model) self.assertEqual(instance.model_version, version) self.assertEqual(instance.keys_data_directory, keys_path)
def test_do_build_is_false___bin_building_functions_are_not_called( self, check_mock, check_tools_mock, create_bin_mock, create_tar_mock): with TemporaryDirectory() as d, responses.RequestsMock() as rsps: Path(os.path.join(d, TAR_FILE)).touch() rsps.add(responses.POST, url='http://localhost:8001/exposure', body=json.dumps({ 'exposures': [{ 'location': 'exposure_location' }] }).encode()) client = OasisAPIClient('http://localhost:8001') client.upload_inputs_from_directory(d, do_build=False) check_mock.assert_not_called() check_tools_mock.assert_not_called() create_bin_mock.assert_not_called() create_tar_mock.assert_not_called()
def test_events_bin_doesnt_not_exist_event_set_is_specified___event_set_specific_bin_is_copied_from_static( self): with TemporaryDirectory() as d: self.make_fake_bins(d) with io.open(os.path.join(d, 'static', 'events_from_set.bin'), 'w', encoding='utf-8') as events_file: events_file.write('events from set bin') events_file.flush() prepare_run_inputs( {'model_settings': { 'event_set': 'from set' }}, d) with io.open(os.path.join(d, 'input', 'events.bin'), 'r', encoding='utf-8') as new_events_file: self.assertEqual('events from set bin', new_events_file.read())
def test_check_gul_and_il_and_multiple_ri_directories(self): with TemporaryDirectory() as d: for p in INPUT_FILES.values(): Path(os.path.join(d, p['name'] + '.csv')).touch() os.mkdir(os.path.join(d, "RI_1")) for p in INPUT_FILES.values(): Path(os.path.join(d, "RI_1", p['name'] + '.csv')).touch() os.mkdir(os.path.join(d, "RI_2")) for p in INPUT_FILES.values(): Path(os.path.join(d, "RI_2", p['name'] + '.csv')).touch() try: check_inputs_directory(d, il=True, ri=True, check_binaries=True) except Exception as e: self.fail('Exception was raised {}: {}'.format(type(e), e))
def test_location_file_invalid_uploaded___parquet_exception_raised(self): content_type = 'text/csv' file_content = b'\xf2hb\xca\xd2\xe6\xf3\xb0\xc1\xc7' with TemporaryDirectory() as d: with override_settings(MEDIA_ROOT=d, PORTFOLIO_PARQUET_STORAGE=True): user = fake_user() portfolio = fake_portfolio() response = self.app.post( portfolio.get_absolute_location_file_url(), headers={ 'Authorization': 'Bearer {}'.format(AccessToken.for_user(user)) }, upload_files=(('file', 'file{}'.format( mimetypes.guess_extension(content_type)), file_content), ), expect_errors=True) self.assertEqual(400, response.status_code)
def test_do_build_is_true_do_il_is_true___bin_building_functions_are_called_with_correct_args( self, check_mock, check_tools_mock, create_bin_mock, create_tar_mock): create_tar_mock.side_effect = fake_build_tar_fn with TemporaryDirectory() as d, responses.RequestsMock() as rsps: rsps.add(responses.POST, url='http://localhost:8001/exposure', body=json.dumps({ 'exposures': [{ 'location': 'exposure_location' }] }).encode()) client = OasisAPIClient('http://localhost:8001') client.upload_inputs_from_directory(d, do_build=True, do_il=True) check_mock.assert_called_once_with(d, do_il=True) check_tools_mock.assert_called_once_with(do_il=True) create_bin_mock.assert_called_once_with(d, d, do_il=True) create_tar_mock.assert_called_once_with(d)
def test_model_settings_file_contains_properties___properties_are_loaded_into_model_settings( self): with TemporaryDirectory() as d: self.create_fake_directory(d) with io.open(os.path.join(d, MODEL_SETTINGS_FILE), 'w', encoding='utf-8') as f: f.writelines([ 'first,3,int\n', 'second,bar,str\n', 'third,4.4,float\n', ]) data = json.loads(create_analysis_settings_json(d)) self.assertEqual(data['model_settings'], { 'first': 3, 'second': 'bar', 'third': 4.4, })
def test_do_clean_is_true_and_bin_dir_is_specified___clean_bin_directory_is_called_on_the_correct_directory( self, clean_mock): with TemporaryDirectory() as d, responses.RequestsMock() as rsps: bin_dir = os.path.join(d, 'alt_bin_dir') os.mkdir(bin_dir) Path(os.path.join(bin_dir, TAR_FILE)).touch() rsps.add(responses.POST, url='http://localhost:8001/exposure', body=json.dumps({ 'exposures': [{ 'location': 'exposure_location' }] }).encode()) client = OasisAPIClient('http://localhost:8001') client.upload_inputs_from_directory(d, bin_directory=bin_dir, do_clean=True) clean_mock.assert_called_once_with(bin_dir)