def test_occurrence_bin_doesnt_not_exist_event_set_isnt_specified___bin_is_copied_from_static(
self):
with TemporaryDirectory() as d:
self.make_fake_bins(d)
with io.open(os.path.join(d, 'static', 'occurrence.bin'),
'w',
encoding='utf-8') as occurrence_file:
occurrence_file.write('occurrence bin')
occurrence_file.flush()
settings = {"gul_summaries": [{"lec_output": True}]}
prepare_run_inputs(settings, d)
with io.open(os.path.join(d, 'input', 'occurrence.bin'),
'r',
encoding='utf-8') as new_occurrence_file:
self.assertEqual('occurrence bin', new_occurrence_file.read())
def test_version_file_missing___correct_exception_raised(self, model):
with TemporaryDirectory() as d:
keys_path = os.path.join(d, 'keys')
os.mkdir(keys_path)
version_path = os.path.join(d, 'version.csv')
module_path = os.path.join(d, '{}_lookup.py'.format(model))
self.write_py_module(model, module_path)
with self.assertRaisesRegex(OasisException,
r"The path .*/version.csv \(model_version_file_path\) is indicated as preexisting"
r" but does not exist"):
_, instance = olf.create(
model_keys_data_path=keys_path,
model_version_file_path=version_path,
lookup_module_path=module_path
)
def test_reinsurance_info_file_is_not_a_valid_format___response_is_400(self):
with TemporaryDirectory() as d:
with override_settings(MEDIA_ROOT=d):
user = fake_user()
portfolio = fake_portfolio()
response = self.app.post(
portfolio.get_absolute_reinsurance_info_file_url(),
headers={
'Authorization': 'Bearer {}'.format(AccessToken.for_user(user))
},
upload_files=(
('file', 'file.tar', b'content'),
),
expect_errors=True,
)
self.assertEqual(400, response.status_code)
def test_returnperiods_bin_already_exists___existing_bin_is_uncahnged(
self):
with TemporaryDirectory() as d:
self.make_fake_bins(d)
with io.open(os.path.join(d, 'input', 'returnperiods.bin'),
'w',
encoding='utf-8') as returnperiods_file:
returnperiods_file.write('returnperiods bin')
returnperiods_file.flush()
prepare_run_inputs({}, d)
with io.open(os.path.join(d, 'input', 'returnperiods.bin'),
'r',
encoding='utf-8') as new_returnperiods_file:
self.assertEqual('returnperiods bin',
new_returnperiods_file.read())
def test_produced_keys_are_passed_to_write_oasis_keys_file(self, data):
with TemporaryDirectory() as d,\
patch('oasislmf.model_preparation.lookup.OasisLookupFactory.get_keys', Mock(return_value=(r for r in data))) as get_keys_mock,\
patch('oasislmf.model_preparation.lookup.OasisLookupFactory.write_oasis_keys_file') as write_oasis_keys_file_mock:
keys_file_path = os.path.join(d, 'piwind-keys.csv')
OasisLookupFactory.save_keys(lookup=self.create_fake_lookup(),
keys_file_path=keys_file_path,
model_exposure=json.dumps(data))
get_keys_mock.assert_called_once_with(
lookup=self.lookup_instance,
model_exposure=json.dumps(data),
model_exposure_fp=None,
success_only=True)
write_oasis_keys_file_mock.assert_called_once_with(data,
keys_file_path,
id_col='id')
def test_should_calculate_median_class_weights_for_single_image_and_single_color(
self):
with TemporaryDirectory() as path:
tfrecord_filename = os.path.join(path, 'data.tfrecord')
get_logger().debug('writing to test tfrecord_filename: %s',
tfrecord_filename)
write_examples_to_tfrecord(
tfrecord_filename,
[dict_to_example({'image': encode_png([[COLOR_1, COLOR_2]])})])
class_weights_map = calculate_median_class_weights_for_tfrecord_paths_and_color_map(
[tfrecord_filename],
'image', {
'color1': COLOR_1,
'color2': COLOR_2,
'color3': COLOR_3
},
channels=['color1', 'color2'])
assert class_weights_map == {'color1': 0.5, 'color2': 0.5}
def test_records_are_given___records_are_written_to_json_keys_files_correctly(self, successes, nonsuccesses):
with TemporaryDirectory() as d:
keys_file_path = os.path.join(d, 'keys.json')
keys_errors_file_path = os.path.join(d, 'keys-errors.json')
_, successes_count = olf.write_json_keys_file(successes, keys_file_path)
_, nonsuccesses_count = olf.write_json_keys_file(nonsuccesses, keys_errors_file_path)
with io.open(keys_file_path, 'r', encoding='utf-8') as f1, io.open(keys_errors_file_path, 'r', encoding='utf-8') as f2:
written_successes = json.load(f1)
written_nonsuccesses = json.load(f2)
self.assertEqual(successes_count, len(successes))
self.assertEqual(written_successes, successes)
self.assertEqual(nonsuccesses_count, len(nonsuccesses))
self.assertEqual(written_nonsuccesses, nonsuccesses)
def test_check_gul_and_il_and_multiple_ri_directories_binaries_fail(self):
with TemporaryDirectory() as d:
for p in INPUT_FILES.values():
Path(os.path.join(d, p['name'] + '.csv')).touch()
Path(os.path.join(d, p['name'] + '.bin')).touch()
os.mkdir(os.path.join(d, "RI_1"))
for p in INPUT_FILES.values():
Path(f=os.path.join(d, "RI_1", p['name'] + '.csv')).touch()
Path(f=os.path.join(d, "RI_1", p['name'] + '.bin')).touch()
os.mkdir(os.path.join(d, "RI_2"))
for p in INPUT_FILES.values():
Path(os.path.join(d, "RI_2", p['name'] + '.bin')).touch()
Path(os.path.join(d, "RI_2", p['name'] + '.bin')).touch()
with self.assertRaises(OasisException):
check_inputs_directory(d,
il=True,
ri=True,
check_binaries=True)
def test_general_settings_file_contains_properties___properties_are_loaded_into_general_settings(
self):
with TemporaryDirectory() as d:
self.create_fake_directory(d)
with io.open(os.path.join(d, GENERAL_SETTINGS_FILE),
'w',
encoding='utf-8') as f:
f.writelines([
'first,1,int\n',
'second,foo,str\n',
'third,2.2,float\n',
])
data = json.loads(create_analysis_settings_json(d))
self.assertEqual(data['first'], 1)
self.assertEqual(data['second'], 'foo')
self.assertEqual(data['third'], 2.2)
def __create_rf_input_generic(self, expected, subdir):
items_file = os.path.join(TEST_INPUT_DIR, subdir, 'complex_items.csv')
with open(items_file, 'r') as f:
items_pd = pd.read_csv(f)
coverages_file = os.path.join(TEST_INPUT_DIR, subdir, 'coverages.csv')
with open(coverages_file, 'r') as f:
coverages_pd = pd.read_csv(f)
with TemporaryDirectory() as tmp_dir:
sqlite_fp = os.path.join(tmp_dir, DEFAULT_DB)
create_rf_input(items_pd, coverages_pd, sqlite_fp,
TEST_MODEL_DATA_DIR)
con = sqlite3.connect(os.path.join(tmp_dir, DEFAULT_DB))
cur = con.cursor()
cur.execute("SELECT * from u_exposure;")
rows = cur.fetchall()
self.assertEqual(1, len(rows))
for i in range(0, len(expected)):
self.assertEqual(expected[i], rows[0][i])
con.close()
def test_with_single_reinsurance_subfolder(self, targets):
with TemporaryDirectory() as d:
os.mkdir(os.path.join(d, 'RI_1'))
for target in targets:
with io.open(os.path.join(d, target), 'w', encoding='utf-8') as f:
f.write(target)
with io.open(os.path.join(d, 'RI_1', target), 'w', encoding='utf-8') as f:
f.write(target)
create_binary_tar_file(d)
all_targets = copy(targets)
for t in targets:
# tarfile converts os-specific separators to forward slashes
all_targets.append("RI_1/{}".format(t))
with tarfile.open(os.path.join(d, TAR_FILE), 'r:gz', encoding='utf-8') as tar:
self.assertEqual(len(all_targets), len(tar.getnames()))
self.assertEqual(set(all_targets), set(tar.getnames()))
def test_cache_engdb(engdb):
"""
Test cache creation
Notes
-----
The engdb fixture is called since the cache routine requires
the live engineering db RESTful service.
"""
with TemporaryDirectory() as db_path:
engdb_mock.cache_engdb(mnemonics=[GOOD_MNEMONIC],
starttime=GOOD_STARTTIME,
endtime=GOOD_ENDTIME,
db_path=db_path)
assert os.path.isfile(os.path.join(db_path, engdb_mock.META))
assert os.path.isfile(
os.path.join(db_path,
engdb_mock.mnemonic_data_fname(GOOD_MNEMONIC)))
def test_source_to_canonical(self, chunk_size):
with TemporaryDirectory() as d:
output_file = os.path.join(d, 'canonical.csv')
translator = Translator(os.path.join(input_data_dir, 'source.csv'),
output_file,
os.path.join(input_data_dir,
'source_to_canonical.xslt'),
os.path.join(input_data_dir,
'source_to_canonical.xsd'),
chunk_size=chunk_size,
append_row_nums=True)
translator()
diff = unified_diff(output_file,
os.path.join(expected_data_dir,
'canonical.csv'),
as_string=True)
self.assertEqual(0, len(diff), diff)
def test_output_tracebackfile__and_status_are_updated(self, traceback):
with TemporaryDirectory() as d:
with override_settings(MEDIA_ROOT=d):
initiator = fake_user()
analysis = fake_analysis()
record_run_analysis_failure(analysis.pk, initiator.pk,
traceback)
analysis.refresh_from_db()
self.assertEqual(analysis.run_traceback_file.file.read(),
traceback.encode())
self.assertEqual(analysis.run_traceback_file.content_type,
'text/plain')
self.assertEqual(analysis.run_traceback_file.creator,
initiator)
self.assertEqual(analysis.status,
analysis.status_choices.RUN_ERROR)
def test_get_custom_venv_choices():
bundled_venv = os.path.join(settings.BASE_VENV_PATH, 'ansible', '')
assert sorted(common.get_custom_venv_choices()) == [bundled_venv]
with TemporaryDirectory(dir=settings.BASE_VENV_PATH, prefix='tmp') as temp_dir:
os.makedirs(os.path.join(temp_dir, 'bin', 'activate'))
custom_venv_dir = os.path.join(temp_dir, 'custom')
custom_venv_1 = os.path.join(custom_venv_dir, 'venv-1')
custom_venv_awx = os.path.join(custom_venv_dir, 'custom', 'awx')
os.makedirs(os.path.join(custom_venv_1, 'bin', 'activate'))
os.makedirs(os.path.join(custom_venv_awx, 'bin', 'activate'))
assert sorted(common.get_custom_venv_choices([custom_venv_dir])) == [
bundled_venv,
os.path.join(temp_dir, ''),
os.path.join(custom_venv_1, '')
]
def local_salt_call(fun, *args, **kwargs):
""" Pythonic equivalent to `salt-call --local ...`. Args and kwargs
are pased to the provided function `fun`, as though they were
passed on the command line.
"""
# pull the grains out of kwargs if applicable
grains = {}
if 'grains' in kwargs:
grains = kwargs['grains']
del kwargs['grains']
# establish cache dir
with TemporaryDirectory('salt-test-cache') as cachedir:
# build configuration around a non-existent config file for a 'local' run
# NOTE: kwargs are reformatted as 'k=v' pairs as though they were passed on
# the commandline
opts = salt.config.minion_config('/dev/null')
opts.update({
'id':
'local',
'file_client':
'local',
'cachedir':
cachedir,
'local':
True,
'grains':
grains,
'fileserver_backend': ['roots'],
'file_roots': {
'base': [os.getcwd()]
},
'fun':
fun,
'arg':
list(args) + ['{}={}'.format(k, v) for k, v in kwargs.iteritems()],
'retcode_passthrough':
True,
})
#print(json.dumps(opts, indent=4))
return BaseCaller(opts).call()
def test_keys_path_not_supplied___correct_instance_is_created_with_correct_model_info_and_keys_path(self, supplier,
model, version):
with TemporaryDirectory() as d:
version_path = os.path.join(d, 'version.csv')
self.write_version_file(supplier, model, version, version_path)
module_path = os.path.join(d, '{}_lookup.py'.format(model))
self.write_py_module(model, module_path)
_, instance = olf.create(
model_version_file_path=version_path,
lookup_module_path=module_path
)
self.assertEqual(type(instance).__name__, '{}KeysLookup'.format(model))
self.assertEqual(instance.supplier, supplier)
self.assertEqual(instance.model_name, model)
self.assertEqual(instance.model_version, version)
self.assertEqual(instance.keys_data_directory, os.path.join(os.sep, 'var', 'oasis', 'keys_data'))
self.assertEqual(instance.complex_lookup_config_fp, None)
def test_output_file_and_status_are_updated(self, output_location):
with TemporaryDirectory() as d:
with override_settings(MEDIA_ROOT=d):
Path(d, output_location).touch()
initiator = fake_user()
analysis = fake_analysis()
run_analysis_success(output_location, analysis.pk,
initiator.pk)
analysis.refresh_from_db()
self.assertEqual(analysis.output_file.file.name,
output_location)
self.assertEqual(analysis.output_file.content_type,
'application/gzip')
self.assertEqual(analysis.output_file.creator, initiator)
self.assertEqual(analysis.status,
analysis.status_choices.RUN_COMPLETED)
def test_il_settings_contains_data___data_is_stored_in_il_summaries(self):
with TemporaryDirectory() as d:
self.create_fake_directory(d)
with io.open(os.path.join(d, IL_SUMMARIES_FILE), 'w', encoding='utf-8') as f:
f.writelines([
'1,leccalc_foo,TRUE\n',
'1,leccalc_bar,FALSE\n',
'1,another,FALSE\n',
'2,leccalc_boo,FALSE\n',
'2,leccalc_far,TRUE\n',
'2,different,TRUE\n',
])
data = json.loads(create_analysis_settings_json(d))
self.assertEqual(data['il_summaries'], [
{'id': 1, 'another': False, 'leccalc': {'leccalc_foo': True, 'leccalc_bar': False}},
{'id': 2, 'different': True, 'leccalc': {'leccalc_boo': False, 'leccalc_far': True}},
])
def test_tar_file_exists_in_specified_bin_dir___correct_file_is_posted(self):
with TemporaryDirectory() as d, responses.RequestsMock() as rsps:
bin_dir = os.path.join(d, 'alt_bin_dir')
os.mkdir(bin_dir)
Path(os.path.join(bin_dir, TAR_FILE)).touch()
rsps.add(responses.POST, url='http://localhost:8001/exposure', body=json.dumps({'exposures': [{'location': 'exposure_location'}]}).encode())
client = OasisAPIClient('http://localhost:8001')
client.upload_inputs_from_directory(d, bin_directory=bin_dir)
self.assertEqual(1, len(rsps.calls))
request = rsps.calls[0].request
self.assertEqual(request.url, 'http://localhost:8001/exposure')
multipart_data = request.body.fields['file']
self.assertEqual('inputs.tar.gz', multipart_data[0])
self.assertEqual(os.path.join(bin_dir, TAR_FILE), multipart_data[1].name)
self.assertEqual('text/plain', multipart_data[2])
def test_sidx_ordering(self, file_id, stream_type):
test_filename = f"test_{file_id}_{stream_type}.csv"
with TemporaryDirectory() as working_dir:
file_fp = os.path.join(TEST_INPUT_DIR, test_filename)
gul_csv_fp = recreate_csv_from_bin(working_dir, file_fp,
stream_type)
expected = pd.read_csv(gul_csv_fp)
last_sid = -4
last_loc_batch = 0
for _, row in expected.iterrows():
current_loc_batch = row[
"coverage_id"] if "coverage_id" in row else row["item_id"]
if last_loc_batch == 0:
last_loc_batch = current_loc_batch
sidx = row["sidx"]
if last_loc_batch != current_loc_batch:
last_sid = -4
self.assertLess(last_sid, sidx)
last_sid = sidx
def test_input_generation_traceback_file_is_present___file_can_be_retrieved(
self, file_content):
with TemporaryDirectory() as d:
with override_settings(MEDIA_ROOT=d):
user = fake_user()
analysis = fake_analysis(
input_generation_traceback_file=fake_related_file(
file=file_content, content_type='text/plain'))
response = self.app.get(
analysis.get_absolute_input_generation_traceback_file_url(
),
headers={
'Authorization':
'Bearer {}'.format(AccessToken.for_user(user))
},
)
self.assertEqual(response.body, file_content)
self.assertEqual(response.content_type, 'text/plain')
def test_keys_path_is_supplied___correct_instance_is_created_with_correct_model_info_and_keys_path(self, supplier, model, version):
with TemporaryDirectory() as d:
keys_path = os.path.join(d, 'keys')
version_path = os.path.join(d, 'version.csv')
self.write_version_file(supplier, model, version, version_path)
module_path = os.path.join(d, '{}_lookup.py'.format(model))
self.write_py_module(model, module_path)
_, instance = OasisKeysLookupFactory.create(
model_keys_data_path=keys_path,
model_version_file_path=version_path,
lookup_package_path=module_path,
)
self.assertEqual(type(instance).__name__, '{}KeysLookup'.format(model))
self.assertEqual(instance.supplier, supplier)
self.assertEqual(instance.model_name, model)
self.assertEqual(instance.model_version, version)
self.assertEqual(instance.keys_data_directory, keys_path)
def test_do_build_is_false___bin_building_functions_are_not_called(
self, check_mock, check_tools_mock, create_bin_mock,
create_tar_mock):
with TemporaryDirectory() as d, responses.RequestsMock() as rsps:
Path(os.path.join(d, TAR_FILE)).touch()
rsps.add(responses.POST,
url='http://localhost:8001/exposure',
body=json.dumps({
'exposures': [{
'location': 'exposure_location'
}]
}).encode())
client = OasisAPIClient('http://localhost:8001')
client.upload_inputs_from_directory(d, do_build=False)
check_mock.assert_not_called()
check_tools_mock.assert_not_called()
create_bin_mock.assert_not_called()
create_tar_mock.assert_not_called()
def test_events_bin_doesnt_not_exist_event_set_is_specified___event_set_specific_bin_is_copied_from_static(
self):
with TemporaryDirectory() as d:
self.make_fake_bins(d)
with io.open(os.path.join(d, 'static', 'events_from_set.bin'),
'w',
encoding='utf-8') as events_file:
events_file.write('events from set bin')
events_file.flush()
prepare_run_inputs(
{'model_settings': {
'event_set': 'from set'
}}, d)
with io.open(os.path.join(d, 'input', 'events.bin'),
'r',
encoding='utf-8') as new_events_file:
self.assertEqual('events from set bin', new_events_file.read())
def test_check_gul_and_il_and_multiple_ri_directories(self):
with TemporaryDirectory() as d:
for p in INPUT_FILES.values():
Path(os.path.join(d, p['name'] + '.csv')).touch()
os.mkdir(os.path.join(d, "RI_1"))
for p in INPUT_FILES.values():
Path(os.path.join(d, "RI_1", p['name'] + '.csv')).touch()
os.mkdir(os.path.join(d, "RI_2"))
for p in INPUT_FILES.values():
Path(os.path.join(d, "RI_2", p['name'] + '.csv')).touch()
try:
check_inputs_directory(d,
il=True,
ri=True,
check_binaries=True)
except Exception as e:
self.fail('Exception was raised {}: {}'.format(type(e), e))
def test_location_file_invalid_uploaded___parquet_exception_raised(self):
content_type = 'text/csv'
file_content = b'\xf2hb\xca\xd2\xe6\xf3\xb0\xc1\xc7'
with TemporaryDirectory() as d:
with override_settings(MEDIA_ROOT=d,
PORTFOLIO_PARQUET_STORAGE=True):
user = fake_user()
portfolio = fake_portfolio()
response = self.app.post(
portfolio.get_absolute_location_file_url(),
headers={
'Authorization':
'Bearer {}'.format(AccessToken.for_user(user))
},
upload_files=(('file', 'file{}'.format(
mimetypes.guess_extension(content_type)),
file_content), ),
expect_errors=True)
self.assertEqual(400, response.status_code)
def test_do_build_is_true_do_il_is_true___bin_building_functions_are_called_with_correct_args(
self, check_mock, check_tools_mock, create_bin_mock,
create_tar_mock):
create_tar_mock.side_effect = fake_build_tar_fn
with TemporaryDirectory() as d, responses.RequestsMock() as rsps:
rsps.add(responses.POST,
url='http://localhost:8001/exposure',
body=json.dumps({
'exposures': [{
'location': 'exposure_location'
}]
}).encode())
client = OasisAPIClient('http://localhost:8001')
client.upload_inputs_from_directory(d, do_build=True, do_il=True)
check_mock.assert_called_once_with(d, do_il=True)
check_tools_mock.assert_called_once_with(do_il=True)
create_bin_mock.assert_called_once_with(d, d, do_il=True)
create_tar_mock.assert_called_once_with(d)
def test_model_settings_file_contains_properties___properties_are_loaded_into_model_settings(
self):
with TemporaryDirectory() as d:
self.create_fake_directory(d)
with io.open(os.path.join(d, MODEL_SETTINGS_FILE),
'w',
encoding='utf-8') as f:
f.writelines([
'first,3,int\n',
'second,bar,str\n',
'third,4.4,float\n',
])
data = json.loads(create_analysis_settings_json(d))
self.assertEqual(data['model_settings'], {
'first': 3,
'second': 'bar',
'third': 4.4,
})
def test_do_clean_is_true_and_bin_dir_is_specified___clean_bin_directory_is_called_on_the_correct_directory(
self, clean_mock):
with TemporaryDirectory() as d, responses.RequestsMock() as rsps:
bin_dir = os.path.join(d, 'alt_bin_dir')
os.mkdir(bin_dir)
Path(os.path.join(bin_dir, TAR_FILE)).touch()
rsps.add(responses.POST,
url='http://localhost:8001/exposure',
body=json.dumps({
'exposures': [{
'location': 'exposure_location'
}]
}).encode())
client = OasisAPIClient('http://localhost:8001')
client.upload_inputs_from_directory(d,
bin_directory=bin_dir,
do_clean=True)
clean_mock.assert_called_once_with(bin_dir)
