def test_records_are_given___records_are_written_to_oasis_keys_files_correctly(self, successes, nonsuccesses):
oasis_keys_file_to_record_metadict = {
'LocID': 'id',
'PerilID': 'peril_id',
'CoverageTypeID': 'coverage_type',
'AreaPerilID': 'area_peril_id',
'VulnerabilityID': 'vulnerability_id'
}
oasis_keys_errors_file_to_record_metadict = {
'LocID': 'id',
'PerilID': 'peril_id',
'CoverageTypeID': 'coverage_type',
'Message': 'message'
}
with TemporaryDirectory() as d:
keys_file_path = os.path.join(d, 'keys.csv')
keys_errors_file_path = os.path.join(d, 'keys-errors.csv')
_, successes_count = olf.write_oasis_keys_file(successes, keys_file_path)
_, nonsuccesses_count = olf.write_oasis_keys_errors_file(nonsuccesses, keys_errors_file_path)
with io.open(keys_file_path, 'r', encoding='utf-8') as f1, io.open(keys_errors_file_path, 'r', encoding='utf-8') as f2:
written_successes = [dict((oasis_keys_file_to_record_metadict[k], r[k]) for k in r) for r in pd.read_csv(f1).T.to_dict().values()]
written_nonsuccesses = [dict((oasis_keys_errors_file_to_record_metadict[k], r[k]) for k in r) for r in pd.read_csv(f2).T.to_dict().values()]
success_matches = list(filter(lambda r: (r['id'] == ws['id'] for ws in written_successes), successes))
nonsuccess_matches = list(filter(lambda r: (r['id'] == ws['id'] for ws in written_nonsuccesses), nonsuccesses))
self.assertEqual(successes_count, len(successes))
self.assertEqual(success_matches, successes)
self.assertEqual(nonsuccesses_count, len(nonsuccesses))
self.assertEqual(nonsuccess_matches, nonsuccesses)
def test_produced_keys_are_passed_to_write_oasis_keys_file(self, data):
write_oasis_keys_file_path = 'oasislmf.model_preparation.lookup.OasisLookupFactory.write_oasis_keys_file'
with TemporaryDirectory() as d, patch(write_oasis_keys_file_path) as write_oasis_keys_file_mock:
keys_file_path = os.path.join(d, 'piwind-keys.csv')
olf.save_keys(
keys_data=data,
keys_file_path=keys_file_path,
)
write_oasis_keys_file_mock.assert_called_once_with(data, keys_file_path, False)
def test_entries_are_dictionaries_success_only_is_false___all_entries_are_included(self, data):
with patch('oasislmf.model_preparation.lookup.OasisLookupFactory.get_exposure'):
self.create_fake_lookup(return_value=data)
res = list(olf.get_keys(lookup=self.lookup_instance, source_exposure_fp='path', success_only=False))
self.assertEqual(res, data)
def test_entries_are_dictionaries_success_only_is_true___only_successes_are_included(self, data):
with patch('oasislmf.model_preparation.lookup.OasisLookupFactory.get_exposure'):
self.create_fake_lookup(return_value=data)
res = list(olf.get_keys(lookup=self.lookup_instance, source_exposure_fp='path'))
self.assertEqual(res, [d for d in data if d['status'] == 'success'])
def test_complex_lookup_config_is_supplied___correct_instance_is_created_with_correct_config_path(self, supplier,
model, version,
complex_lookup_data):
with TemporaryDirectory() as d:
keys_path = os.path.join(d, 'keys')
os.mkdir(keys_path)
version_path = os.path.join(d, 'version.csv')
self.write_version_file(supplier, model, version, version_path)
module_path = os.path.join(d, '{}_lookup.py'.format(model))
self.write_py_module(model, module_path)
complex_lookup_config_path = os.path.join(d, 'lookup_config.json'.format(model))
self.write_complex_config_file(complex_lookup_data, complex_lookup_config_path)
output_directory = os.path.join(d, 'output')
os.mkdir(output_directory)
_, instance = olf.create(
model_keys_data_path=keys_path,
model_version_file_path=version_path,
lookup_module_path=module_path,
complex_lookup_config_fp=complex_lookup_config_path,
output_directory=output_directory
)
self.assertEqual(type(instance).__name__, '{}KeysLookup'.format(model))
self.assertEqual(instance.supplier, supplier)
self.assertEqual(instance.model_name, model)
self.assertEqual(instance.model_version, version)
self.assertEqual(instance.keys_data_directory, keys_path)
self.assertEqual(instance.complex_lookup_config_fp, complex_lookup_config_path)
self.assertEqual(instance.output_directory, output_directory)
def test_exposure_string_is_provided___file_content_is_loaded(self, data):
columns=['first', 'second']
exposure_str = _unicode(pd.DataFrame(columns=columns, data=data).to_csv(index=False))
res_str = olf.get_exposure(source_exposure=exposure_str).to_csv(index=False)
self.assertEqual(exposure_str, res_str)
def test_exposure_string_is_provided___file_content_is_loaded(self, data):
columns = loc_required_cols
exposure_str = pd.DataFrame(columns=columns, data=data).to_csv(index=False)
res_str = olf.get_exposure(source_exposure=exposure_str).to_csv(index=False)
self.assertEqual(exposure_str, res_str)
def test_produced_keys_are_passed_to_write_oasis_keys_file(self, data):
with TemporaryDirectory() as d,\
patch('oasislmf.model_preparation.lookup.OasisLookupFactory.get_keys', Mock(return_value=(r for r in data))) as get_keys_mock,\
patch('oasislmf.model_preparation.lookup.OasisLookupFactory.write_oasis_keys_file') as write_oasis_keys_file_mock:
keys_file_path = os.path.join(d, 'piwind-keys.csv')
olf.save_keys(lookup=self.create_fake_lookup(),
keys_file_path=keys_file_path,
source_exposure=json.dumps(data))
get_keys_mock.assert_called_once_with(
lookup=self.lookup_instance,
source_exposure=json.dumps(data),
source_exposure_fp=None,
success_only=True)
write_oasis_keys_file_mock.assert_called_once_with(
data, keys_file_path, id_col='locnumber')
def test_records_are_given___records_are_written_to_json_keys_files_correctly(self, successes, nonsuccesses):
with TemporaryDirectory() as d:
keys_file_path = os.path.join(d, 'keys.json')
keys_errors_file_path = os.path.join(d, 'keys-errors.json')
_, successes_count = olf.write_json_keys_file(successes, keys_file_path)
_, nonsuccesses_count = olf.write_json_keys_file(nonsuccesses, keys_errors_file_path)
with io.open(keys_file_path, 'r', encoding='utf-8') as f1, io.open(keys_errors_file_path, 'r',
encoding='utf-8') as f2:
written_successes = json.load(f1)
written_nonsuccesses = json.load(f2)
self.assertEqual(successes_count, len(successes))
self.assertEqual(written_successes, successes)
self.assertEqual(nonsuccesses_count, len(nonsuccesses))
self.assertEqual(written_nonsuccesses, nonsuccesses)
def test_lookup_package_path_not_supplied___correct_exception_raised(
self, supplier, model, version):
with TemporaryDirectory() as d:
version_path = os.path.join(d, 'version.csv')
self.write_version_file(supplier, model, version, version_path)
with self.assertRaisesRegexp(
OasisException,
r"Unable to import lookup package without lookup_package_path"
):
_, instance = olf.create(model_version_file_path=version_path)
def test_version_file_missing___correct_exception_raised(self, model):
with TemporaryDirectory() as d:
version_path = os.path.join(d, 'version.csv')
module_path = os.path.join(d, '{}_lookup.py'.format(model))
self.write_py_module(model, module_path)
with self.assertRaisesRegexp(
OasisException,
r"model_version_file_path does not exist.*"):
_, instance = olf.create(model_version_file_path=version_path)
def test_model_version_file_path_not_supplied___correct_exception_raised(
self, model):
with TemporaryDirectory() as d:
module_path = os.path.join(d, '{}_lookup.py'.format(model))
self.write_py_module(model, module_path)
with self.assertRaisesRegexp(
OasisException,
r"Unable to get model version data without model_version_file_path"
):
_, instance = olf.create(lookup_package_path=module_path)
def test_entries_are_dictionaries_success_only_is_false___all_entries_are_included(
self, data):
with patch(
'oasislmf.model_preparation.lookup.OasisLookupFactory.get_exposure'
):
self.create_fake_lookup(return_value=data)
mock_df = pd.DataFrame.from_dict(data)
res = list(
olf.get_keys_base(lookup=self.lookup_instance,
loc_df=mock_df,
success_only=False))
self.assertEqual(res, data)
def test_lookup_package_path_missing___correct_exception_raised(
self, supplier, model, version):
with TemporaryDirectory() as d:
version_path = os.path.join(d, 'version.csv')
self.write_version_file(supplier, model, version, version_path)
module_path = os.path.join(d, '{}_lookup.py'.format(model))
with self.assertRaisesRegex(
OasisException, r"lookup_package_path does not exist.*"):
_, instance = olf.create(model_version_file_path=version_path,
lookup_package_path=module_path)
def test_file_is_provided___file_content_is_loaded(self, data):
columns = ['first', 'second']
exposure_str = _unicode(pd.DataFrame(columns=columns, data=data).to_csv(index=False))
with NamedTemporaryFile('w') as f:
f.writelines(exposure_str)
f.flush()
res_str = olf.get_exposure(source_exposure_fp=f.name).to_csv(index=False)
self.assertEqual(exposure_str, res_str)
def test_source_exposure_are_provided___exposure_are_passed_to_get_model_exposure_result_is_passed_to_lookup_process_locations(
self, exposure, result):
with patch(
'oasislmf.model_preparation.lookup.OasisLookupFactory.get_exposure',
Mock(return_value=result)):
list(
olf.get_keys(self.create_fake_lookup(),
source_exposure=exposure))
olf.get_exposure.assert_called_once_with(source_exposure=exposure,
source_exposure_fp=None)
self.lookup_instance.process_locations.assert_called_once_with(
result)
def test_model_version_file_path_not_supplied___correct_exception_raised(self, model):
with TemporaryDirectory() as d:
keys_path = os.path.join(d, 'keys')
os.mkdir(keys_path)
module_path = os.path.join(d, '{}_lookup.py'.format(model))
self.write_py_module(model, module_path)
with self.assertRaisesRegex(OasisException,
r"The path None \(model_version_file_path\) is indicated as preexisting"
r" but is not a valid path"):
_, instance = olf.create(
model_keys_data_path=keys_path,
lookup_module_path=module_path
)
def test_file_is_provided___file_content_is_loaded(self, data):
columns = loc_required_cols
exposure_str = pd.DataFrame(columns=columns, data=data).to_csv(index=False)
f = NamedTemporaryFile('w', delete=False)
try:
f.writelines(exposure_str)
f.close()
res_str = olf.get_exposure(source_exposure_fp=f.name).to_csv(index=False)
self.assertEqual(exposure_str, res_str)
finally:
os.remove(f.name)
def test_model_keys_data_path_missing___correct_exception_raised(self, supplier, model, version):
with TemporaryDirectory() as d:
keys_path = os.path.join(d, 'keys')
version_path = os.path.join(d, 'version.csv')
self.write_version_file(supplier, model, version, version_path)
module_path = os.path.join(d, '{}_lookup.py'.format(model))
self.write_py_module(model, module_path)
with self.assertRaisesRegex(OasisException,
r"The path .*/keys \(model_keys_data_path\) is indicated as preexisting"
r" but does not exist"):
_, instance = olf.create(
model_version_file_path=version_path,
lookup_module_path=module_path,
model_keys_data_path=keys_path
)
def test_keys_path_not_supplied___correct_instance_is_created_with_correct_model_info_and_keys_path(self, supplier,
model, version):
with TemporaryDirectory() as d:
version_path = os.path.join(d, 'version.csv')
self.write_version_file(supplier, model, version, version_path)
module_path = os.path.join(d, '{}_lookup.py'.format(model))
self.write_py_module(model, module_path)
_, instance = olf.create(
model_version_file_path=version_path,
lookup_module_path=module_path
)
self.assertEqual(type(instance).__name__, '{}KeysLookup'.format(model))
self.assertEqual(instance.supplier, supplier)
self.assertEqual(instance.model_name, model)
self.assertEqual(instance.model_version, version)
self.assertEqual(instance.keys_data_directory, os.path.join(os.sep, 'var', 'oasis', 'keys_data'))
self.assertEqual(instance.complex_lookup_config_fp, None)
def test_keys_path_is_supplied___correct_instance_is_created_with_correct_model_info_and_keys_path(self, supplier, model, version):
with TemporaryDirectory() as d:
keys_path = os.path.join(d, 'keys')
os.mkdir(keys_path)
version_path = os.path.join(d, 'version.csv')
self.write_version_file(supplier, model, version, version_path)
module_path = os.path.join(d, '{}_lookup.py'.format(model))
self.write_py_module(model, module_path)
_, instance = olf.create(
model_keys_data_path=keys_path,
model_version_file_path=version_path,
lookup_package_path=module_path,
)
self.assertEqual(type(instance).__name__, '{}KeysLookup'.format(model))
self.assertEqual(instance.supplier, supplier)
self.assertEqual(instance.model_name, model)
self.assertEqual(instance.model_version, version)
self.assertEqual(instance.keys_data_directory, keys_path)
def test_no_file_or_exposure_are_provided___oasis_exception_is_raised(
self):
with self.assertRaises(OasisException):
olf.get_exposure()
def test_no_model_exposure_are_provided___oasis_exception_is_raised(self):
with self.assertRaises(OasisException):
list(olf.get_keys(self.create_fake_lookup()))
def test_no_file_or_exposure_are_provided___oasis_exception_is_raised(
self):
with self.assertRaises(OasisException):
OasisLookupFactory.get_model_exposure()
def test_entries_are_dictionaries_success_only_is_true___only_successes_are_included(self, data):
self.create_fake_lookup(return_value=data)
mock_df = pd.DataFrame.from_dict(data)
res = list(olf.get_keys_base(lookup=self.lookup_instance, loc_df=mock_df, success_only=True))
self.assertEqual(res, [d for d in data if d['status'] == 'success'])
def test_entries_are_dictionaries_success_only_is_false___all_entries_are_included(self, data):
self.create_fake_lookup(return_value=data)
mock_df = pd.DataFrame.from_dict(data)
res = list(olf.get_keys_base(lookup=self.lookup_instance, loc_df=mock_df, success_only=False))
self.assertEqual(res, data)