def test_tar_file_exists_in_specified_bin_dir___correct_file_is_posted(
self):
with TemporaryDirectory() as d, responses.RequestsMock() as rsps:
bin_dir = os.path.join(d, 'alt_bin_dir')
os.mkdir(bin_dir)
Path(os.path.join(bin_dir, TAR_FILE)).touch()
rsps.add(responses.POST,
url='http://localhost:8001/exposure',
body=json.dumps({
'exposures': [{
'location': 'exposure_location'
}]
}).encode())
client = OasisAPIClient('http://localhost:8001')
client.upload_inputs_from_directory(d, bin_directory=bin_dir)
self.assertEqual(1, len(rsps.calls))
request = rsps.calls[0].request
self.assertEqual(request.url, 'http://localhost:8001/exposure')
multipart_data = request.body.fields['file']
self.assertEqual('inputs.tar.gz', multipart_data[0])
self.assertEqual(os.path.join(bin_dir, TAR_FILE),
multipart_data[1].name)
self.assertEqual('text/plain', multipart_data[2])
def test_request_response_is_ok___response_location_is_returned(self):
client = OasisAPIClient('http://localhost:8001')
with responses.RequestsMock() as rsps:
rsps.add(responses.POST, 'http://localhost:8001/analysis/foo', status=200, body=b'{"location": "output-location"}')
self.assertEqual('output-location', client.run_analysis({'analysis': 'data'}, 'foo'))
def test_no_errors_are_raised___completed_is_incremented(
self, input_dir, output_dir, settings, do_il, do_ri,
initial_complete, initial_failed):
client = OasisAPIClient('http://localhost:8001')
client.upload_inputs_from_directory = Mock(
return_value='input_location')
client.run_analysis_and_poll = Mock()
counter = Counter({
'completed': initial_complete,
'failed': initial_failed,
})
TestModelApiCmd().run_analysis(
(client, input_dir, output_dir, settings, do_il, do_ri, counter,
[], 1))
self.assertEqual(initial_complete + 1, counter['completed'])
self.assertEqual(initial_failed, counter['failed'])
client.upload_inputs_from_directory.assert_called_once_with(
input_dir,
bin_directory=ANY,
do_il=do_il,
do_ri=do_ri,
do_build=True)
client.run_analysis_and_poll.assert_called_once_with(
settings, 'input_location', output_dir)
def test_delete_resource_is_called_with_the_correct_parameters(self):
client = OasisAPIClient('http://localhost:8001')
client.delete_resource = Mock()
client.delete_exposure('foo')
client.delete_resource.assert_called_once_with('/exposure/foo')
def test_response_is_ok___file_is_created_with_correct_content(self):
client = OasisAPIClient('http://localhost:8001')
client.DOWNLOAD_CHUCK_SIZE_IN_BYTES = 10
expected_content = ''.join(
choice(string.ascii_letters)
for i in range(2 * client.DOWNLOAD_CHUCK_SIZE_IN_BYTES)).encode()
with NamedTemporaryFile('w') as f:
local_filename = f.name
with responses.RequestsMock() as rsps:
rsps.add(responses.GET,
'http://localhost:8001/foo',
status=200,
body=expected_content)
client.download_resource('foo', local_filename)
self.assertTrue(os.path.exists(local_filename))
with io.open(local_filename, 'rb') as f:
dld_content = f.read()
os.remove(local_filename)
self.assertEqual(expected_content, dld_content)
def test_download_resource_is_called_with_the_correct_parameters(self):
client = OasisAPIClient('http://localhost:8001')
client.download_resource = Mock()
client.download_outputs('foo', 'local_filename')
client.download_resource.assert_called_once_with('/outputs/foo', 'local_filename')
def test_do_build_is_truebin_dir_is_supplied___bin_building_functions_are_called_with_correct_args(
self, check_mock, check_tools_mock, create_bin_mock,
create_tar_mock):
create_tar_mock.side_effect = fake_build_tar_fn
with TemporaryDirectory() as d, responses.RequestsMock() as rsps:
bin_dir = os.path.join(d, 'alt_bin_dir')
os.mkdir(bin_dir)
rsps.add(responses.POST,
url='http://localhost:8001/exposure',
body=json.dumps({
'exposures': [{
'location': 'exposure_location'
}]
}).encode())
client = OasisAPIClient('http://localhost:8001')
client.upload_inputs_from_directory(d,
bin_directory=bin_dir,
do_build=True)
check_mock.assert_called_once_with(d, do_il=False)
check_tools_mock.assert_called_once_with(do_il=False)
create_bin_mock.assert_called_once_with(d, bin_dir, do_il=False)
create_tar_mock.assert_called_once_with(bin_dir)
def test_request_is_not_ok___exception_is_raised(self):
client = OasisAPIClient('http://localhost:8001')
with responses.RequestsMock() as rsps:
rsps.add(responses.GET, 'http://localhost:8001/analysis_status/foo', status=400)
with self.assertRaises(OasisException):
client.get_analysis_status('foo')
def test_response_is_ok___info_is_logged(self):
client = OasisAPIClient('http://localhost:8001', Mock())
with responses.RequestsMock() as rsps:
rsps.add(responses.DELETE, 'http://localhost:8001/foo', status=200)
client.delete_resource('foo')
client._logger.info.assert_called_with('Deleted http://localhost:8001/foo')
def test_response_is_not_ok___warning_is_logged(self):
client = OasisAPIClient('http://localhost:8001', Mock())
with responses.RequestsMock() as rsps:
rsps.add(responses.DELETE, 'http://localhost:8001/foo', status=400)
client.delete_resource('foo')
client._logger.warning.assert_called_with("DELETE http://localhost:8001/foo failed: 400")
def test_tar_file_exists___exposure_location_from_result_is_returned(self):
with TemporaryDirectory() as d, responses.RequestsMock() as rsps:
Path(os.path.join(d, TAR_FILE)).touch()
rsps.add(responses.POST, url='http://localhost:8001/exposure', body=json.dumps({'exposures': [{'location': 'exposure_location'}]}).encode())
client = OasisAPIClient('http://localhost:8001')
result = client.upload_inputs_from_directory(d)
self.assertEqual('exposure_location', result)
def test_do_clean_is_true___clean_bin_directory_is_called_on_the_correct_directory(self, clean_mock):
with TemporaryDirectory() as d, responses.RequestsMock() as rsps:
Path(os.path.join(d, TAR_FILE)).touch()
rsps.add(responses.POST, url='http://localhost:8001/exposure', body=json.dumps({'exposures': [{'location': 'exposure_location'}]}).encode())
client = OasisAPIClient('http://localhost:8001')
client.upload_inputs_from_directory(d, do_clean=True)
clean_mock.assert_called_once_with(d)
def test_response_from_server_is_not_ok___oasis_error_is_raised(self):
with TemporaryDirectory() as d, responses.RequestsMock() as rsps:
Path(os.path.join(d, TAR_FILE)).touch()
rsps.add(responses.POST, status=400, url='http://localhost:8001/exposure', body=json.dumps({'exposures': [{'location': 'exposure_location'}]}).encode())
client = OasisAPIClient('http://localhost:8001')
with self.assertRaises(OasisException):
client.upload_inputs_from_directory(d)
def test_heath_check_returns_200___result_is_true(self):
with responses.RequestsMock() as rsps:
rsps.add(responses.GET, 'http://localhost:8001/healthcheck', status=200)
client = OasisAPIClient('http://localhost:8001')
result = client.health_check()
self.assertTrue(result)
self.assertEqual(1, len(rsps.calls))
self.assertEqual('http://localhost:8001/healthcheck', rsps.calls[0].request.url)
def test_local_file_already_exists___exception_is_raised_and_file_is_unchanged(self):
client = OasisAPIClient('http://localhost:8001')
with NamedTemporaryFile('w+') as f:
f.write('foobarboo')
f.flush()
with self.assertRaises(OasisException):
client.download_resource('foo', f.name)
f.seek(0)
self.assertEqual('foobarboo', f.read())
def test_request_response_is_not_ok___exception_is_raised(self):
client = OasisAPIClient('http://localhost:8001')
with responses.RequestsMock() as rsps:
rsps.add(responses.POST, 'http://localhost:8001/analysis/foo', status=400)
with self.assertRaises(OasisException):
client.run_analysis({'analysis': 'data'}, 'foo')
self.assertEqual(1, len(rsps.calls))
self.assertEqual('http://localhost:8001/analysis/foo', rsps.calls[0].request.url)
self.assertEqual({'analysis': 'data'}, json.loads(rsps.calls[0].request.body.decode()))
def test_do_build_is_false___bin_building_functions_are_not_called(self, check_mock, check_tools_mock, create_bin_mock, create_tar_mock):
with TemporaryDirectory() as d, responses.RequestsMock() as rsps:
Path(os.path.join(d, TAR_FILE)).touch()
rsps.add(responses.POST, url='http://localhost:8001/exposure', body=json.dumps({'exposures': [{'location': 'exposure_location'}]}).encode())
client = OasisAPIClient('http://localhost:8001')
client.upload_inputs_from_directory(d, do_build=False)
check_mock.assert_not_called()
check_tools_mock.assert_not_called()
create_bin_mock.assert_not_called()
create_tar_mock.assert_not_called()
def test_request_status_is_failure___exception_is_raised(self):
client = OasisAPIClient('http://localhost:8001')
with responses.RequestsMock() as rsps:
rsps.add(
responses.GET,
'http://localhost:8001/analysis_status/foo',
status=200,
body=json.dumps({'status': STATUS_FAILURE, 'message': 'oops'}).encode()
)
with self.assertRaises(OasisException):
client.get_analysis_status('foo')
def test_response_is_not_ok___exception_is_raised_and_file_is_not_created(self):
client = OasisAPIClient('http://localhost:8001')
with NamedTemporaryFile('w') as f:
local_filename = f.name
with responses.RequestsMock() as rsps:
rsps.add(responses.GET, 'http://localhost:8001/foo', status=400)
with self.assertRaises(OasisException):
client.download_resource('foo', local_filename)
self.assertFalse(os.path.exists(local_filename))
def test_heath_check_raise_an_exception_on_each_call___result_is_false(self, max_attempts):
with patch('requests.get', Mock(side_effect=RequestException())) as get_mock:
client = OasisAPIClient('http://localhost:8001')
result = client.health_check(max_attempts, retry_delay=0)
self.assertFalse(result)
self.assertEqual(max_attempts, get_mock.call_count)
call_urls = [args[0] for args in get_mock.call_args_list]
self.assertEqual(
[('http://localhost:8001/healthcheck',) for i in range(max_attempts)],
call_urls,
)
def test_heath_check_returns_non_200_on_each_call___result_is_false(self, max_attempts):
with responses.RequestsMock() as rsps:
rsps.add(responses.GET, 'http://localhost:8001/healthcheck', status=404)
client = OasisAPIClient('http://localhost:8001')
result = client.health_check(max_attempts, retry_delay=0)
self.assertFalse(result)
self.assertEqual(max_attempts, len(rsps.calls))
self.assertEqual(
['http://localhost:8001/healthcheck' for i in range(max_attempts)],
[call.request.url for call in rsps.calls],
)
def test_run_and_poll_raises_an_error___failed_counter_is_incremented(self, input_dir, output_dir, settings, do_il, initial_complete, initial_failed):
client = OasisAPIClient('http://localhost:8001')
client.upload_inputs_from_directory = Mock()
client.run_analysis_and_poll = Mock(side_effect=OasisException())
counter = Counter({
'completed': initial_complete,
'failed': initial_failed,
})
TestModelApiCmd().run_analysis((client, input_dir, output_dir, settings, do_il, counter))
self.assertEqual(initial_complete, counter['completed'])
self.assertEqual(initial_failed + 1, counter['failed'])
def test_request_status_is_pending___result_status_is_pending_location_is_empty(self):
client = OasisAPIClient('http://localhost:8001')
with responses.RequestsMock() as rsps:
rsps.add(
responses.GET,
'http://localhost:8001/analysis_status/foo',
status=200,
body=json.dumps({'status': STATUS_PENDING}).encode()
)
status, outputs_location = client.get_analysis_status('foo')
self.assertEqual(status, STATUS_PENDING)
self.assertEqual(outputs_location, '')
def test_request_status_is_success___result_status_is_success_location_is_from_response_body(self):
client = OasisAPIClient('http://localhost:8001')
with responses.RequestsMock() as rsps:
rsps.add(
responses.GET,
'http://localhost:8001/analysis_status/foo',
status=200,
body=json.dumps({'status': STATUS_SUCCESS, 'outputs_location': 'outputs-location'}).encode()
)
status, outputs_location = client.get_analysis_status('foo')
self.assertEqual(status, STATUS_SUCCESS)
self.assertEqual(outputs_location, 'outputs-location')
def get_mocked_client(self):
client = OasisAPIClient('http://localhost:8001')
client.run_analysis = Mock(return_value=self.analysis_status_location)
client.get_analysis_status = Mock(return_value=(STATUS_SUCCESS, self.analysis_output_location))
client.download_outputs = Mock()
client.delete_exposure = Mock()
client.delete_outputs = Mock()
return client
def run_analysis(c):
try:
upload_directory = os.path.join("upload", str(uuid.uuid1()))
shutil.copytree(os.path.join(input_data_directory, "csv"),
upload_directory)
client = OasisAPIClient(api_url, logging.getLogger())
input_location = client.upload_inputs_from_directory(
upload_directory, do_il, do_validation=False)
client.run_analysis(analysis_settings,
input_location,
output_data_directory,
do_clean=False)
c.increment_num_completed()
except Exception:
logging.exception("API test failed")
c.increment_num_failed()
def do_run_prog_oasis(processrunid):
'''
Run a programme model combination.
'''
element_run_ids = list()
element_run_id = -1
flamingo_db_utils.update_process_run_status(processrunid, "In Progress")
try:
base_url = flamingo_db_utils.get_base_url(processrunid)
element_run_ids = \
flamingo_db_utils.get_element_run_ids(processrunid)
upload_directory = generate_summary_files(processrunid)
logging.getLogger().debug(
"Upload_directory: {}".format(upload_directory))
analysis_settings_json = get_analysis_settings_json(processrunid)
logging.getLogger().debug(analysis_settings_json)
if 'il_output' in analysis_settings_json['analysis_settings']:
create_il_bins = analysis_settings_json['analysis_settings'][
'il_output']
else:
create_il_bins = False
analysis_poll_interval_in_seconds = 5
client = OasisAPIClient(base_url, logging.getLogger())
element_run_id = element_run_ids[0][0]
input_location = client.upload_inputs_from_directory(
upload_directory,
do_il=create_il_bins,
do_build=True,
do_clean=True)
logging.getLogger().info("Input location: {}".format(input_location))
flamingo_db_utils.log_element_run_to_db(
element_run_id, "Success",
"Exposure files location: {}".format(input_location))
element_run_id = element_run_ids[1][0]
analysis_status_location = client.run_analysis(analysis_settings_json,
input_location)
flamingo_db_utils.log_element_run_to_db(element_run_id, "Success",
"Started analysis")
element_run_id = element_run_ids[2][0]
outputs_location = ""
while True:
logging.getLogger().debug("Polling analysis status for: {}".format(
analysis_status_location))
(status, outputs_location) = \
client.get_analysis_status(analysis_status_location)
flamingo_db_utils.log_element_run_to_db(element_run_id, status,
"In Progress")
if status == status_code.STATUS_SUCCESS:
if outputs_location is None:
raise Exception("Complete but no outputs location")
flamingo_db_utils.log_element_run_to_db(
element_run_id, status, "Analysis Completed")
break
elif status == status_code.STATUS_FAILURE:
error_message = "Analysis failed: {}".format(message)
logging.getLogger().error(error_message)
raise Exception(error_message)
time.sleep(analysis_poll_interval_in_seconds)
element_run_id = element_run_ids[3][0]
# download outputs and cleanup
outputs_file = os.path.join(upload_directory,
outputs_location + ".tar.gz")
client.download_outputs(outputs_location, outputs_file)
client.delete_exposure(input_location)
client.delete_outputs(outputs_location)
flamingo_db_utils.log_element_run_to_db(
element_run_id, 'Success', 'Downloaded output files successfully')
extract_tarball(
os.path.join(upload_directory, outputs_location + ".tar.gz"),
upload_directory)
output_file_list = ','.join(
map(str, os.listdir(os.path.join(upload_directory, "output"))))
logging.getLogger().debug(
"Output_file_list: {}".format(output_file_list))
db.execute(
"exec dbo.linkOutputFileToProcessRun @ProcessRunId = ?, @OutputFiles = ?",
processrunid, output_file_list)
flamingo_db_utils.update_process_run_status(processrunid, 'Completed')
# append summary id meanings to output files
output_file_details = flamingo_db_utils.get_output_file_details(
processrunid)
logging.getLogger().debug(
"output_file_details: {}".format(output_file_details))
columns = [
"FileName", "FileDesc", "PerspectiveName", "OutputID", "LECFlag",
"AnalysisFileNameStub", "SummaryLevelName"
]
df_output_file_details = pd.DataFrame(columns=columns)
recs = map(lambda tup: dict(zip(columns, list(tup))),
output_file_details)
df_output_file_details = df_output_file_details.append(recs)
logging.getLogger().debug(
"df_output_file_details:\n{}".format(df_output_file_details))
prog_oasis_location = flamingo_db_utils.get_prog_oasis_location(
processrunid)
itemdict = prog_oasis_location + '/ItemDict.csv'
fmdict = prog_oasis_location + '/FMDict.csv'
df_itemdict = pd.read_csv(itemdict)
df_fmdict = pd.read_csv(fmdict)
df_fmdict["policy_layer"] = df_fmdict["policy_name"].map(
str) + '--' + df_fmdict["layer_name"].map(str)
for index, row in df_output_file_details.iterrows():
output = upload_directory + '/output/' + row['FileName']
logging.getLogger().debug("FileName: {}".format(output))
output_tmp = output + '.tmp'
SummaryLevelName = row['SummaryLevelName']
df_output = pd.read_csv(output)
SummaryLevelId = SummaryLevelName.lower() + '_id'
SummaryLevelDesc = SummaryLevelName.lower() + '_desc'
logging.getLogger().debug(
"SummaryLevelName: {}".format(SummaryLevelName))
if SummaryLevelName != "Portfolio":
if SummaryLevelName == "Policy":
# join fmdict to file
df_summarydict = df_fmdict[['agg_id', 'policy_layer']]
logging.getLogger().debug(
"df_summarydict: {}".format(df_summarydict))
df_summarydict_distinct = df_summarydict.drop_duplicates()
df_output_temp = df_output.join(
df_summarydict_distinct.set_index('agg_id'),
on='summary_id')
else:
# join itemdict to file
df_summarydict = df_itemdict[[
SummaryLevelId, SummaryLevelDesc
]]
df_summarydict_distinct = df_summarydict.drop_duplicates()
df_output_temp = df_output.join(
df_summarydict_distinct.set_index(SummaryLevelId),
on='summary_id')
logging.getLogger().debug(
"df_summarydict_distinct: {}".format(
df_summarydict_distinct))
df_output_temp.to_csv(output, encoding='utf-8', index=False)
except Exception as e:
flamingo_db_utils.update_process_run_status(processrunid, "Failed")
if element_run_id != -1:
flamingo_db_utils.log_element_run_to_db(element_run_id, 'Failed: ',
str(e))
logging.getLogger().exception(
"Failed to run prog oasis: {}".format(processrunid))