def test_input_location_is_not_a_tar___exception_is_raised(self):
with TemporaryDirectory() as inputs_dir:
with SettingsPatcher(INPUTS_DATA_DIRECTORY=inputs_dir):
Path(inputs_dir, 'not-tar-file.tar').touch()
self.assertRaises(InvalidInputsException, start_analysis, {},
'not-tar-file')
def test_analysis_settings_are_valid___celery_task_is_started(self, location, task_id):
with TemporaryDirectory() as inputs_dir, patch('src.server.app.CELERY') as celery_mock:
with SettingsPatcher(INPUTS_DATA_DIRECTORY=inputs_dir):
self.create_input_file('{}.tar'.format(location), 100)
analysis_settings = {
'analysis_settings': {
'module_supplier_id': 'supplier',
'model_version_id': 'version',
},
}
celery_mock.send_task = Mock(return_value=namedtuple('Task', ['task_id'])(task_id))
response = self.app.post(
'/analysis/{}'.format(location),
data=json.dumps(analysis_settings),
content_type='application/json',
)
celery_mock.send_task.assert_called_once_with(
'run_analysis',
(location, [analysis_settings]),
queue='supplier-version'
)
self.assertEqual(response._status_code, 200)
def test_do_clear_working_is_false___working_directory_is_not_removed_after_run(
self):
with TemporaryDirectory() as inputs_dir, \
TemporaryDirectory() as model_data_dir, \
TemporaryDirectory() as work_dir, \
TemporaryDirectory() as out_dir:
with SettingsPatcher(INPUTS_DATA_DIRECTORY=inputs_dir,
MODEL_DATA_DIRECTORY=model_data_dir,
WORKING_DIRECTORY=work_dir,
OUTPUTS_DATA_DIRECTORY=out_dir,
DO_CLEAr_WORKING='False'):
self.create_tar(str(Path(inputs_dir, 'location.tar')))
Path(model_data_dir, 'supplier', 'version').mkdir(parents=True)
with patch('src.model_execution_worker.tasks.runner'):
start_analysis(
{
'analysis_settings': {
'source_tag': 'source',
'analysis_tag': 'source',
'module_supplier_id': 'supplier',
'model_version_id': 'version'
}
},
'location',
)
self.assertGreater(len(os.listdir(work_dir)), 0)
def test_custom_model_runner_does_not_exist___default_runner_is_used(self):
with TemporaryDirectory() as inputs_dir, \
TemporaryDirectory() as model_data_dir, \
TemporaryDirectory() as work_dir, \
TemporaryDirectory() as out_dir:
with SettingsPatcher(INPUTS_DATA_DIRECTORY=inputs_dir,
MODEL_DATA_DIRECTORY=model_data_dir,
WORKING_DIRECTORY=work_dir,
OUTPUTS_DATA_DIRECTORY=out_dir):
self.create_tar(str(Path(inputs_dir, 'location.tar')))
Path(model_data_dir, 'supplier', 'version').mkdir(parents=True)
with patch('src.model_execution_worker.tasks.runner'
) as default_mock:
start_analysis(
{
'analysis_settings': {
'source_tag': 'source',
'analysis_tag': 'source',
'module_supplier_id': 'supplier',
'model_version_id': 'version'
}
},
'location',
)
default_mock.run.assert_called_once_with(
{
'source_tag': 'source',
'analysis_tag': 'source',
'module_supplier_id': 'supplier',
'model_version_id': 'version'
}, settings.getint('worker', 'KTOOLS_BATCH_COUNT'))
def test_no_data_is_provided___response_is_400(self):
with TemporaryDirectory() as inputs_dir:
with SettingsPatcher(INPUTS_DATA_DIRECTORY=inputs_dir):
self.create_input_file('test.tar', 100)
response = self.app.post('/analysis/test')
self.assertEqual(response._status_code, 400)
def test_location_exists___response_contains_tar_data(self, location, data):
with TemporaryDirectory() as inputs_dir:
with SettingsPatcher(INPUTS_DATA_DIRECTORY=inputs_dir):
self.create_input_file('{}.tar'.format(location), data=data)
response = self.app.get("/exposure/{}".format(location))
self.assertEqual(response._status_code, 200)
self.assertEqual(response.data, data)
def test_get_exposure_summary_for_a_particular_location_that_does_not_exist___exposures_are_empty(self, location, size):
with TemporaryDirectory() as inputs_dir:
with SettingsPatcher(INPUTS_DATA_DIRECTORY=inputs_dir):
self.create_input_file('other_{}.tar'.format(location), size)
response = self.app.get("/exposure_summary/{}".format(location))
self.assertEqual(response._status_code, 404)
def test_no_location_is_given_on_delete___all_outputs_are_deleted(self):
with TemporaryDirectory() as d:
with SettingsPatcher(OUTPUTS_DATA_DIRECTORY=d):
Path(os.path.join(d, 'test1.tar')).touch()
Path(os.path.join(d, 'test2.tar')).touch()
response = self.app.delete("/outputs")
self.assertEqual(response._status_code, 200)
self.assertEqual(len(os.listdir(d)), 0)
def test_file_exists___content_is_correct(self, location, data):
with TemporaryDirectory() as d:
with SettingsPatcher(OUTPUTS_DATA_DIRECTORY=d):
with open(os.path.join(d, '{}.tar'.format(location)), 'wb') as f:
f.write(data)
response = self.app.get('/outputs/{}'.format(location))
self.assertEqual(response._status_code, 200)
self.assertEqual(response.data, data)
def test_location_does_not_exist___response_is_404(self, locations):
first_location, second_location = locations
with TemporaryDirectory() as d:
with SettingsPatcher(OUTPUTS_DATA_DIRECTORY=d):
Path(os.path.join(d, '{}.tar'.format(first_location))).touch()
response = self.app.delete("/outputs/{}".format(second_location))
self.assertEqual(response._status_code, 404)
self.assertEqual(os.listdir(d), ['{}.tar'.format(first_location)])
def test_module_version_id_is_missing_from_post_data___response_is_400(self):
with TemporaryDirectory() as inputs_dir:
with SettingsPatcher(INPUTS_DATA_DIRECTORY=inputs_dir):
self.create_input_file('test.tar', 100)
response = self.app.post(
'/analysis/test',
data=json.dumps({'analysis_settings': {'module_supplier_id': 'supplier'}}),
content_type='application/json',
)
self.assertEqual(response._status_code, 400)
def test_get_exposure_summary_for_a_particular_location___correct_location_name_and_size_are_returned(self, location, size):
with TemporaryDirectory() as inputs_dir:
with SettingsPatcher(INPUTS_DATA_DIRECTORY=inputs_dir):
self.create_input_file('{}.tar'.format(location), size)
response = self.app.get("/exposure_summary/{}".format(location))
self.assertEqual(response._status_code, 200)
exposures = json.loads(response.data.decode('utf-8'))['exposures']
self.assertEqual(len(exposures), 1)
self.assertEqual(exposures[0]['location'], location)
self.assertEqual(exposures[0]['size'], size)
def test_no_location_is_given_on_delete___all_exposures_are_deleted(self):
with TemporaryDirectory() as inputs_dir:
with SettingsPatcher(INPUTS_DATA_DIRECTORY=inputs_dir):
Path(os.path.join(inputs_dir, 'test1.tar')).touch()
Path(os.path.join(inputs_dir, 'test2.tar')).touch()
self.app.delete("/exposure")
response = self.app.get("/exposure_summary")
exposures = json.loads(response.data.decode('utf-8'))['exposures']
self.assertEqual(response._status_code, 200)
self.assertEqual(len(exposures), 0)
def test_no_location_is_given_on_delete___non_tar_files_and_child_dirs_are_left(self):
with TemporaryDirectory() as d:
with SettingsPatcher(OUTPUTS_DATA_DIRECTORY=d):
Path(os.path.join(d, 'test1.tar')).touch()
Path(os.path.join(d, 'test2.tar')).touch()
Path(os.path.join(d, 'test.nottar')).touch()
os.makedirs(os.path.join(d, 'child.tar'))
Path(os.path.join(d, 'child.tar', 'test3.tar')).touch()
self.app.delete("/outputs")
self.assertTrue(os.path.exists(os.path.join(d, 'test.nottar')))
self.assertTrue(os.path.exists(os.path.join(d, 'child.tar', 'test3.tar')))
def test_location_does_not_exist_on_delete___response_is_404(self, locations):
first_location, second_location = locations
with TemporaryDirectory() as inputs_dir:
with SettingsPatcher(INPUTS_DATA_DIRECTORY=inputs_dir):
Path(os.path.join(inputs_dir, '{}.tar'.format(first_location))).touch()
response = self.app.delete("/exposure/{}".format(second_location))
self.assertEqual(response._status_code, 404)
response = self.app.get("/exposure_summary")
exposures = json.loads(response.data.decode('utf-8'))['exposures']
self.assertEqual(len(exposures), 1)
self.assertEqual(first_location, exposures[0]['location'])
def test_exposure_file_is_posted___exposure_is_stored_in_inputs_directory(self, data):
with TemporaryDirectory() as inputs_dir, TemporaryDirectory() as upload_directory:
with SettingsPatcher(INPUTS_DATA_DIRECTORY=inputs_dir):
filepath = os.path.join(upload_directory, 'file.tar')
self.create_input_file(filepath, data=data)
with io.open(filepath, 'rb') as file_to_upload:
response = self.app.post(
"/exposure",
data={
'file': (file_to_upload, filepath),
},
content_type='multipart/form-data'
)
location = json.loads(response.data.decode('utf-8'))['exposures'][0]['location']
response = self.app.get("/exposure/{}".format(location))
self.assertEqual(response._status_code, 200)
self.assertEqual(response.data, data)
def test_input_model_data_does_not_exist___exception_is_raised(self):
with TemporaryDirectory() as inputs_dir, TemporaryDirectory(
) as model_data_dir:
with SettingsPatcher(INPUTS_DATA_DIRECTORY=inputs_dir,
MODEL_DATA_DIRECTORY=model_data_dir):
self.create_tar(str(Path(inputs_dir, 'location.tar')))
self.assertRaises(
MissingModelDataException,
start_analysis,
{
'analysis_settings': {
'source_tag': 'source',
'analysis_tag': 'source',
'module_supplier_id': 'supplier',
'model_version_id': 'version'
}
},
'location',
)
def test_several_exposures_exist___all_exposure_names_and_sizes_are_returned(self):
with TemporaryDirectory() as inputs_dir:
with SettingsPatcher(INPUTS_DATA_DIRECTORY=inputs_dir):
self.create_input_file('test1.tar', 100)
self.create_input_file('test2.tar', 200)
self.create_input_file('test3.tar', 300)
self.create_input_file('test4.tar', 400)
response = self.app.get("/exposure_summary")
self.assertEqual(response._status_code, 200)
exposures = json.loads(response.data.decode('utf-8'))['exposures']
self.assertEqual(len(exposures), 4)
self.assertEqual(exposures[0]['location'], 'test1')
self.assertEqual(exposures[0]['size'], 100)
self.assertEqual(exposures[1]['location'], 'test2')
self.assertEqual(exposures[1]['size'], 200)
self.assertEqual(exposures[2]['location'], 'test3')
self.assertEqual(exposures[2]['size'], 300)
self.assertEqual(exposures[3]['location'], 'test4')
self.assertEqual(exposures[3]['size'], 400)
def test_custom_model_runner_exists___custom_runner_is_used(self):
with TemporaryDirectory() as inputs_dir, \
TemporaryDirectory() as model_data_dir, \
TemporaryDirectory() as work_dir, \
TemporaryDirectory() as out_dir, \
TemporaryDirectory() as module_dir:
with SettingsPatcher(INPUTS_DATA_DIRECTORY=inputs_dir,
MODEL_DATA_DIRECTORY=model_data_dir,
WORKING_DIRECTORY=work_dir,
OUTPUTS_DATA_DIRECTORY=out_dir,
SUPPLIER_MODULE_DIRECTORY=module_dir):
self.create_tar(str(Path(inputs_dir, 'location.tar')))
Path(model_data_dir, 'supplier', 'version').mkdir(parents=True)
Path(module_dir, 'supplier').mkdir()
Path(module_dir, 'supplier', '__init__.py').touch()
with open(
str(
Path(module_dir, 'supplier').joinpath(
'supplier_model_runner.py')), 'w') as module:
module.writelines([
'from pathlib2 import Path\n',
'def run(settings, location):\n',
' Path("{}", "custom_model").touch()\n'.format(
out_dir)
])
start_analysis(
{
'analysis_settings': {
'source_tag': 'source',
'analysis_tag': 'source',
'module_supplier_id': 'supplier',
'model_version_id': 'version'
}
},
'location',
)
self.assertTrue(Path(out_dir, "custom_model").exists())
def test_file_does_not_exist___response_is_404(self):
with TemporaryDirectory() as d:
with SettingsPatcher(OUTPUTS_DATA_DIRECTORY=d):
response = self.app.get('/outputs/somefile')
self.assertEqual(response._status_code, 404)
def test_input_tar_file_does_not_exist___exception_is_raised(self):
with TemporaryDirectory() as inputs_dir:
with SettingsPatcher(INPUTS_DATA_DIRECTORY=inputs_dir):
self.assertRaises(MissingInputsException, start_analysis, {},
'non-existant-location')
def test_location_does_not_exist___response_is_404(self):
with TemporaryDirectory() as inputs_dir:
with SettingsPatcher(INPUTS_DATA_DIRECTORY=inputs_dir):
Path(os.path.join(inputs_dir, 'test1.tar')).touch()
response = self.app.get("/exposure/test2")
self.assertEqual(response._status_code, 404)
