def test_send_reduction_message_(self, mock_log_debug, mock_log_info):
"""
Test: reduction status message has been sent and logged
When: called within reduce method
"""
amq_messages = [
attr for attr in dir(ActiveMQSettings)
if not callable(getattr(ActiveMQSettings, attr))
and attr.startswith("reduction")
]
amq_client_mock = Mock()
ppa = PostProcessAdmin(self.message, amq_client_mock)
for message in amq_messages:
amq_message = getattr(ACTIVEMQ_SETTINGS, message)
ppa.send_reduction_message(message="status",
amq_message=amq_message)
mock_log_debug.assert_called_with(
"Calling: %s\n%s", amq_message,
self.message.serialize(limit_reduction_script=True))
amq_client_mock.send.assert_called_with(amq_message, ppa.message)
self.assertEqual(len(amq_messages), amq_client_mock.send.call_count)
mock_log_info.assert_called_with("Reduction: %s", 'status')
def test_result_and_log_directory(self, mock_nrdp, mock_logging):
"""
Test: final result and log directories are returned
When: called with temp root directory, result and log locations
"""
ppa = PostProcessAdmin(self.message, None)
instrument_output_dir = MISC["ceph_directory"] % (
ppa.instrument, ppa.proposal, ppa.run_number)
mock_nrdp.return_value = append_path(instrument_output_dir, "0")
instrument_output_directory = instrument_output_dir[:
instrument_output_dir
.rfind('/') + 1]
reduce_directory = MISC[
"temp_root_directory"] + instrument_output_directory
reduction_log = "/reduction_log/"
actual_final_result, actual_log = ppa.create_final_result_and_log_directory(
temporary_root_directory=MISC["temp_root_directory"],
reduce_dir=reduce_directory)
expected_log = f"{instrument_output_directory}0{reduction_log}"
expected_logs_called_with = [
call("Final Result Directory = %s", actual_final_result),
call("Final log directory: %s", actual_log)
]
mock_nrdp.assert_called_once_with(instrument_output_dir)
self.assertEqual(mock_logging.call_count, 2)
self.assertEqual(mock_logging.call_args_list,
expected_logs_called_with)
self.assertEqual(expected_log, actual_log)
def test_validate_reduction_as_module_exp(self, mock_sid, mock_ram,
mock_ctd, mock_dtd, *_):
"""
Test: Exception is called and temporary dir path handling takes place
When: Reduction fails to load as module
"""
mock_ram.side_effect = Exception('warning')
log_dir = mock_sid + "/reduction_log"
error_str = f"Error in user reduction script: {type(Exception('warning')).__name__} - " \
f"{Exception('warning')}"
ppa = PostProcessAdmin(self.message, None)
script_out = os.path.join(log_dir,
f"{self.log_and_err_name}Script.out")
mantid_log = os.path.join(log_dir,
f"{self.log_and_err_name}Mantid.log")
instrument_output_directory = self.ceph_directory[:self.ceph_directory.
rfind('/') + 1]
final_result_dir = instrument_output_directory + "0/"
actual = ppa.validate_reduction_as_module(
script_out=script_out,
mantid_log=mantid_log,
reduce_result=self.reduce_result_dir,
final_result=final_result_dir)
self.assertEqual(open(script_out).read(), "data")
mock_ram.assert_called_with(self.reduce_result_dir)
mock_ctd.assert_called()
mock_dtd.assert_called()
self.assertEqual(str(Exception(error_str)), str(actual))
def test_create_final_result_and_log_directory_non_flat_output(self, mock_append, mock_logging):
"""
Test: final result and log directories are returned
When: called with temp root directory, result and log locations
"""
self.message.instrument = "LARMOR"
ppa = PostProcessAdmin(self.message, None)
instrument_output_directory = MISC["ceph_directory"] % (ppa.instrument,
ppa.proposal,
ppa.run_number)
reduce_directory = MISC["temp_root_directory"] + instrument_output_directory
reduction_log = "/reduction_log/"
mock_append.return_value = instrument_output_directory + "/run-version-0/"
actual_final_result, actual_log = ppa.create_final_result_and_log_directory(
temporary_root_directory=MISC["temp_root_directory"],
reduce_dir=reduce_directory)
expected_log = f"{instrument_output_directory}/run-version-0{reduction_log}"
expected_logs_called_with = [call("Final Result Directory = %s", actual_final_result),
call("Final log directory: %s", actual_log)]
mock_append.assert_called_once_with(instrument_output_directory)
self.assertEqual(mock_logging.call_count, 2)
self.assertEqual(mock_logging.call_args_list, expected_logs_called_with)
self.assertEqual(expected_log, actual_log)
def test_validate_reduction_as_module(self, mock_sid, mock_ram, mock_ctd,
mock_dtd, _):
"""
Test: reduce_result returned
When: Called with correct arguments and no error produced by reduction_as_module()
"""
ppa = PostProcessAdmin(self.message, None)
mock_ram.return_value = True
log_dir = mock_sid + "/reduction_log"
script_out = os.path.join(log_dir,
f"{self.log_and_err_name}Script.out")
mantid_log = os.path.join(log_dir,
f"{self.log_and_err_name}Mantid.log")
instrument_output_directory = self.ceph_directory[:self.ceph_directory.
rfind('/') + 1]
final_result_dir = instrument_output_directory + "0/"
actual = ppa.validate_reduction_as_module(
script_out=script_out,
mantid_log=mantid_log,
reduce_result=self.reduce_result_dir,
final_result=final_result_dir)
mock_ram.assert_called_with(self.reduce_result_dir)
self.assertEqual(mock_ctd.call_count, 0)
self.assertEqual(mock_dtd.call_count, 0)
self.assertEqual(True, actual)
def test_append_run_version_none_existing(self, _):
"""
Test: run version 0 is appended
When: overwrite is false and no runs exist
"""
ppa = PostProcessAdmin(self.message, None)
test_path = '/some/test/path'
expected = '/some/test/path/run-version-0/'
self.assertEqual(expected, ppa._append_run_version(test_path))
def test_append_run_version_no_overwrite(self, _):
"""
Test: Next run version is appended
When: Overwrite is false and runs exist
"""
ppa = PostProcessAdmin(self.message, None)
test_path = '/some/test/path'
expected = '/some/test/path/run-version-3/'
self.assertEqual(expected, ppa._append_run_version(test_path))
def test_load_reduction_script(self):
"""
Test: reduction script location is correct
When: reduction script loaded for a given instrument
"""
ppa = PostProcessAdmin(self.message, None)
file_path = ppa._load_reduction_script('WISH')
self.assertEqual(file_path, os.path.join(MISC['scripts_directory'] % 'WISH',
'reduce.py'))
def test_empty_log_and_message(self, mock_logger):
"""
Test: Log and message are correctly cleared repopulated with method argument
When: Called and message.message = ''
"""
ppa = PostProcessAdmin(self.message, None)
ppa.message.message = ''
ppa.log_and_message('test')
self.assertEqual(ppa.message.message, 'test')
mock_logger.assert_called_with('test')
def test_load_and_message_with_preexisting_message(self, mock_logger):
"""
Test: Assert existing message persists and new message logged.
When: called with new message
"""
ppa = PostProcessAdmin(self.message, None)
ppa.message.message = 'Old message'
ppa.log_and_message('New message')
self.assertEqual(ppa.message.message, 'Old message')
mock_logger.assert_called_with('New message')
def test_additional_save_directories_check_string(self, mock_ctd):
"""
Test: Correctly copies temp directory
When: Called with valid path as string
"""
out_directories = "valid/path"
reduce_result_dir = self.temporary_directory + self.ceph_directory
ppa = PostProcessAdmin(self.message, None)
ppa.additional_save_directories_check(out_directories, reduce_result_dir)
mock_ctd.assert_called_with(reduce_result_dir, out_directories)
def test_remove_with_wait_file(self):
"""
Test: File removed
When: Called
"""
file_to_remove = NamedTemporaryFile(delete=False).name
self.assertTrue(os.path.exists(str(file_to_remove)))
ppa = PostProcessAdmin(self.message, None)
ppa._remove_with_wait(False, file_to_remove)
self.assertFalse(os.path.exists(file_to_remove))
def test_remove_directory(self):
"""
Test: Directory removed
When: Called
"""
directory_to_remove = mkdtemp()
self.assertTrue(os.path.exists(directory_to_remove))
ppa = PostProcessAdmin(self.message, None)
ppa._remove_directory(directory_to_remove)
self.assertFalse(os.path.exists(directory_to_remove))
def test_write_and_readability_checks_invalid_input(self):
"""
Test: ValueError returned
When: Called with invalid read_write argument
"""
ppa = PostProcessAdmin(self.message, None)
write_list = ["directory/path/"]
with self.assertRaises(KeyError):
ppa.write_and_readability_checks(write_list, 'INVALID_KEY')
def test_append_run_version_overwrite_true(self, mock_glob):
"""
Test: version 0 is appended
When: overwrite is true
"""
self.message.overwrite = True
ppa = PostProcessAdmin(self.message, None)
test_path = '/some/test/path/'
expected = '/some/test/path/run-version-0/'
self.assertEqual(expected, ppa._append_run_version(test_path))
mock_glob.assert_not_called()
def test_validate_input_failure(self, _):
"""
Test: A ValueError is raised
When: validate_input is called with an attribute who's value is None
"""
mock_self = Mock()
mock_self.message = self.message
mock_self.message.facility = None
with self.assertRaises(ValueError):
PostProcessAdmin.validate_input(mock_self, 'facility')
def test_delete_temp_dir_valid(self, mock_logger, mock_remove_dir):
"""
Test: Assert deletion of temporary directory
When: Called with valid arguments
"""
temp_dir = mkdtemp()
PostProcessAdmin.delete_temp_directory(temp_dir)
rm_args = {'ignore_errors': True}
mock_remove_dir.assert_called_once_with(temp_dir, **rm_args)
mock_logger.assert_called_once_with('Remove temp dir %s', temp_dir)
shutil.rmtree(temp_dir)
def test_verify_directory_access_invalid(self, mock_os_access):
"""
Test: OSError is raised if there is a problem with directory path access
When: Called with valid path
"""
ppa = PostProcessAdmin(self.message, None)
location = "directory/path/"
mock_os_access.return_value = False
with self.assertRaises(Exception):
ppa.verify_directory_access(location, 'W')
def test_copy_temp_dir_with_excitation(self, _, mock_copy):
"""
Test: Excitation instrument temporary directories are handled correctly
When: When called for an excitation instrument
"""
result_dir = mkdtemp()
ppa = PostProcessAdmin(self.message, None)
ppa.instrument = 'WISH'
ppa.message.reduction_data = ['']
ppa.copy_temp_directory(result_dir, 'copy-dir')
mock_copy.assert_called_once_with(result_dir, 'copy-dir')
shutil.rmtree(result_dir)
def test_create_directory(self, mock_wrc, mock_dir, mock_os_access):
"""
Test: None returned
When: Path checks pass
"""
mock_wrc.return_value = True
mock_dir.return_value = True
mock_os_access.return_value = False
ppa = PostProcessAdmin(self.message, None)
self.assertFalse(
ppa.create_directory(list_of_paths=['should/be/writeable']))
def test_additional_save_directories_check_invalid_argument(self, mock_ctd, mock_logger):
"""
Test: Logs invalid argument
When: Called with invalid argument type
"""
out_directories = {404}
reduce_result_dir = self.temporary_directory + self.ceph_directory
ppa = PostProcessAdmin(self.message, None)
ppa.additional_save_directories_check(out_directories, reduce_result_dir)
self.assertEqual(mock_ctd.call_count, 0)
mock_logger.assert_called_once_with("Optional output directories of reduce.py must "
f"be a string or list of stings: {out_directories}")
def test_write_and_readability_checks_invalid_path(self, mock_vda):
"""
Test: Exception is raised
When: Called with invalid path/incorrect path access
"""
ppa = PostProcessAdmin(self.message, None)
write_list = ["directory/path/"]
mock_vda.return_value = False
with self.assertRaises(OSError):
ppa.write_and_readability_checks(write_list, 'W')
def test_write_and_readability_checks(self, mock_vda):
"""
Test: True is returned if there is no problem with directory path and logged as successful
When: Called with valid path and access type
"""
ppa = PostProcessAdmin(self.message, None)
write_list = ["directory/path/"]
mock_vda.return_type = True
actual_write = ppa.write_and_readability_checks(write_list, 'W')
self.assertTrue(actual_write)
def test_additional_save_directories_check_list(self, mock_ctd):
"""
Test: Correctly copies N temp directories
When: Called with valid list of paths
"""
# mock_ctd.return_value =
out_directories = ["valid/path/", "valid/path/"]
reduce_result_dir = self.temporary_directory + self.ceph_directory
ppa = PostProcessAdmin(self.message, None)
ppa.additional_save_directories_check(out_directories, reduce_result_dir)
for path in out_directories:
mock_ctd.assert_called_with(reduce_result_dir, path)
self.assertEqual(mock_ctd.call_count, 2)
def test_delete_temp_dir_invalid(self, mock_logger, mock_remove_dir):
"""
Test: Assert the inability to delete a record
When: Runtime error raised
"""
def raise_runtime(): # pragma: no cover
"""Raise Runtime Error"""
raise RuntimeError('test')
mock_remove_dir.side_effect = raise_runtime
PostProcessAdmin.delete_temp_directory('not-a-file-path.test')
mock_logger.assert_has_calls([call('Remove temp dir %s', 'not-a-file-path.test'),
call('Unable to remove temporary directory - %s',
'not-a-file-path.test')])
def test_determine_reduction_status_complete(self, mock_srm):
"""
Test: check the correct reduction message is used
When: when calling send_reduction_message() with complete status
"""
# message_status = ["skipped", "error"]
amq_client_mock = Mock()
self.message.message = None
ppa = PostProcessAdmin(self.message, amq_client_mock)
ppa.determine_reduction_status()
mock_srm.assert_called_with(message="Complete",
amq_message=ACTIVEMQ_SETTINGS.reduction_complete)
def test_additional_save_directories_check_invalid_list(self, mock_ctd, mock_logger):
"""
Test: Logs invalid list input
When: List containing non strings is passed
"""
out_directories = ["valid/path/", 404, "valid/path/"]
reduce_result_dir = self.temporary_directory + self.ceph_directory
ppa = PostProcessAdmin(self.message, None)
ppa.additional_save_directories_check(out_directories, reduce_result_dir)
mock_ctd.assert_called_with(reduce_result_dir, out_directories[0])
mock_ctd.assert_called_with(reduce_result_dir, out_directories[2])
self.assertEqual(mock_ctd.call_count, 2)
mock_logger.assert_called_once_with("Optional output directories of reduce.py must be "
f"strings: {out_directories[1]}")
def test_determine_reduction_status_exception(self, mock_srm, mock_log):
"""
Test: Assert correct number of logs are performed when
When: exception triggered by send_reduction_message
"""
amq_client_mock = Mock()
self.message.message = False
mock_srm.return_value = Exception("invalid")
ppa = PostProcessAdmin(self.message, amq_client_mock)
ppa.determine_reduction_status()
self.assertEqual(mock_log.call_count, 2)
def test_send_reduction_message_exception(self, mock_log_debug):
"""
Test: reduction status message has been sent and logged
When: called within reduce method
"""
amq_client_mock = Mock()
amq_client_mock.send.return_value = Exception
amq_message = "invalid"
ppa = PostProcessAdmin(self.message, amq_message)
ppa.send_reduction_message(message="status", amq_message=amq_message)
mock_log_debug.assert_called_with("Failed to find send reduction message: %s", amq_message)
def test_verify_directory_access(self, mock_os_access, mock_logging):
"""
Test: True is returned if there is no problem with directory path and logged as successful
When: Called with valid path with write access
"""
ppa = PostProcessAdmin(self.message, None)
location = "directory/path/"
mock_os_access.return_value = True
actual = ppa.verify_directory_access(location, "W")
expected_logs_called_with = [call("Successful %s access to %s", "write", location)]
self.assertTrue(actual)
self.assertEqual(expected_logs_called_with, mock_logging.call_args_list)