def __init__(self, input_data_path, instrument, proposal, run_number): """ Use templates from the settings file to populate reduction paths :param input_data_path: path to data file :param instrument: instrument for reduction :param proposal: RB number that relates to the data file :param run_number: The run number for the input file """ reduction_script_path = paths.append_path(MISC['scripts_directory'] % instrument, ['reduce.py']) reduction_script_vars_path = paths.append_path(MISC['scripts_directory'] % instrument, ['reduce_vars.py']) temp_output_dir = MISC['temp_root_directory'] output_directory = MISC['ceph_directory'] % (instrument, proposal, run_number) # Excitations would like to remove the run number folder at the end if instrument in MISC['excitation_instruments']: output_directory = output_directory[:output_directory.rfind('/') + 1] ReductionPathManager.__init__(self, input_data_path=input_data_path, reduction_script_path=reduction_script_path, reduction_variables_path=reduction_script_vars_path, temporary_root_directory=temp_output_dir, final_output_directory=output_directory)
def test_valid_init(self, mock_validate): """ Create expected paths from AutoreductionProcessor.settings.MISC and test that the paths in the ISISReductionPathManager are set up correctly to match these Validation is tested somewhere else so we can mock it here instead """ mock_validate.return_value = True isis_paths = ISISReductionPathManager( input_data_path=os.path.realpath(__file__), instrument='GEM', proposal='12345', run_number='54321') # Input Paths expected_data_path = os.path.realpath(__file__) expected_script_path = append_path(MISC['scripts_directory'] % 'GEM', ['reduce.py']) expected_vars_path = append_path(MISC['scripts_directory'] % 'GEM', ['reduce_vars.py']) self.assertEqual(expected_data_path, isis_paths.input_paths.data_path.value) self.assertEqual(expected_script_path, isis_paths.input_paths.reduction_script_path.value) self.assertEqual(expected_vars_path, isis_paths.input_paths.reduction_variables_path.value) # Output Paths expected_out_path = MISC['ceph_directory'] % ('GEM', '12345', '54321') expected_log_path = append_path( MISC['ceph_directory'] % ('GEM', '12345', '54321'), ['reduction_log']) self.assertEqual(expected_out_path, isis_paths.output_paths.data_directory.value) self.assertEqual(expected_log_path, isis_paths.output_paths.log_directory.value) # Temporary Paths expected_temp_path = MISC['temp_root_directory'] expected_temp_out_path = append_path(expected_temp_path, split(expected_out_path)) expected_temp_log_path = append_path(expected_temp_path, split(expected_log_path)) self.assertEqual(expected_temp_path, isis_paths.temporary_paths.root_directory.value) self.assertEqual(expected_temp_out_path, isis_paths.temporary_paths.data_directory.value) self.assertEqual(expected_temp_log_path, isis_paths.temporary_paths.log_directory.value)
def test_result_and_log_directory(self, mock_nrdp, mock_logging): """ Test: final result and log directories are returned When: called with temp root directory, result and log locations """ ppa = PostProcessAdmin(self.message, None) instrument_output_dir = MISC["ceph_directory"] % ( ppa.instrument, ppa.proposal, ppa.run_number) mock_nrdp.return_value = append_path(instrument_output_dir, "0") instrument_output_directory = instrument_output_dir[: instrument_output_dir .rfind('/') + 1] reduce_directory = MISC[ "temp_root_directory"] + instrument_output_directory reduction_log = "/reduction_log/" actual_final_result, actual_log = ppa.create_final_result_and_log_directory( temporary_root_directory=MISC["temp_root_directory"], reduce_dir=reduce_directory) expected_log = f"{instrument_output_directory}0{reduction_log}" expected_logs_called_with = [ call("Final Result Directory = %s", actual_final_result), call("Final log directory: %s", actual_log) ] mock_nrdp.assert_called_once_with(instrument_output_dir) self.assertEqual(mock_logging.call_count, 2) self.assertEqual(mock_logging.call_args_list, expected_logs_called_with) self.assertEqual(expected_log, actual_log)
def create_final_result_and_log_directory(self, temporary_root_directory, reduce_dir): """ Create final result and final log directories, stripping temporary path off of the front of temporary directories :param temporary_root_directory: (str) temporary root directory :param reduce_dir: (str) final reduce directory :return (tuple) - (str, str) final result and final log directory paths """ # validate dir before slicing if reduce_dir.startswith(temporary_root_directory): result_directory = reduce_dir[len(temporary_root_directory):] else: return ValueError( "The reduce directory does not start by following the expected " "format: %s \n", temporary_root_directory) final_result_directory = self._new_reduction_data_path( result_directory) final_log_directory = append_path(final_result_directory, ['reduction_log']) logger.info("Final Result Directory = %s", final_result_directory) logger.info("Final log directory: %s", final_log_directory) return final_result_directory, final_log_directory
def _append_run_version(self, path): """ Append the run version to the output directory. If its the first run run-version-0 will be added, if overwrite is true run-version-0 will always be overwritten :param path: The reduction output path :return: The reduction output path with the run version appended """ if self.message.overwrite: return append_path(path, ["run-version-0"]) run_versions = [ int(i.split("-")[-1]) for i in glob.glob(f"{path}/run-version-[0-9]*") ] try: return append_path(path, [f"run-version-{max(run_versions) + 1}"]) except ValueError: return append_path(path, ["run-version-0"])
def test_new_reduction_data_only_root_path_exists(self): """ Test: The given path is returned with a 0 directory appended When: _new_reduction_data_path is called on a path without version sub-directories """ self.setup_test_dir_structure([self.test_root]) mock_self = Mock() mock_self.message = Message(overwrite=None) expected = append_path(self.test_root, "0") actual = PostProcessAdmin._new_reduction_data_path( mock_self, self.test_root) self.assertEqual(expected, actual)
def test_new_reduction_data_path_overwrite_paths_exist(self, _): """ Test: The given path is returned with a 0 directory appended When: _new_reduction_data_path is called on an existing path with overwrite: True """ self.setup_test_dir_structure(self.test_paths) mock_self = Mock() mock_self.message = Message(overwrite=True) expected = append_path(self.test_root, "0") actual = PostProcessAdmin._new_reduction_data_path( mock_self, self.test_root) self.assertEqual(expected, actual)
def test_new_reduction_data_path_no_overwrite_paths_exist(self, _): """ Test: A path is returned with a final directory one higher than the current highest When: _new_reduction_data_path is called on an existing path with overwrite: None """ self.setup_test_dir_structure(self.test_paths) mock_self = Mock() mock_self.message = Message(overwrite=None) expected = append_path(self.test_root, "3") actual = PostProcessAdmin._new_reduction_data_path( mock_self, self.test_root) self.assertEqual(expected, actual)
def __init__(self, input_data_path, reduction_script_path, reduction_variables_path, temporary_root_directory, final_output_directory): # Holds all of the paths to the input locations for data self.input_paths = InputPaths( data_path=input_data_path, reduction_script_path=reduction_script_path, reduction_variables_path=reduction_variables_path) # Holds all of the paths for the final output location of the data self.output_paths = OutputPaths(data_directory=final_output_directory, log_directory=append_path( final_output_directory, ['reduction_log'])) # Holds all of the paths for temporary output locations before data is migrated # to final destination temp_dir = append_path(temporary_root_directory, split(final_output_directory)) self.temporary_paths = TemporaryPaths( root_directory=temporary_root_directory, data_directory=temp_dir, log_directory=append_path(temp_dir, ['reduction_log']))
def _new_reduction_data_path(self, path): """ Creates a pathname for the reduction data, factoring in existing run data. :param path: Base path for the run data (should follow convention, without version number) :return: A pathname for the new reduction data """ logger.info("_new_reduction_data_path argument: %s", path) # if there is an 'overwrite' key/member with a None/False value if not self.message.overwrite: if os.path.isdir(path): # if the given path already exists.. contents = os.listdir(path) highest_vers = -1 for item in contents: # ..for every item, if it's a dir and a int.. if os.path.isdir(os.path.join(path, item)): try: # ..store the highest int vers = int(item) highest_vers = max(highest_vers, vers) except ValueError: pass this_vers = highest_vers + 1 return append_path(path, [str(this_vers)]) # (else) if no overwrite, overwrite true, or the path doesn't exist: return version 0 path return append_path(path, "0")
def test_add_to_unix_path(self): self.assertEqual( utils.append_path('/unix', ['path', 'to', 'add', 'to']), '/unix/path/to/add/to/')
def test_add_to_windows_path(self): self.assertEqual( utils.append_path('C:\\windows', ['path', 'to', 'add', 'to']), 'C:\\windows\\path\\to\\add\\to\\')
# ############################################################################### # # Autoreduction Repository : https://github.com/ISISScientificComputing/autoreduce # # Copyright © 2019 ISIS Rutherford Appleton Laboratory UKRI # SPDX - License - Identifier: GPL-3.0-or-later # ############################################################################### # # pylint: skip-file """ Settings for ActiveMQ and reduction variables """ import os from paths.path_manipulation import append_path from utils.project.structure import get_project_root # MISC # "scripts_directory": "/isis/NDX%s/user/scripts/autoreduction", # "ceph_directory": "/instrument/%s/RBNumber/RB%s/autoreduced/%s", MISC = { "script_timeout": 3600, # The max time to wait for a user script to finish running (seconds) "mantid_path": "/opt/Mantid/lib", "scripts_directory": append_path(get_project_root(), ['data-archive', 'NDX%s', 'user', 'scripts', 'autoreduction']), "post_process_directory": append_path(os.path.dirname(os.path.realpath(__file__)), ["post_process_admin.py"]), "ceph_directory": append_path(get_project_root(), ['reduced-data', '%s', 'RB%s', 'autoreduced', '%s']), "temp_root_directory": "/autoreducetmp", "excitation_instruments": ["LET", "MARI", "MAPS", "MERLIN", "WISH", "GEM"] }