def __init__(self, input_data_path, instrument, proposal, run_number):
"""
Use templates from the settings file to populate reduction paths
:param input_data_path: path to data file
:param instrument: instrument for reduction
:param proposal: RB number that relates to the data file
:param run_number: The run number for the input file
"""
reduction_script_path = paths.append_path(MISC['scripts_directory'] % instrument,
['reduce.py'])
reduction_script_vars_path = paths.append_path(MISC['scripts_directory'] % instrument,
['reduce_vars.py'])
temp_output_dir = MISC['temp_root_directory']
output_directory = MISC['ceph_directory'] % (instrument, proposal, run_number)
# Excitations would like to remove the run number folder at the end
if instrument in MISC['excitation_instruments']:
output_directory = output_directory[:output_directory.rfind('/') + 1]
ReductionPathManager.__init__(self,
input_data_path=input_data_path,
reduction_script_path=reduction_script_path,
reduction_variables_path=reduction_script_vars_path,
temporary_root_directory=temp_output_dir,
final_output_directory=output_directory)
def test_valid_init(self, mock_validate):
"""
Create expected paths from AutoreductionProcessor.settings.MISC and test that
the paths in the ISISReductionPathManager are set up correctly to match these
Validation is tested somewhere else so we can mock it here instead
"""
mock_validate.return_value = True
isis_paths = ISISReductionPathManager(
input_data_path=os.path.realpath(__file__),
instrument='GEM',
proposal='12345',
run_number='54321')
# Input Paths
expected_data_path = os.path.realpath(__file__)
expected_script_path = append_path(MISC['scripts_directory'] % 'GEM',
['reduce.py'])
expected_vars_path = append_path(MISC['scripts_directory'] % 'GEM',
['reduce_vars.py'])
self.assertEqual(expected_data_path,
isis_paths.input_paths.data_path.value)
self.assertEqual(expected_script_path,
isis_paths.input_paths.reduction_script_path.value)
self.assertEqual(expected_vars_path,
isis_paths.input_paths.reduction_variables_path.value)
# Output Paths
expected_out_path = MISC['ceph_directory'] % ('GEM', '12345', '54321')
expected_log_path = append_path(
MISC['ceph_directory'] % ('GEM', '12345', '54321'),
['reduction_log'])
self.assertEqual(expected_out_path,
isis_paths.output_paths.data_directory.value)
self.assertEqual(expected_log_path,
isis_paths.output_paths.log_directory.value)
# Temporary Paths
expected_temp_path = MISC['temp_root_directory']
expected_temp_out_path = append_path(expected_temp_path,
split(expected_out_path))
expected_temp_log_path = append_path(expected_temp_path,
split(expected_log_path))
self.assertEqual(expected_temp_path,
isis_paths.temporary_paths.root_directory.value)
self.assertEqual(expected_temp_out_path,
isis_paths.temporary_paths.data_directory.value)
self.assertEqual(expected_temp_log_path,
isis_paths.temporary_paths.log_directory.value)
def test_result_and_log_directory(self, mock_nrdp, mock_logging):
"""
Test: final result and log directories are returned
When: called with temp root directory, result and log locations
"""
ppa = PostProcessAdmin(self.message, None)
instrument_output_dir = MISC["ceph_directory"] % (
ppa.instrument, ppa.proposal, ppa.run_number)
mock_nrdp.return_value = append_path(instrument_output_dir, "0")
instrument_output_directory = instrument_output_dir[:
instrument_output_dir
.rfind('/') + 1]
reduce_directory = MISC[
"temp_root_directory"] + instrument_output_directory
reduction_log = "/reduction_log/"
actual_final_result, actual_log = ppa.create_final_result_and_log_directory(
temporary_root_directory=MISC["temp_root_directory"],
reduce_dir=reduce_directory)
expected_log = f"{instrument_output_directory}0{reduction_log}"
expected_logs_called_with = [
call("Final Result Directory = %s", actual_final_result),
call("Final log directory: %s", actual_log)
]
mock_nrdp.assert_called_once_with(instrument_output_dir)
self.assertEqual(mock_logging.call_count, 2)
self.assertEqual(mock_logging.call_args_list,
expected_logs_called_with)
self.assertEqual(expected_log, actual_log)
def create_final_result_and_log_directory(self, temporary_root_directory,
reduce_dir):
"""
Create final result and final log directories, stripping temporary path off of the
front of temporary directories
:param temporary_root_directory: (str) temporary root directory
:param reduce_dir: (str) final reduce directory
:return (tuple) - (str, str) final result and final log directory paths
"""
# validate dir before slicing
if reduce_dir.startswith(temporary_root_directory):
result_directory = reduce_dir[len(temporary_root_directory):]
else:
return ValueError(
"The reduce directory does not start by following the expected "
"format: %s \n", temporary_root_directory)
final_result_directory = self._new_reduction_data_path(
result_directory)
final_log_directory = append_path(final_result_directory,
['reduction_log'])
logger.info("Final Result Directory = %s", final_result_directory)
logger.info("Final log directory: %s", final_log_directory)
return final_result_directory, final_log_directory
def _append_run_version(self, path):
"""
Append the run version to the output directory. If its the first run run-version-0 will be
added, if overwrite is true run-version-0 will always be overwritten
:param path: The reduction output path
:return: The reduction output path with the run version appended
"""
if self.message.overwrite:
return append_path(path, ["run-version-0"])
run_versions = [
int(i.split("-")[-1])
for i in glob.glob(f"{path}/run-version-[0-9]*")
]
try:
return append_path(path, [f"run-version-{max(run_versions) + 1}"])
except ValueError:
return append_path(path, ["run-version-0"])
def test_new_reduction_data_only_root_path_exists(self):
"""
Test: The given path is returned with a 0 directory appended
When: _new_reduction_data_path is called on a path without version sub-directories
"""
self.setup_test_dir_structure([self.test_root])
mock_self = Mock()
mock_self.message = Message(overwrite=None)
expected = append_path(self.test_root, "0")
actual = PostProcessAdmin._new_reduction_data_path(
mock_self, self.test_root)
self.assertEqual(expected, actual)
def test_new_reduction_data_path_overwrite_paths_exist(self, _):
"""
Test: The given path is returned with a 0 directory appended
When: _new_reduction_data_path is called on an existing path with overwrite: True
"""
self.setup_test_dir_structure(self.test_paths)
mock_self = Mock()
mock_self.message = Message(overwrite=True)
expected = append_path(self.test_root, "0")
actual = PostProcessAdmin._new_reduction_data_path(
mock_self, self.test_root)
self.assertEqual(expected, actual)
def test_new_reduction_data_path_no_overwrite_paths_exist(self, _):
"""
Test: A path is returned with a final directory one higher than the current highest
When: _new_reduction_data_path is called on an existing path with overwrite: None
"""
self.setup_test_dir_structure(self.test_paths)
mock_self = Mock()
mock_self.message = Message(overwrite=None)
expected = append_path(self.test_root, "3")
actual = PostProcessAdmin._new_reduction_data_path(
mock_self, self.test_root)
self.assertEqual(expected, actual)
def __init__(self, input_data_path, reduction_script_path,
reduction_variables_path, temporary_root_directory,
final_output_directory):
# Holds all of the paths to the input locations for data
self.input_paths = InputPaths(
data_path=input_data_path,
reduction_script_path=reduction_script_path,
reduction_variables_path=reduction_variables_path)
# Holds all of the paths for the final output location of the data
self.output_paths = OutputPaths(data_directory=final_output_directory,
log_directory=append_path(
final_output_directory,
['reduction_log']))
# Holds all of the paths for temporary output locations before data is migrated
# to final destination
temp_dir = append_path(temporary_root_directory,
split(final_output_directory))
self.temporary_paths = TemporaryPaths(
root_directory=temporary_root_directory,
data_directory=temp_dir,
log_directory=append_path(temp_dir, ['reduction_log']))
def _new_reduction_data_path(self, path):
"""
Creates a pathname for the reduction data, factoring in existing run data.
:param path: Base path for the run data (should follow convention, without version number)
:return: A pathname for the new reduction data
"""
logger.info("_new_reduction_data_path argument: %s", path)
# if there is an 'overwrite' key/member with a None/False value
if not self.message.overwrite:
if os.path.isdir(path): # if the given path already exists..
contents = os.listdir(path)
highest_vers = -1
for item in contents: # ..for every item, if it's a dir and a int..
if os.path.isdir(os.path.join(path, item)):
try: # ..store the highest int
vers = int(item)
highest_vers = max(highest_vers, vers)
except ValueError:
pass
this_vers = highest_vers + 1
return append_path(path, [str(this_vers)])
# (else) if no overwrite, overwrite true, or the path doesn't exist: return version 0 path
return append_path(path, "0")
def test_add_to_unix_path(self):
self.assertEqual(
utils.append_path('/unix', ['path', 'to', 'add', 'to']),
'/unix/path/to/add/to/')
def test_add_to_windows_path(self):
self.assertEqual(
utils.append_path('C:\\windows', ['path', 'to', 'add', 'to']),
'C:\\windows\\path\\to\\add\\to\\')
# ############################################################################### #
# Autoreduction Repository : https://github.com/ISISScientificComputing/autoreduce
#
# Copyright © 2019 ISIS Rutherford Appleton Laboratory UKRI
# SPDX - License - Identifier: GPL-3.0-or-later
# ############################################################################### #
# pylint: skip-file
"""
Settings for ActiveMQ and reduction variables
"""
import os
from paths.path_manipulation import append_path
from utils.project.structure import get_project_root
# MISC
# "scripts_directory": "/isis/NDX%s/user/scripts/autoreduction",
# "ceph_directory": "/instrument/%s/RBNumber/RB%s/autoreduced/%s",
MISC = {
"script_timeout": 3600, # The max time to wait for a user script to finish running (seconds)
"mantid_path": "/opt/Mantid/lib",
"scripts_directory": append_path(get_project_root(), ['data-archive', 'NDX%s', 'user', 'scripts', 'autoreduction']),
"post_process_directory": append_path(os.path.dirname(os.path.realpath(__file__)), ["post_process_admin.py"]),
"ceph_directory": append_path(get_project_root(), ['reduced-data', '%s', 'RB%s', 'autoreduced', '%s']),
"temp_root_directory": "/autoreducetmp",
"excitation_instruments": ["LET", "MARI", "MAPS", "MERLIN", "WISH", "GEM"]
}
