def setUp(self):
""" Start all external services """
# Get all clients
self.database_client = DatabaseClient()
self.database_client.connect()
try:
self.queue_client, self.listener = main()
except ConnectionException as err:
raise RuntimeError(
"Could not connect to ActiveMQ - check you credentials. If running locally check that "
"ActiveMQ is running and started by `python setup.py start`"
) from err
# Add placeholder variables:
# these are used to ensure runs are deleted even if test fails before completion
self.instrument = 'ARMI'
self.rb_number = 1234567
self.run_number = 101
# Create test archive and add data
self.data_archive = DataArchive([self.instrument], 19, 19)
self.data_archive.create()
# Create and send json message to ActiveMQ
self.data_ready_message = Message(rb_number=self.rb_number,
instrument=self.instrument,
run_number=self.run_number,
description="This is a system test",
facility="ISIS",
started_by=0)
def test_valid_connection(self):
"""
Test: Access is established with a valid connection
When: connect is called while valid credentials are held
"""
client = DatabaseClient()
client.connect()
self.assertTrue(client._test_connection())
def __init__(self, instrument):
"""
:param instrument: (str) The name of the instrument associated with runs
"""
self.database = DatabaseClient()
self.database.connect()
self.to_delete = {}
self.instrument = instrument
def start_database():
"""
Create and connect a database client
:return: The connected database client
"""
database = DatabaseClient()
database.connect()
return database
def test_model_access_after_connect(self):
"""
Test: Models variables can be accessed
When: connect has been called successfully
"""
client = DatabaseClient()
client.connect()
self.assertIsNotNone(client.data_model)
self.assertIsNotNone(client.variable_model)
self.assertIsNotNone(client.data_model.Instrument.objects.all())
self.assertIsNotNone(client.variable_model.Variable.objects.all())
def test_get_from_database(self):
"""
Test: Data for a given run can be retrieved from the database in the expected format
When: get_location_and_rb_from_database is called and the data is present
in the database
"""
db_client = DatabaseClient()
db_client.connect()
actual = ms.get_location_and_rb_from_database(db_client, 'MUSR', 2)
# Values from testing database
expected = ('test/file/path/2.raw', 123)
self.assertEqual(expected, actual)
def login_database():
"""
Log into the DatabaseClient
:return: The client connected, or None if failed
"""
print("Logging into Database")
database_client = DatabaseClient()
try:
database_client.connect()
except ConnectionException:
print(
"Couldn't connect to Database. Continuing without Database connection."
)
database_client = None
return database_client
def test_default_init(self):
"""
Test: Class variables are created and set
When: DatabaseClient is initialised with default credentials
"""
client = DatabaseClient()
self.assertIsNone(client.data_model)
self.assertIsNone(client.variable_model)
def test_invalid_connection(self, mock_var_query, mock_data_query):
"""
Test: A ConnectionException is raised
When: connect is called while invalid connection are held
"""
mock_data_query.Instrument.objects.first.side_effect = RuntimeError
mock_var_query.Instrument.objects.first.side_effect = RuntimeError
client = DatabaseClient()
self.assertRaises(ConnectionException, client._test_connection)
def setUp(self):
""" Start all external services """
# Get all clients
self.database_client = DatabaseClient()
self.database_client.connect()
self.queue_client = QueueClient(ACTIVEMQ_SETTINGS)
self.queue_client.connect()
# Create test archive and add data
self.data_archive_creator = DataArchiveCreator(os.path.join(
get_project_root()),
overwrite=True)
self.archive_explorer = ArchiveExplorer(
os.path.join(get_project_root(), 'data-archive'))
# Add placeholder variables:
# these are used to ensure runs are deleted even if test fails before completion
self.instrument = None
self.rb_number = None
self.run_number = None
def test_disconnect_client(self):
"""
Test: Connection is stopped and connection variables are set to None
When: disconnect is called while a valid connection is currently established
"""
django_client = DatabaseClient()
django_client.connect()
self.assertTrue(django_client._test_connection())
django_client.disconnect()
self.assertIsNone(django_client.data_model)
self.assertIsNone(django_client.variable_model)
def setUpClass(cls):
""" Start all external services """
super().setUpClass()
cls.database_client = DatabaseClient()
cls.database_client.connect()
try:
cls.queue_client, cls.listener = main()
except ConnectionException as err:
raise RuntimeError(
"Could not connect to ActiveMQ - check you credentials. If running locally check that "
"ActiveMQ is running and started by `python setup.py start`"
) from err
cls.instrument_name = "TestInstrument"
cls.rb_number = 1234567
cls.run_number = 99999
cls.data_archive = DataArchive([cls.instrument_name], 21, 21)
cls.data_archive.create()
cls.data_archive.add_reduce_vars_script(
cls.instrument_name,
"""standard_vars={"variable1":"test_variable_value_123"}""")
def setUpClass(cls):
super().setUpClass()
cls.instrument_name = "TestInstrument"
cls.data_archive = DataArchive([cls.instrument_name], 21, 21)
cls.data_archive.create()
cls.data_archive.add_reduction_script(cls.instrument_name,
"""print('some text')""")
cls.data_archive.add_reduce_vars_script(
cls.instrument_name,
f"""standard_vars={{"variable1":"{REDUCE_VARS_DEFAULT_VALUE}"}}""")
cls.database_client = DatabaseClient()
cls.database_client.connect()
try:
cls.queue_client, cls.listener = main()
except ConnectionException as err:
raise RuntimeError(
"Could not connect to ActiveMQ - check you credentials. If running locally check that "
"ActiveMQ is running and started by `python setup.py start`"
) from err
cls.instrument_name = "TestInstrument"
cls.rb_number = 1234567
cls.run_number = 99999
class ManualRemove:
"""
Handles removing a run from the database
"""
def __init__(self, instrument):
"""
:param instrument: (str) The name of the instrument associated with runs
"""
self.database = DatabaseClient()
self.database.connect()
self.to_delete = {}
self.instrument = instrument
def find_runs_in_database(self, run_number):
"""
Find all run versions in the database that relate to a given instrument and run number
:param run_number: (int) The run to search for in the database
:return: The result of the query
"""
instrument_record = db.get_instrument(self.instrument)
result = self.database.data_model.ReductionRun.objects \
.filter(instrument=instrument_record.id) \
.filter(run_number=run_number) \
.order_by('-created')
self.to_delete[run_number] = result
return result
def process_results(self):
"""
Process all the results what to do with the run based on the result of database query
"""
copy_to_delete = self.to_delete.copy()
for key, value in copy_to_delete.items():
if not value:
self.run_not_found(run_number=key)
if len(value) == 1:
continue
if len(value) > 1:
self.multiple_versions_found(run_number=key)
def run_not_found(self, run_number):
"""
Inform user and remove key from dictionary
:param run_number: (int) The run to remove from the dictionary
"""
print('No runs found associated with {} for instrument {}'.format(
run_number, self.instrument))
del self.to_delete[run_number]
def multiple_versions_found(self, run_number):
"""
Ask the user which versions they want to remove
Update the self.to_delete dictionary by removing unwanted versions
:param run_number: (int) The run number with multiple versions
"""
# Display run_number - title - version for all matching runs
print("Discovered multiple reduction versions for {}{}:".format(
self.instrument, run_number))
for run in self.to_delete[run_number]:
print("\tv{} - {}".format(run.run_version, run.run_name))
# Get user input for which versions they wish to delete
user_input = input(
"Which runs would you like to delete (e.g. 1,2,3): ")
input_valid, user_input = self.validate_csv_input(user_input)
while input_valid is False:
user_input = input(
'Input of \'{}\' was invalid. '
'Please provide a comma separated list of values:')
input_valid, user_input = self.validate_csv_input(user_input)
# Remove runs that the user does NOT want to delete from the delete list
for reduction_job in self.to_delete[run_number]:
if not int(reduction_job.run_version) in user_input:
self.to_delete[run_number].remove(reduction_job)
def delete_records(self):
"""
Delete all records from the database that match those found in self.to_delete
"""
# Make a copy to ensure dict being iterated stays same size through processing
to_delete_copy = self.to_delete.copy()
for run_number, job_list in to_delete_copy.items():
for version in job_list:
# Delete the specified version
print('{}{}:'.format(self.instrument, run_number))
self.delete_reduction_location(version.id)
self.delete_data_location(version.id)
self.delete_variables(version.id)
self.delete_reduction_run(version.id)
# Remove deleted run from dictionary
del self.to_delete[run_number]
def delete_reduction_location(self, reduction_run_id):
"""
Delete a ReductionLocation record from the database
:param reduction_run_id: (int) The id of the associated reduction job
"""
self.database.data_model.ReductionLocation.objects \
.filter(reduction_run_id=reduction_run_id) \
.delete()
def delete_data_location(self, reduction_run_id):
"""
Delete a DataLocation record from the database
:param reduction_run_id: (int) The id of the associated reduction job
"""
self.database.data_model.DataLocation.objects \
.filter(reduction_run_id=reduction_run_id) \
.delete()
def delete_variables(self, reduction_run_id):
"""
Removes all the RunVariable records associated with a given ReductionRun from the database
:param reduction_run_id: (int) The id of the associated reduction job
"""
run_variables = self.find_variables_of_reduction(reduction_run_id)
for record in run_variables:
self.database.variable_model.RunVariable.objects \
.filter(variable_ptr_id=record.variable_ptr_id) \
.delete()
def find_variables_of_reduction(self, reduction_run_id):
"""
Find all the RunVariable records in the database associated with a reduction job
:param reduction_run_id: (int) The id of the reduction job to filter by
:return: (QuerySet) of the associated RunVariables
"""
return self.database.variable_model.RunVariable.objects \
.filter(reduction_run_id=reduction_run_id)
def delete_reduction_run(self, reduction_run_id):
"""
Delete a ReductionRun record from the database
:param reduction_run_id: (int) The id of the associated reduction job
"""
self.database.data_model.ReductionRun.objects \
.filter(id=reduction_run_id) \
.delete()
@staticmethod
def validate_csv_input(user_input):
"""
checks if a comma separated list was provided
:return: (tuple) = (bool - is valid? , list - csv as list (empty list if invalid))
"""
processed_input = []
if ',' in user_input:
versions_to_delete = user_input.split(',')
for number in versions_to_delete:
try:
number = int(number)
processed_input.append(number)
except ValueError:
return False, []
else:
try:
user_input = int(user_input)
processed_input.append(user_input)
except ValueError:
return False, []
return True, processed_input
class TestEndToEnd(unittest.TestCase):
""" Class to test pipelines in autoreduction"""
def setUp(self):
""" Start all external services """
# Get all clients
self.database_client = DatabaseClient()
self.database_client.connect()
try:
self.queue_client, self.listener = main()
except ConnectionException as err:
raise RuntimeError(
"Could not connect to ActiveMQ - check you credentials. If running locally check that "
"ActiveMQ is running and started by `python setup.py start`"
) from err
# Add placeholder variables:
# these are used to ensure runs are deleted even if test fails before completion
self.instrument = 'ARMI'
self.rb_number = 1234567
self.run_number = 101
# Create test archive and add data
self.data_archive = DataArchive([self.instrument], 19, 19)
self.data_archive.create()
# Create and send json message to ActiveMQ
self.data_ready_message = Message(rb_number=self.rb_number,
instrument=self.instrument,
run_number=self.run_number,
description="This is a system test",
facility="ISIS",
started_by=0)
def tearDown(self):
""" Disconnect from services, stop external services and delete data archive """
self.queue_client.disconnect()
self.database_client.disconnect()
self._remove_run_from_database(self.instrument, self.run_number)
self.data_archive.delete()
self._delete_reduction_directory()
def _setup_data_structures(self, reduce_script, vars_script):
"""
Sets up a fake archive and reduced data save location on the system
:param reduce_script: The content to use in the reduce.py file
:param vars_script: The content to use in the reduce_vars.py file
:return: file_path to the reduced data
"""
raw_file = '{}{}.nxs'.format(self.instrument, self.run_number)
self.data_archive.add_reduction_script(self.instrument, reduce_script)
self.data_archive.add_reduce_vars_script(self.instrument, vars_script)
raw_file = self.data_archive.add_data_file(self.instrument, raw_file,
19, 1)
return raw_file
def _find_run_in_database(self):
"""
Find a ReductionRun record in the database
This includes a timeout to wait for several seconds to ensure the database has received
the record in question
:return: The resulting record
"""
instrument = db.get_instrument(self.instrument)
return instrument.reduction_runs.filter(run_number=self.run_number)
@staticmethod
def _remove_run_from_database(instrument, run_number):
"""
Uses the scripts.manual_operations.manual_remove script
to remove records added to the database
"""
if not isinstance(run_number, list):
run_number = [run_number]
for run in run_number:
remove.remove(instrument, run, delete_all_versions=True)
@staticmethod
def _delete_reduction_directory():
""" Delete the temporary reduction directory"""
path = Path(os.path.join(PROJECT_ROOT, 'reduced-data'))
if path.exists():
shutil.rmtree(path.absolute())
def send_and_wait_for_result(self, message):
"""Sends the message to the queue and waits until the listener has finished processing it"""
# forces the is_processing to return True so that the listener has time to actually start processing the message
self.listener._processing = True #pylint:disable=protected-access
self.queue_client.send('/queue/DataReady', message)
while self.listener.is_processing_message():
time.sleep(0.5)
# Get Result from database
results = self._find_run_in_database()
assert results
return results
def test_end_to_end_wish_invalid_rb_number_skipped(self):
"""
Test that data gets skipped when the RB Number doesn't validate
"""
# Set meta data for test
self.rb_number = 222
self.data_ready_message.rb_number = self.rb_number
# Create supporting data structures e.g. Data Archive, Reduce directory
file_location = self._setup_data_structures(
reduce_script=REDUCE_SCRIPT, vars_script='')
self.data_ready_message.data = file_location
results = self.send_and_wait_for_result(self.data_ready_message)
# Validate
self.assertEqual(self.instrument, results[0].instrument.name)
self.assertEqual(self.rb_number,
results[0].experiment.reference_number)
self.assertEqual(self.run_number, results[0].run_number)
self.assertEqual("This is a system test", results[0].run_name)
self.assertEqual('Skipped', results[0].status.value_verbose())
def test_end_to_end_wish_completed(self):
"""
Test that runs gets completed when everything is OK
"""
# Create supporting data structures e.g. Data Archive, Reduce directory
file_location = self._setup_data_structures(
reduce_script=REDUCE_SCRIPT, vars_script='')
self.data_ready_message.data = file_location
results = self.send_and_wait_for_result(self.data_ready_message)
# Validate
self.assertEqual(self.instrument, results[0].instrument.name)
self.assertEqual(self.rb_number,
results[0].experiment.reference_number)
self.assertEqual(self.run_number, results[0].run_number)
self.assertEqual("This is a system test", results[0].run_name)
self.assertEqual('Completed', results[0].status.value_verbose())
def test_end_to_end_wish_bad_script_syntax_error(self):
"""
Test that run gets marked as error when the script has a syntax error
"""
# Create supporting data structures e.g. Data Archive, Reduce directory
file_location = self._setup_data_structures(
reduce_script=SYNTAX_ERROR_REDUCE_SCRIPT, vars_script='')
self.data_ready_message.data = file_location
results = self.send_and_wait_for_result(self.data_ready_message)
# Validate
self.assertEqual(self.instrument, results[0].instrument.name)
self.assertEqual(self.rb_number,
results[0].experiment.reference_number)
self.assertEqual(self.run_number, results[0].run_number)
self.assertEqual("This is a system test", results[0].run_name)
self.assertEqual('Error', results[0].status.value_verbose())
self.assertIn("REDUCTION Error", results[0].message)
self.assertIn("Error encountered when running the reduction script",
results[0].message)
self.assertIn("SyntaxError('EOL while scanning string literal'",
results[0].message)
def test_end_to_end_wish_bad_script_raises_exception(self):
"""
Test that WISH data goes through the system without issue
"""
# Create supporting data structures e.g. Data Archive, Reduce directory
file_location = self._setup_data_structures(
reduce_script="raise ValueError('hello from the other side')",
vars_script='')
self.data_ready_message.data = file_location
results = self.send_and_wait_for_result(self.data_ready_message)
# Validate
self.assertEqual(self.instrument, results[0].instrument.name)
self.assertEqual(self.rb_number,
results[0].experiment.reference_number)
self.assertEqual(self.run_number, results[0].run_number)
self.assertEqual("This is a system test", results[0].run_name)
self.assertEqual('Error', results[0].status.value_verbose())
self.assertIn('ValueError', results[0].message)
self.assertIn('hello from the other side', results[0].message)
def test_end_to_end_wish_vars_script_gets_new_variable(self):
"""Test running the same run twice, but the second time the reduce_vars has a new variable"""
# Create supporting data structures e.g. Data Archive, Reduce directory
file_location = self._setup_data_structures(
reduce_script=REDUCE_SCRIPT, vars_script='')
self.data_ready_message.data = file_location
result_one = self.send_and_wait_for_result(self.data_ready_message)
assert len(result_one) == 1
run_without_vars = result_one[0]
self.data_archive.add_reduce_vars_script(self.instrument, VARS_SCRIPT)
result_two = self.send_and_wait_for_result(self.data_ready_message)
assert len(result_two) == 2
assert run_without_vars == result_two[
0] # check that the first run is queried again
run_with_vars = result_two[1]
assert run_without_vars.run_variables.count() == 0
assert run_with_vars.run_variables.count(
) == 1 # the one standard variable in the VARS_SCRIPT
var = run_with_vars.run_variables.first().variable
assert var.name == "variable1"
assert var.value == "value1"
def test_end_to_end_wish_vars_script_loses_variable(self):
"""Test running the same run twice, but the second time the reduce_vars has one less variable"""
# Create supporting data structures e.g. Data Archive, Reduce directory
file_location = self._setup_data_structures(
reduce_script=REDUCE_SCRIPT, vars_script=VARS_SCRIPT)
self.data_ready_message.data = file_location
result_one = self.send_and_wait_for_result(self.data_ready_message)
assert len(result_one) == 1
run_with_vars = result_one[0]
assert run_with_vars.run_variables.count(
) == 1 # the one standard variable in the VARS_SCRIPT
var = run_with_vars.run_variables.first().variable
assert var.name == "variable1"
assert var.value == "value1"
self.data_archive.add_reduce_vars_script(self.instrument, "")
result_two = self.send_and_wait_for_result(self.data_ready_message)
assert len(result_two) == 2
assert run_with_vars == result_two[0]
run_without_vars = result_two[1]
assert run_without_vars.run_variables.count() == 0
def test_end_to_end_vars_script_has_variable_value_changed(self):
"""Test that reducing the same run after changing the reduce_vars updates the variable's value"""
# Create supporting data structures e.g. Data Archive, Reduce directory
file_location = self._setup_data_structures(
reduce_script=REDUCE_SCRIPT, vars_script=VARS_SCRIPT)
self.data_ready_message.data = file_location
result_one = self.send_and_wait_for_result(self.data_ready_message)
assert len(result_one) == 1
run_with_initial_var = result_one[0]
assert run_with_initial_var.run_variables.count(
) == 1 # the one standard variable in the VARS_SCRIPT
var = run_with_initial_var.run_variables.first().variable
assert var.name == "variable1"
assert var.value == "value1"
self.data_archive.add_reduce_vars_script(
self.instrument, 'standard_vars={"variable1": 123}')
result_two = self.send_and_wait_for_result(self.data_ready_message)
assert len(result_two) == 2
assert run_with_initial_var == result_two[0]
run_with_changed_var = result_two[1]
assert run_with_initial_var.run_variables.count() == 1
assert run_with_changed_var.run_variables.count() == 1
initial_var = run_with_initial_var.run_variables.first().variable
changed_var = run_with_changed_var.run_variables.first().variable
assert initial_var == changed_var
def test_end_to_end_wish_vars_script_has_variable_reused_on_new_run_number(
self):
"""Test that the variables are reused on new run numbers, IF their value has not changed"""
# Create supporting data structures e.g. Data Archive, Reduce directory
file_location = self._setup_data_structures(
reduce_script=REDUCE_SCRIPT, vars_script=VARS_SCRIPT)
self.data_ready_message.data = file_location
result_one = self.send_and_wait_for_result(self.data_ready_message)
run_with_initial_var = result_one[0]
self.data_ready_message.run_number = 1234568
result_two = self.send_and_wait_for_result(self.data_ready_message)
run_with_different_run_number = result_two[0]
assert run_with_initial_var.run_variables.count() == 1
assert run_with_different_run_number.run_variables.count() == 1
initial_var = run_with_initial_var.run_variables.first().variable
new_var = run_with_different_run_number.run_variables.first().variable
assert initial_var == new_var
def test_end_to_end_wish_vars_script_has_variable_copied_on_new_run_number_when_value_changed(
self):
"""Test that the variable is copied for a new run WHEN it's value has been changed"""
# Create supporting data structures e.g. Data Archive, Reduce directory
file_location = self._setup_data_structures(
reduce_script=REDUCE_SCRIPT, vars_script=VARS_SCRIPT)
self.run_number = 101
self.data_ready_message.data = file_location
result_one = self.send_and_wait_for_result(self.data_ready_message)
assert len(result_one) == 1
run_with_initial_var = result_one[0]
assert run_with_initial_var.run_variables.count(
) == 1 # the one standard variable in the VARS_SCRIPT
var = run_with_initial_var.run_variables.first().variable
assert var.name == "variable1"
assert var.value == "value1"
# update the run number in the class because it's used to query for the correct run
self.data_ready_message.run_number = self.run_number = 102
self.data_archive.add_reduce_vars_script(
self.instrument, 'standard_vars={"variable1": 123}')
result_two = self.send_and_wait_for_result(self.data_ready_message)
# making the run_number a list so that they can be deleted by the tearDown!
self.run_number = [101, 102]
assert len(result_two) == 1
run_with_changed_var = result_two[0]
assert run_with_initial_var.run_variables.count() == 1
assert run_with_changed_var.run_variables.count() == 1
initial_var = run_with_initial_var.run_variables.first().variable
changed_var = run_with_changed_var.run_variables.first().variable
assert initial_var != changed_var
assert initial_var.name == changed_var.name
assert initial_var.value != changed_var.value
assert initial_var.type != changed_var.type
assert initial_var.instrumentvariable.start_run < changed_var.instrumentvariable.start_run
class ManualRemove:
"""
Handles removing a run from the database
"""
def __init__(self, instrument):
"""
:param instrument: (str) The name of the instrument associated with runs
"""
self.database = DatabaseClient()
self.database.connect()
self.to_delete = {}
self.instrument = instrument
def find_run_versions_in_database(self, run_number):
"""
Find all run versions in the database that relate to a given instrument and run number
:param run_number: (int) The run to search for in the database
:return: (QuerySet) The result of the query
"""
instrument_record = db.get_instrument(self.instrument)
result = self.database.data_model.ReductionRun.objects \
.filter(instrument=instrument_record.id) \
.filter(run_number=run_number) \
.order_by('-created')
self.to_delete[run_number] = list(result)
return result
def process_results(self, delete_all_versions: bool):
"""
Process all the results what to do with the run based on the result of database query
"""
copy_to_delete = self.to_delete.copy()
for key, value in copy_to_delete.items():
if not value:
self.run_not_found(run_number=key)
if len(value) == 1:
continue
if len(value) > 1 and not delete_all_versions:
self.multiple_versions_found(run_number=key)
def run_not_found(self, run_number):
"""
Inform user and remove key from dictionary
:param run_number: (int) The run to remove from the dictionary
"""
print('No runs found associated with {} for instrument {}'.format(run_number, self.instrument))
del self.to_delete[run_number]
def multiple_versions_found(self, run_number):
"""
Ask the user which versions they want to remove
Update the self.to_delete dictionary by removing unwanted versions
:param run_number: (int) The run number with multiple versions
"""
# Display run_number - title - version for all matching runs
print("Discovered multiple reduction versions for {}{}:".format(self.instrument, run_number))
for run in self.to_delete[run_number]:
print("\tv{} - {}".format(run.run_version, run.run_name))
# Get user input for which versions they wish to delete
user_input = input("Which runs would you like to delete (e.g. 0,1,2,3 or 0-3): ")
input_valid, user_input = self.validate_csv_input(user_input)
while input_valid is False:
user_input = input('Input was invalid. ' 'Please provide a comma separated list or a range of values: ')
input_valid, user_input = self.validate_csv_input(user_input)
# Remove runs that the user does NOT want to delete from the delete list
self.to_delete[run_number] = [
reduction_job for reduction_job in self.to_delete[run_number] if reduction_job.run_version in user_input
]
def delete_records(self):
"""
Delete all records from the database that match those found in self.to_delete
"""
# Make a copy to ensure dict being iterated stays same size through processing
to_delete_copy = self.to_delete.copy()
for run_number, job_list in to_delete_copy.items():
for version in job_list:
print(f'Deleting {self.instrument}{run_number} - v{version.run_version}')
try:
version.delete()
except IntegrityError as err:
print(f"Encountered integrity error: {err}\n\n"
"Reverting to old behaviour - manual deletion. This can take much longer.")
# For some reason some entries can throw an integrity error.
# In that case we revert to the previous (much slower) way of manually
# deleting everything. Perhaps there is a badly configured relation
# but I am not sure why it works on _most_
self.delete_reduction_location(version.id)
self.delete_data_location(version.id)
self.delete_variables(version.id)
self.delete_reduction_run(version.id)
def delete_reduction_location(self, reduction_run_id):
"""
Delete a ReductionLocation record from the database
:param reduction_run_id: (int) The id of the associated reduction job
"""
self.database.data_model.ReductionLocation.objects \
.filter(reduction_run_id=reduction_run_id) \
.delete()
def delete_data_location(self, reduction_run_id):
"""
Delete a DataLocation record from the database
:param reduction_run_id: (int) The id of the associated reduction job
"""
self.database.data_model.DataLocation.objects \
.filter(reduction_run_id=reduction_run_id) \
.delete()
def delete_variables(self, reduction_run_id):
"""
Removes all the RunVariable records associated with a given ReductionRun from the database
:param reduction_run_id: (int) The id of the associated reduction job
"""
run_variables = self.find_variables_of_reduction(reduction_run_id)
for record in run_variables:
self.database.variable_model.RunVariable.objects \
.filter(variable_ptr_id=record.variable_ptr_id) \
.delete()
def find_variables_of_reduction(self, reduction_run_id):
"""
Find all the RunVariable records in the database associated with a reduction job
:param reduction_run_id: (int) The id of the reduction job to filter by
:return: (QuerySet) of the associated RunVariables
"""
return self.database.variable_model.RunVariable.objects \
.filter(reduction_run_id=reduction_run_id)
def delete_reduction_run(self, reduction_run_id):
"""
Delete a ReductionRun record from the database
:param reduction_run_id: (int) The id of the associated reduction job
"""
self.database.data_model.ReductionRun.objects \
.filter(id=reduction_run_id) \
.delete()
@staticmethod
def validate_csv_input(user_input):
"""
checks if a comma separated list was provided
:return: (tuple) = (bool - is valid? , list - csv as list (empty list if invalid))
"""
processed_input = []
if ',' in user_input:
versions_to_delete = user_input.split(',')
for number in versions_to_delete:
try:
number = int(number)
processed_input.append(number)
except ValueError:
return False, []
elif "-" in user_input:
range_of_versions_to_delete = user_input.split('-')
if len(range_of_versions_to_delete) != 2:
return False, []
sorted_range_of_versions = sorted(map(int, range_of_versions_to_delete))
smaller_version = int(sorted_range_of_versions[0])
larger_version = int(sorted_range_of_versions[1])
return True, list(range(smaller_version, larger_version + 1))
else:
try:
user_input = int(user_input)
processed_input.append(user_input)
except ValueError:
return False, []
return True, processed_input
class TestEndToEnd(unittest.TestCase):
""" Class to test pipelines in autoreduction"""
@classmethod
def setUpClass(cls):
# Start all services
external.start_queue_processors()
def setUp(self):
""" Start all external services """
# Get all clients
self.database_client = DatabaseClient()
self.database_client.connect()
self.queue_client = QueueClient(ACTIVEMQ_SETTINGS)
self.queue_client.connect()
# Create test archive and add data
self.data_archive_creator = DataArchiveCreator(os.path.join(
get_project_root()),
overwrite=True)
self.archive_explorer = ArchiveExplorer(
os.path.join(get_project_root(), 'data-archive'))
# Add placeholder variables:
# these are used to ensure runs are deleted even if test fails before completion
self.instrument = None
self.rb_number = None
self.run_number = None
def test_end_to_end_wish(self):
"""
Test that WISH data goes through the system without issue
"""
# Set meta data for test
self.instrument = 'WISH'
self.rb_number = 222
self.run_number = 101
reduce_script = \
'def main(input_file, output_dir):\n' \
'\tprint("WISH system test")\n' \
'\n' \
'if __name__ == "__main__":\n' \
'\tmain()\n'
# Create supporting data structures e.g. Data Archive, Reduce directory
file_location = self._setup_data_structures(
reduce_script=reduce_script, vars_script='')
# Create and send json message to ActiveMQ
data_ready_message = Message(rb_number=self.rb_number,
instrument=self.instrument,
data=file_location,
run_number=self.run_number,
facility="ISIS",
started_by=0)
self.queue_client.send('/queue/DataReady', data_ready_message)
# Get Result from database
results = self._find_run_in_database()
# Validate
self.assertEqual(self.instrument, results[0].instrument.name)
self.assertEqual(self.rb_number,
results[0].experiment.reference_number)
self.assertEqual(self.run_number, results[0].run_number)
self.assertEqual('Completed', results[0].status.value_verbose())
def test_wish_user_script_failure(self):
"""
Test that WISH data goes through the system without issue
"""
# Set meta data for test
self.instrument = 'WISH'
self.rb_number = 222
self.run_number = 101
# Create supporting data structures e.g. Data Archive, Reduce directory
file_location = self._setup_data_structures(reduce_script='fail',
vars_script='')
# Create and send json message to ActiveMQ
data_ready_message = Message(rb_number=self.rb_number,
instrument=self.instrument,
data=file_location,
run_number=self.run_number,
facility="ISIS",
started_by=0)
self.queue_client.send('/queue/DataReady', data_ready_message)
# Get Result from database
results = self._find_run_in_database()
# Validate
self.assertEqual(self.instrument, results[0].instrument.name)
self.assertEqual(self.rb_number,
results[0].experiment.reference_number)
self.assertEqual(self.run_number, results[0].run_number)
self.assertEqual(
'e', results[0].status.value) # verbose value = "Error"
def _setup_data_structures(self, reduce_script, vars_script):
"""
Sets up a fake archive and reduced data save location on the system
:param reduce_script: The content to use in the reduce.py file
:param vars_script: The content to use in the reduce_vars.py file
:return: file_path to the reduced data
"""
raw_file = '{}{}.nxs'.format(self.instrument, self.run_number)
# Create and add data to archive
self.data_archive_creator.make_data_archive([self.instrument], 19,
19, 1)
self.data_archive_creator.add_reduce_script(
instrument=self.instrument, file_content=reduce_script)
self.data_archive_creator.add_reduce_vars_script(
self.instrument, vars_script)
self.data_archive_creator.add_data_to_most_recent_cycle(
self.instrument, raw_file)
# Make temporary location to add reduced files to
self._make_reduction_directory(self.instrument, self.rb_number,
self.run_number)
# Submit message to activemq
cycle_path = self.archive_explorer.get_cycle_directory(
self.instrument, 19, 1)
return os.path.join(cycle_path, raw_file)
@staticmethod
def _make_reduction_directory(instrument, rb_number, run_number):
"""
Make a directory in the expected location for reduced runs to be written to
"""
reduced_dir = os.path.join(get_project_root(), 'reduced-data')
reduced_inst = os.path.join(reduced_dir, str(instrument))
reduced_rb = os.path.join(reduced_inst,
'RB{}'.format(str(rb_number)))
reduced_auto = os.path.join(reduced_rb, 'autoreduced')
reduced_run = os.path.join(reduced_auto, str(run_number))
os.mkdir(reduced_dir)
os.mkdir(reduced_inst)
os.mkdir(reduced_rb)
os.mkdir(reduced_auto)
os.mkdir(reduced_run)
def _find_run_in_database(self):
"""
Find a ReductionRun record in the database
This includes a timeout to wait for several seconds to ensure the database has received
the record in question
:return: The resulting record
"""
wait_times = [0, 1, 2, 3, 5]
results = []
for timeout in wait_times:
# Wait before attempting database access
print(f"Waiting for: {timeout}")
time.sleep(timeout)
# Check database has expected values
instrument_record = db.get_instrument(self.instrument)
results = self.database_client.data_model.ReductionRun.objects \
.filter(instrument=instrument_record.id) \
.filter(run_number=self.run_number) \
.select_related() \
.all()
try:
actual = results[0]
except IndexError:
# If no results found yet then continue
continue
# verbose values = "Completed" or "Error"
if actual.status.value == 'c' or actual.status.value == 'e':
print(
f"Job reached {actual.status.value} status after {timeout} seconds"
)
break
return results
@staticmethod
def _remove_run_from_database(instrument, run_number):
"""
Uses the scripts.manual_operations.manual_remove script
to remove records added to the database
"""
remove.remove(instrument, run_number)
@staticmethod
def _delete_reduction_directory():
""" Delete the temporary reduction directory"""
shutil.rmtree(os.path.join(get_project_root(), 'reduced-data'))
def tearDown(self):
""" Disconnect from services, stop external services and delete data archive """
self.queue_client.disconnect()
self.database_client.disconnect()
self._delete_reduction_directory()
del self.data_archive_creator
# Done in tearDown to ensure run is removed even if test fails early
self._remove_run_from_database(self.instrument, self.run_number)
@classmethod
def tearDownClass(cls):
# Stop external services
external.stop_queue_processors()