def test_disconnect_client(self):
"""
Test: Connection is stopped and connection variables are set to None
When: disconnect is called while a valid connection is currently established
"""
django_client = DatabaseClient()
django_client.connect()
self.assertTrue(django_client._test_connection())
django_client.disconnect()
self.assertIsNone(django_client.data_model)
self.assertIsNone(django_client.variable_model)
class TestEndToEnd(unittest.TestCase):
""" Class to test pipelines in autoreduction"""
@classmethod
def setUpClass(cls):
# Start all services
external.start_queue_processors()
def setUp(self):
""" Start all external services """
# Get all clients
self.database_client = DatabaseClient()
self.database_client.connect()
self.queue_client = QueueClient(ACTIVEMQ_SETTINGS)
self.queue_client.connect()
# Create test archive and add data
self.data_archive_creator = DataArchiveCreator(os.path.join(
get_project_root()),
overwrite=True)
self.archive_explorer = ArchiveExplorer(
os.path.join(get_project_root(), 'data-archive'))
# Add placeholder variables:
# these are used to ensure runs are deleted even if test fails before completion
self.instrument = None
self.rb_number = None
self.run_number = None
def test_end_to_end_wish(self):
"""
Test that WISH data goes through the system without issue
"""
# Set meta data for test
self.instrument = 'WISH'
self.rb_number = 222
self.run_number = 101
reduce_script = \
'def main(input_file, output_dir):\n' \
'\tprint("WISH system test")\n' \
'\n' \
'if __name__ == "__main__":\n' \
'\tmain()\n'
# Create supporting data structures e.g. Data Archive, Reduce directory
file_location = self._setup_data_structures(
reduce_script=reduce_script, vars_script='')
# Create and send json message to ActiveMQ
data_ready_message = Message(rb_number=self.rb_number,
instrument=self.instrument,
data=file_location,
run_number=self.run_number,
facility="ISIS",
started_by=0)
self.queue_client.send('/queue/DataReady', data_ready_message)
# Get Result from database
results = self._find_run_in_database()
# Validate
self.assertEqual(self.instrument, results[0].instrument.name)
self.assertEqual(self.rb_number,
results[0].experiment.reference_number)
self.assertEqual(self.run_number, results[0].run_number)
self.assertEqual('Completed', results[0].status.value_verbose())
def test_wish_user_script_failure(self):
"""
Test that WISH data goes through the system without issue
"""
# Set meta data for test
self.instrument = 'WISH'
self.rb_number = 222
self.run_number = 101
# Create supporting data structures e.g. Data Archive, Reduce directory
file_location = self._setup_data_structures(reduce_script='fail',
vars_script='')
# Create and send json message to ActiveMQ
data_ready_message = Message(rb_number=self.rb_number,
instrument=self.instrument,
data=file_location,
run_number=self.run_number,
facility="ISIS",
started_by=0)
self.queue_client.send('/queue/DataReady', data_ready_message)
# Get Result from database
results = self._find_run_in_database()
# Validate
self.assertEqual(self.instrument, results[0].instrument.name)
self.assertEqual(self.rb_number,
results[0].experiment.reference_number)
self.assertEqual(self.run_number, results[0].run_number)
self.assertEqual(
'e', results[0].status.value) # verbose value = "Error"
def _setup_data_structures(self, reduce_script, vars_script):
"""
Sets up a fake archive and reduced data save location on the system
:param reduce_script: The content to use in the reduce.py file
:param vars_script: The content to use in the reduce_vars.py file
:return: file_path to the reduced data
"""
raw_file = '{}{}.nxs'.format(self.instrument, self.run_number)
# Create and add data to archive
self.data_archive_creator.make_data_archive([self.instrument], 19,
19, 1)
self.data_archive_creator.add_reduce_script(
instrument=self.instrument, file_content=reduce_script)
self.data_archive_creator.add_reduce_vars_script(
self.instrument, vars_script)
self.data_archive_creator.add_data_to_most_recent_cycle(
self.instrument, raw_file)
# Make temporary location to add reduced files to
self._make_reduction_directory(self.instrument, self.rb_number,
self.run_number)
# Submit message to activemq
cycle_path = self.archive_explorer.get_cycle_directory(
self.instrument, 19, 1)
return os.path.join(cycle_path, raw_file)
@staticmethod
def _make_reduction_directory(instrument, rb_number, run_number):
"""
Make a directory in the expected location for reduced runs to be written to
"""
reduced_dir = os.path.join(get_project_root(), 'reduced-data')
reduced_inst = os.path.join(reduced_dir, str(instrument))
reduced_rb = os.path.join(reduced_inst,
'RB{}'.format(str(rb_number)))
reduced_auto = os.path.join(reduced_rb, 'autoreduced')
reduced_run = os.path.join(reduced_auto, str(run_number))
os.mkdir(reduced_dir)
os.mkdir(reduced_inst)
os.mkdir(reduced_rb)
os.mkdir(reduced_auto)
os.mkdir(reduced_run)
def _find_run_in_database(self):
"""
Find a ReductionRun record in the database
This includes a timeout to wait for several seconds to ensure the database has received
the record in question
:return: The resulting record
"""
wait_times = [0, 1, 2, 3, 5]
results = []
for timeout in wait_times:
# Wait before attempting database access
print(f"Waiting for: {timeout}")
time.sleep(timeout)
# Check database has expected values
instrument_record = db.get_instrument(self.instrument)
results = self.database_client.data_model.ReductionRun.objects \
.filter(instrument=instrument_record.id) \
.filter(run_number=self.run_number) \
.select_related() \
.all()
try:
actual = results[0]
except IndexError:
# If no results found yet then continue
continue
# verbose values = "Completed" or "Error"
if actual.status.value == 'c' or actual.status.value == 'e':
print(
f"Job reached {actual.status.value} status after {timeout} seconds"
)
break
return results
@staticmethod
def _remove_run_from_database(instrument, run_number):
"""
Uses the scripts.manual_operations.manual_remove script
to remove records added to the database
"""
remove.remove(instrument, run_number)
@staticmethod
def _delete_reduction_directory():
""" Delete the temporary reduction directory"""
shutil.rmtree(os.path.join(get_project_root(), 'reduced-data'))
def tearDown(self):
""" Disconnect from services, stop external services and delete data archive """
self.queue_client.disconnect()
self.database_client.disconnect()
self._delete_reduction_directory()
del self.data_archive_creator
# Done in tearDown to ensure run is removed even if test fails early
self._remove_run_from_database(self.instrument, self.run_number)
@classmethod
def tearDownClass(cls):
# Stop external services
external.stop_queue_processors()
class TestEndToEnd(unittest.TestCase):
""" Class to test pipelines in autoreduction"""
def setUp(self):
""" Start all external services """
# Get all clients
self.database_client = DatabaseClient()
self.database_client.connect()
try:
self.queue_client, self.listener = main()
except ConnectionException as err:
raise RuntimeError(
"Could not connect to ActiveMQ - check you credentials. If running locally check that "
"ActiveMQ is running and started by `python setup.py start`"
) from err
# Add placeholder variables:
# these are used to ensure runs are deleted even if test fails before completion
self.instrument = 'ARMI'
self.rb_number = 1234567
self.run_number = 101
# Create test archive and add data
self.data_archive = DataArchive([self.instrument], 19, 19)
self.data_archive.create()
# Create and send json message to ActiveMQ
self.data_ready_message = Message(rb_number=self.rb_number,
instrument=self.instrument,
run_number=self.run_number,
description="This is a system test",
facility="ISIS",
started_by=0)
def tearDown(self):
""" Disconnect from services, stop external services and delete data archive """
self.queue_client.disconnect()
self.database_client.disconnect()
self._remove_run_from_database(self.instrument, self.run_number)
self.data_archive.delete()
self._delete_reduction_directory()
def _setup_data_structures(self, reduce_script, vars_script):
"""
Sets up a fake archive and reduced data save location on the system
:param reduce_script: The content to use in the reduce.py file
:param vars_script: The content to use in the reduce_vars.py file
:return: file_path to the reduced data
"""
raw_file = '{}{}.nxs'.format(self.instrument, self.run_number)
self.data_archive.add_reduction_script(self.instrument, reduce_script)
self.data_archive.add_reduce_vars_script(self.instrument, vars_script)
raw_file = self.data_archive.add_data_file(self.instrument, raw_file,
19, 1)
return raw_file
def _find_run_in_database(self):
"""
Find a ReductionRun record in the database
This includes a timeout to wait for several seconds to ensure the database has received
the record in question
:return: The resulting record
"""
instrument = db.get_instrument(self.instrument)
return instrument.reduction_runs.filter(run_number=self.run_number)
@staticmethod
def _remove_run_from_database(instrument, run_number):
"""
Uses the scripts.manual_operations.manual_remove script
to remove records added to the database
"""
if not isinstance(run_number, list):
run_number = [run_number]
for run in run_number:
remove.remove(instrument, run, delete_all_versions=True)
@staticmethod
def _delete_reduction_directory():
""" Delete the temporary reduction directory"""
path = Path(os.path.join(PROJECT_ROOT, 'reduced-data'))
if path.exists():
shutil.rmtree(path.absolute())
def send_and_wait_for_result(self, message):
"""Sends the message to the queue and waits until the listener has finished processing it"""
# forces the is_processing to return True so that the listener has time to actually start processing the message
self.listener._processing = True #pylint:disable=protected-access
self.queue_client.send('/queue/DataReady', message)
while self.listener.is_processing_message():
time.sleep(0.5)
# Get Result from database
results = self._find_run_in_database()
assert results
return results
def test_end_to_end_wish_invalid_rb_number_skipped(self):
"""
Test that data gets skipped when the RB Number doesn't validate
"""
# Set meta data for test
self.rb_number = 222
self.data_ready_message.rb_number = self.rb_number
# Create supporting data structures e.g. Data Archive, Reduce directory
file_location = self._setup_data_structures(
reduce_script=REDUCE_SCRIPT, vars_script='')
self.data_ready_message.data = file_location
results = self.send_and_wait_for_result(self.data_ready_message)
# Validate
self.assertEqual(self.instrument, results[0].instrument.name)
self.assertEqual(self.rb_number,
results[0].experiment.reference_number)
self.assertEqual(self.run_number, results[0].run_number)
self.assertEqual("This is a system test", results[0].run_name)
self.assertEqual('Skipped', results[0].status.value_verbose())
def test_end_to_end_wish_completed(self):
"""
Test that runs gets completed when everything is OK
"""
# Create supporting data structures e.g. Data Archive, Reduce directory
file_location = self._setup_data_structures(
reduce_script=REDUCE_SCRIPT, vars_script='')
self.data_ready_message.data = file_location
results = self.send_and_wait_for_result(self.data_ready_message)
# Validate
self.assertEqual(self.instrument, results[0].instrument.name)
self.assertEqual(self.rb_number,
results[0].experiment.reference_number)
self.assertEqual(self.run_number, results[0].run_number)
self.assertEqual("This is a system test", results[0].run_name)
self.assertEqual('Completed', results[0].status.value_verbose())
def test_end_to_end_wish_bad_script_syntax_error(self):
"""
Test that run gets marked as error when the script has a syntax error
"""
# Create supporting data structures e.g. Data Archive, Reduce directory
file_location = self._setup_data_structures(
reduce_script=SYNTAX_ERROR_REDUCE_SCRIPT, vars_script='')
self.data_ready_message.data = file_location
results = self.send_and_wait_for_result(self.data_ready_message)
# Validate
self.assertEqual(self.instrument, results[0].instrument.name)
self.assertEqual(self.rb_number,
results[0].experiment.reference_number)
self.assertEqual(self.run_number, results[0].run_number)
self.assertEqual("This is a system test", results[0].run_name)
self.assertEqual('Error', results[0].status.value_verbose())
self.assertIn("REDUCTION Error", results[0].message)
self.assertIn("Error encountered when running the reduction script",
results[0].message)
self.assertIn("SyntaxError('EOL while scanning string literal'",
results[0].message)
def test_end_to_end_wish_bad_script_raises_exception(self):
"""
Test that WISH data goes through the system without issue
"""
# Create supporting data structures e.g. Data Archive, Reduce directory
file_location = self._setup_data_structures(
reduce_script="raise ValueError('hello from the other side')",
vars_script='')
self.data_ready_message.data = file_location
results = self.send_and_wait_for_result(self.data_ready_message)
# Validate
self.assertEqual(self.instrument, results[0].instrument.name)
self.assertEqual(self.rb_number,
results[0].experiment.reference_number)
self.assertEqual(self.run_number, results[0].run_number)
self.assertEqual("This is a system test", results[0].run_name)
self.assertEqual('Error', results[0].status.value_verbose())
self.assertIn('ValueError', results[0].message)
self.assertIn('hello from the other side', results[0].message)
def test_end_to_end_wish_vars_script_gets_new_variable(self):
"""Test running the same run twice, but the second time the reduce_vars has a new variable"""
# Create supporting data structures e.g. Data Archive, Reduce directory
file_location = self._setup_data_structures(
reduce_script=REDUCE_SCRIPT, vars_script='')
self.data_ready_message.data = file_location
result_one = self.send_and_wait_for_result(self.data_ready_message)
assert len(result_one) == 1
run_without_vars = result_one[0]
self.data_archive.add_reduce_vars_script(self.instrument, VARS_SCRIPT)
result_two = self.send_and_wait_for_result(self.data_ready_message)
assert len(result_two) == 2
assert run_without_vars == result_two[
0] # check that the first run is queried again
run_with_vars = result_two[1]
assert run_without_vars.run_variables.count() == 0
assert run_with_vars.run_variables.count(
) == 1 # the one standard variable in the VARS_SCRIPT
var = run_with_vars.run_variables.first().variable
assert var.name == "variable1"
assert var.value == "value1"
def test_end_to_end_wish_vars_script_loses_variable(self):
"""Test running the same run twice, but the second time the reduce_vars has one less variable"""
# Create supporting data structures e.g. Data Archive, Reduce directory
file_location = self._setup_data_structures(
reduce_script=REDUCE_SCRIPT, vars_script=VARS_SCRIPT)
self.data_ready_message.data = file_location
result_one = self.send_and_wait_for_result(self.data_ready_message)
assert len(result_one) == 1
run_with_vars = result_one[0]
assert run_with_vars.run_variables.count(
) == 1 # the one standard variable in the VARS_SCRIPT
var = run_with_vars.run_variables.first().variable
assert var.name == "variable1"
assert var.value == "value1"
self.data_archive.add_reduce_vars_script(self.instrument, "")
result_two = self.send_and_wait_for_result(self.data_ready_message)
assert len(result_two) == 2
assert run_with_vars == result_two[0]
run_without_vars = result_two[1]
assert run_without_vars.run_variables.count() == 0
def test_end_to_end_vars_script_has_variable_value_changed(self):
"""Test that reducing the same run after changing the reduce_vars updates the variable's value"""
# Create supporting data structures e.g. Data Archive, Reduce directory
file_location = self._setup_data_structures(
reduce_script=REDUCE_SCRIPT, vars_script=VARS_SCRIPT)
self.data_ready_message.data = file_location
result_one = self.send_and_wait_for_result(self.data_ready_message)
assert len(result_one) == 1
run_with_initial_var = result_one[0]
assert run_with_initial_var.run_variables.count(
) == 1 # the one standard variable in the VARS_SCRIPT
var = run_with_initial_var.run_variables.first().variable
assert var.name == "variable1"
assert var.value == "value1"
self.data_archive.add_reduce_vars_script(
self.instrument, 'standard_vars={"variable1": 123}')
result_two = self.send_and_wait_for_result(self.data_ready_message)
assert len(result_two) == 2
assert run_with_initial_var == result_two[0]
run_with_changed_var = result_two[1]
assert run_with_initial_var.run_variables.count() == 1
assert run_with_changed_var.run_variables.count() == 1
initial_var = run_with_initial_var.run_variables.first().variable
changed_var = run_with_changed_var.run_variables.first().variable
assert initial_var == changed_var
def test_end_to_end_wish_vars_script_has_variable_reused_on_new_run_number(
self):
"""Test that the variables are reused on new run numbers, IF their value has not changed"""
# Create supporting data structures e.g. Data Archive, Reduce directory
file_location = self._setup_data_structures(
reduce_script=REDUCE_SCRIPT, vars_script=VARS_SCRIPT)
self.data_ready_message.data = file_location
result_one = self.send_and_wait_for_result(self.data_ready_message)
run_with_initial_var = result_one[0]
self.data_ready_message.run_number = 1234568
result_two = self.send_and_wait_for_result(self.data_ready_message)
run_with_different_run_number = result_two[0]
assert run_with_initial_var.run_variables.count() == 1
assert run_with_different_run_number.run_variables.count() == 1
initial_var = run_with_initial_var.run_variables.first().variable
new_var = run_with_different_run_number.run_variables.first().variable
assert initial_var == new_var
def test_end_to_end_wish_vars_script_has_variable_copied_on_new_run_number_when_value_changed(
self):
"""Test that the variable is copied for a new run WHEN it's value has been changed"""
# Create supporting data structures e.g. Data Archive, Reduce directory
file_location = self._setup_data_structures(
reduce_script=REDUCE_SCRIPT, vars_script=VARS_SCRIPT)
self.run_number = 101
self.data_ready_message.data = file_location
result_one = self.send_and_wait_for_result(self.data_ready_message)
assert len(result_one) == 1
run_with_initial_var = result_one[0]
assert run_with_initial_var.run_variables.count(
) == 1 # the one standard variable in the VARS_SCRIPT
var = run_with_initial_var.run_variables.first().variable
assert var.name == "variable1"
assert var.value == "value1"
# update the run number in the class because it's used to query for the correct run
self.data_ready_message.run_number = self.run_number = 102
self.data_archive.add_reduce_vars_script(
self.instrument, 'standard_vars={"variable1": 123}')
result_two = self.send_and_wait_for_result(self.data_ready_message)
# making the run_number a list so that they can be deleted by the tearDown!
self.run_number = [101, 102]
assert len(result_two) == 1
run_with_changed_var = result_two[0]
assert run_with_initial_var.run_variables.count() == 1
assert run_with_changed_var.run_variables.count() == 1
initial_var = run_with_initial_var.run_variables.first().variable
changed_var = run_with_changed_var.run_variables.first().variable
assert initial_var != changed_var
assert initial_var.name == changed_var.name
assert initial_var.value != changed_var.value
assert initial_var.type != changed_var.type
assert initial_var.instrumentvariable.start_run < changed_var.instrumentvariable.start_run