def test_find_highest_run_version(self):
"""
Test: The expected highest version number is returned
When: Calling find_highest_run_version
"""
experiment, _ = self.database.data_model.Experiment.objects.get_or_create(
reference_number=1231231)
instrument, _ = self.database.data_model.Instrument.objects.get_or_create(
name="ARMI", is_active=1, is_paused=0)
status = access.get_status("q")
fake_script_text = "scripttext"
reduction_run_v0 = create_reduction_run_record(experiment, instrument,
FakeMessage(), 0,
fake_script_text,
status)
reduction_run_v0.save()
reduction_run_v1 = create_reduction_run_record(experiment, instrument,
FakeMessage(), 1,
fake_script_text,
status)
reduction_run_v1.save()
reduction_run_v2 = create_reduction_run_record(experiment, instrument,
FakeMessage(), 2,
fake_script_text,
status)
reduction_run_v2.save()
assert access.find_highest_run_version(experiment, 1234567) == 3
reduction_run_v0.delete()
reduction_run_v1.delete()
reduction_run_v2.delete()
experiment.delete()
instrument.delete()
def test_create_reduction_record_starts_db(db_layer):
"""
Test: The correct DB accesses are made
When: Creating the reduction record
"""
# We do not actually care about what was passed in for this test
arg = mock.NonCallableMock()
records.create_reduction_run_record(arg, arg, arg, arg, arg, arg)
db_layer.start_database.assert_called_once()
db_layer.start_database.return_value.data_model\
.ReductionRun.assert_called_once()
def create_run_records(self, message: Message):
"""
Creates or gets the necessary records to construct a ReductionRun
"""
# This must be done before looking up the run version to make sure the record exists
experiment = db_access.get_experiment(message.rb_number)
run_version = db_access.find_highest_run_version(
experiment, run_number=str(message.run_number))
message.run_version = run_version
instrument = db_access.get_instrument(str(message.instrument))
script = ReductionScript(instrument.name)
script_text = script.text()
# Make the new reduction run with the information collected so far
reduction_run = db_records.create_reduction_run_record(
experiment=experiment,
instrument=instrument,
message=message,
run_version=run_version,
script_text=script_text,
status=self.status.get_queued())
reduction_run.save()
# Create a new data location entry which has a foreign key linking it to the current
# reduction run. The file path itself will point to a datafile
# (e.g. "/isis/inst$/NDXWISH/Instrument/data/cycle_17_1/WISH00038774.nxs")
data_location = self.data_model.DataLocation(
file_path=message.data, reduction_run_id=reduction_run.pk)
data_location.save()
return reduction_run, message, instrument
def test_new_reduction_run(self, _):
"""
Tests with a never before seen Reduction Run
"""
reduction_run = create_reduction_run_record(self.experiment,
self.instrument,
FakeMessage(), 0,
self.fake_script_text,
self.status)
reduction_run.save()
before_creating_variables = self.variable_model.InstrumentVariable.objects.count(
)
new_variables = InstrumentVariablesUtils().create_run_variables(
reduction_run)
after_creating_variables = self.variable_model.InstrumentVariable.objects.count(
)
self.assertGreater(after_creating_variables, before_creating_variables)
self.assertEqual(new_variables[0].variable.name, "standard_var1")
self.assertEqual(new_variables[0].variable.value, "standard_value1")
self.assertEqual(new_variables[1].variable.name, "advanced_var1")
self.assertEqual(new_variables[1].variable.value, "advanced_value1")
self.delete_on_teardown = [reduction_run, new_variables]
def test_imported_module_no_variables(self):
"""
Test: that no variables get created
When: the imported reduce_vars has no variables in it
"""
reduction_run = create_reduction_run_record(self.experiment,
self.instrument,
FakeMessage(), 0,
self.fake_script_text,
self.status)
reduction_run.save()
before_creating_variables = self.variable_model.InstrumentVariable.objects.count(
)
with patch(
"queue_processors.queue_processor.reduction.service.ReductionScript.load",
return_value=FakeModule(standard_vars={}, advanced_vars={})):
new_variables = InstrumentVariablesUtils().create_run_variables(
reduction_run)
assert not new_variables
after_creating_variables = self.variable_model.InstrumentVariable.objects.count(
)
assert after_creating_variables == before_creating_variables
def test_two_reduction_runs_only_creates_one_set_of_variables(self, _):
"""
Tests that creating variables for a module that has the same variables will
re-use the variables once they have been created
"""
reduction_run = create_reduction_run_record(self.experiment,
self.instrument,
FakeMessage(), 0,
self.fake_script_text,
self.status)
reduction_run.save()
before_creating_variables = self.variable_model.InstrumentVariable.objects.count(
)
new_variables = InstrumentVariablesUtils().create_run_variables(
reduction_run)
after_creating_variables = self.variable_model.InstrumentVariable.objects.count(
)
new_variables_again = InstrumentVariablesUtils().create_run_variables(
reduction_run)
after_creating_variables_again = self.variable_model.InstrumentVariable.objects.count(
)
self.assertGreater(after_creating_variables, before_creating_variables)
self.assertEqual(after_creating_variables,
after_creating_variables_again)
self.assertEqual(new_variables[0].variable,
new_variables_again[0].variable)
self.assertEqual(new_variables[1].variable,
new_variables_again[1].variable)
self.delete_on_teardown = [reduction_run, new_variables]
def setUp(self):
self.mocked_client = mock.Mock(spec=QueueListener)
self.msg = Message()
self.msg.populate({
"run_number": 7654321,
"rb_number": 1234567,
"run_version": 0,
"reduction_data": "/path/1",
"started_by": -1,
"data": "/path",
"description": "This is a fake description",
"instrument": "ARMI" # Autoreduction Mock Instrument
})
with patch("logging.getLogger") as patched_logger:
self.handler = HandleMessage(self.mocked_client)
self.mocked_logger = patched_logger.return_value
db_handle = model.database.access.start_database()
self.data_model = db_handle.data_model
self.variable_model = db_handle.variable_model
self.experiment, _ = self.data_model.Experiment.objects.get_or_create(
reference_number=1231231)
self.instrument, _ = self.data_model.Instrument.objects.get_or_create(
name="ARMI")
status = STATUS.get_queued()
fake_script_text = "scripttext"
self.reduction_run = create_reduction_run_record(
self.experiment, self.instrument, FakeMessage(), 0,
fake_script_text, status)
self.reduction_run.save()
def test_imported_module_one_dict_loses_a_new_variable(
self, param_variable_dict):
"""
Test: removed variables are not used accidentally
when the variable module has less variables a variable (e.g. one has been removed)
"""
reduction_run = create_reduction_run_record(self.experiment,
self.instrument,
FakeMessage(), 0,
self.fake_script_text,
self.status)
reduction_run.save()
before_creating_variables = self.variable_model.InstrumentVariable.objects.count(
)
with patch(
"queue_processors.queue_processor.reduction.service.ReductionScript.load",
return_value=FakeModule(**param_variable_dict)):
new_variables = InstrumentVariablesUtils().create_run_variables(
reduction_run)
after_creating_variables = self.variable_model.InstrumentVariable.objects.count(
)
assert after_creating_variables > before_creating_variables
new_variables_again = None
# loop twice and check that no new variables are created
for _ in range(2):
# MODIFIES an advanced value so that they no longer match
with patch(
"queue_processors.queue_processor.reduction.service.ReductionScript.load",
return_value=FakeModule()):
new_variables_again = InstrumentVariablesUtils(
).create_run_variables(reduction_run)
after_creating_variables_again = self.variable_model.InstrumentVariable.objects.count(
)
assert after_creating_variables == after_creating_variables_again
# check that the previous variables are contained in the new ones
assert new_variables_again[0].variable in [
nv.variable for nv in new_variables
]
assert new_variables_again[1].variable in [
nv.variable for nv in new_variables
]
# check that ONE variable (the new one) is not contained in the first variable creation
assert len({nva.variable
for nva in new_variables} -
{nv.variable
for nv in new_variables_again}) == 1
self.delete_on_teardown = [
reduction_run, new_variables, new_variables_again
]
def make_test_run(experiment, instrument, run_version: str):
"Creates a test run and saves it to the database"
status = STATUS.get_queued()
fake_script_text = "scripttext"
msg1 = FakeMessage()
msg1.run_number = 101
run = create_reduction_run_record(experiment, instrument, msg1,
run_version, fake_script_text, status)
run.save()
return run
def test_variable_changed_for_new_run_gets_copied(self):
"""
Test: Existing variable that tracks the script gets copied when its
value/type/help is updated and the run_number is different
When: The variable was created for a previous reduction run, but the value was changed in reduce_vars
"""
reduction_run = create_reduction_run_record(self.experiment,
self.instrument,
FakeMessage(), 0,
self.fake_script_text,
self.status)
reduction_run.save()
with patch(
"queue_processors.queue_processor.reduction.service.ReductionScript.load",
return_value=FakeModule(advanced_vars={})):
variables = InstrumentVariablesUtils().create_run_variables(
reduction_run)
newer_reduction_run = create_reduction_run_record(
self.experiment, self.instrument, FakeMessage(run_number=7654321),
0, self.fake_script_text, self.status)
newer_reduction_run.save()
with patch(
"queue_processors.queue_processor.reduction.service.ReductionScript.load",
return_value=FakeModule(standard_vars={"standard_var1": 123},
advanced_vars={},
variable_help={
"standard_vars": {
"standard_var1":
"CHANGED HELP FOR VARIABLE"
}
})):
newer_variables = InstrumentVariablesUtils().create_run_variables(
newer_reduction_run)
assert variables[0].variable != newer_variables[0].variable
self.delete_on_teardown = [
reduction_run, variables, newer_reduction_run, newer_variables
]
def test_imported_module_variable_dict_changed(self, param_variable_dict):
"""
Test that only the current variables in reduce_vars are created
When: the reduce_vars module gets changed
"""
reduction_run = create_reduction_run_record(self.experiment,
self.instrument,
FakeMessage(), 0,
self.fake_script_text,
self.status)
reduction_run.save()
before_creating_variables = self.variable_model.InstrumentVariable.objects.count(
)
with patch(
"queue_processors.queue_processor.reduction.service.ReductionScript.load",
return_value=FakeModule()):
new_variables = InstrumentVariablesUtils().create_run_variables(
reduction_run)
after_creating_variables = self.variable_model.InstrumentVariable.objects.count(
)
assert after_creating_variables > before_creating_variables
new_variables_again = None
# loop twice and check that no new variables are created
for _ in range(2):
# MODIFIES an advanced value so that they no longer match
with patch(
"queue_processors.queue_processor.reduction.service.ReductionScript.load",
return_value=FakeModule(**param_variable_dict)):
new_variables_again = InstrumentVariablesUtils(
).create_run_variables(reduction_run)
after_creating_variables_again = self.variable_model.InstrumentVariable.objects.count(
)
assert after_creating_variables + 1 == after_creating_variables_again
if "standard_vars" in param_variable_dict:
ops = [operator.ne, operator.eq]
else:
ops = [operator.eq, operator.ne]
assert ops[0](new_variables[0].variable,
new_variables_again[0].variable)
assert ops[1](new_variables[1].variable,
new_variables_again[1].variable)
self.delete_on_teardown = [
reduction_run, new_variables, new_variables_again
]
def test_variable_that_exists_and_does_not_track_script_gets_ignored(self):
"""
Test: Existing variable that tracks the script gets its value/type/help updated
When: The variable was created for a previous reduction run, but the value was changed in reduce_vars
"""
reduction_run = create_reduction_run_record(self.experiment,
self.instrument,
FakeMessage(), 0,
self.fake_script_text,
self.status)
reduction_run.save()
before_creating_variables = self.variable_model.InstrumentVariable.objects.count(
)
with patch(
"queue_processors.queue_processor.reduction.service.ReductionScript.load",
return_value=FakeModule(advanced_vars={})):
new_variables = InstrumentVariablesUtils().create_run_variables(
reduction_run)
new_variables[0].variable.tracks_script = False
new_variables[0].variable.save()
after_creating_variables = self.variable_model.InstrumentVariable.objects.count(
)
assert after_creating_variables > before_creating_variables
# change the VALUE and the TYPE of the variable
with patch(
"queue_processors.queue_processor.reduction.service.ReductionScript.load",
return_value=FakeModule(standard_vars={"standard_var1": 123},
advanced_vars={},
variable_help={
"standard_vars": {
"standard_var1":
"CHANGED HELP FOR VARIABLE"
}
})):
new_variables_again = InstrumentVariablesUtils(
).create_run_variables(reduction_run)
var = new_variables_again[0].variable
assert var.name == "standard_var1"
assert var.value == "standard_value1"
assert var.type == "text"
assert var.help_text == "This is help for standard_value1"
self.delete_on_teardown = [
reduction_run, new_variables, new_variables_again
]
def test_create_reduction_record_forwards_correctly(
self, datetime_patch, db_layer):
"""
Test: Reduction Record uses args correctly.
Any fields which are hard-coded are mocked with ANY to prevent
this test becoming brittle to future change. If we expect a user
to pass them in as an arg we should test they get unpacked correctly
When: Called to simplify ORM record creation
"""
mock_experiment = mock.NonCallableMock()
mock_inst = mock.NonCallableMock()
mock_msg = mock.NonCallableMock()
mock_run_version = mock.NonCallableMock()
mock_script_text = mock.NonCallableMock()
mock_status = mock.NonCallableMock()
returned = records.create_reduction_run_record(
experiment=mock_experiment,
instrument=mock_inst,
message=mock_msg,
run_version=mock_run_version,
script_text=mock_script_text,
status=mock_status)
mock_record_orm = db_layer.start_database.return_value.data_model
self.assertEqual(mock_record_orm.ReductionRun.return_value, returned)
mock_record_orm.ReductionRun.assert_called_once_with(
run_number=mock_msg.run_number,
run_version=mock_run_version,
created=datetime_patch.utcnow.return_value,
last_updated=datetime_patch.utcnow.return_value,
experiment_id=mock_experiment.id,
instrument_id=mock_inst.id,
status_id=mock_status.id,
script=mock_script_text,
started_by=mock_msg.started_by,
# Hardcoded below
run_name=mock.ANY,
cancel=mock.ANY,
hidden_in_failviewer=mock.ANY,
admin_log=mock.ANY,
reduction_log=mock.ANY)
def test_get_reduction_run_valid(self):
"""
Test: A ReductionRun record is returned
When: get_reduction_run is called with values that match a database record
"""
experiment, _ = self.database.data_model.Experiment.objects.get_or_create(
reference_number=1231231)
instrument, _ = self.database.data_model.Instrument.objects.get_or_create(
name="ARMI", is_active=1, is_paused=0)
status = access.get_status("q")
fake_script_text = "scripttext"
reduction_run = create_reduction_run_record(experiment, instrument,
FakeMessage(), 0,
fake_script_text, status)
reduction_run.save()
assert access.get_reduction_run('ARMI',
1234567).first() == reduction_run
reduction_run.delete()
experiment.delete()
instrument.delete()
def data_ready(self, message: Message):
"""
Called when destination queue was data_ready.
Updates the reduction run in the database.
"""
self._logger.info("Data ready for processing run %s on %s",
message.run_number, message.instrument)
if not validate_rb_number(message.rb_number):
# rb_number is invalid so send message to skip queue and early return
message.message = f"Found non-integer RB number: {message.rb_number}"
self._logger.warning("%s. Skipping %s%s.", message.message,
message.instrument, message.run_number)
message.rb_number = 0
run_no = str(message.run_number)
instrument = self._get_and_activate_db_inst(message.instrument)
status = self._utils.status.get_skipped() if instrument.is_paused \
else self._utils.status.get_queued()
# This must be done before looking up the run version to make sure
# the record exists
experiment = db_access.get_experiment(message.rb_number, create=True)
run_version = db_access.find_highest_run_version(run_number=run_no,
experiment=experiment)
run_version += 1
message.run_version = run_version
# Get the script text for the current instrument. If the script text
# is null then send to
# error queue
script_text = self._utils. \
instrument_variable.get_current_script_text(instrument.name)[0]
if script_text is None:
self.reduction_error(message)
raise InvalidStateException(
"Script text for current instrument is null")
# Make the new reduction run with the information collected so far
# and add it into the database
reduction_run = db_records.create_reduction_run_record(
experiment=experiment,
instrument=instrument,
message=message,
run_version=run_version,
script_text=script_text,
status=status)
db_access.save_record(reduction_run)
# Create a new data location entry which has a foreign key linking
# it to the current
# reduction run. The file path itself will point to a datafile
# (e.g. "\isis\inst$\NDXWISH\Instrument\data\cycle_17_1\WISH00038774
# .nxs")
data_location = self._data_model.DataLocation(
file_path=message.data, reduction_run_id=reduction_run.id)
db_access.save_record(data_location)
# We now need to create all of the variables for the run such that
# the script can run
# through in the desired way
self._logger.info('Creating variables for run')
variables = self._utils.instrument_variable.create_variables_for_run(
reduction_run)
if not variables:
self._logger.warning(
"No instrument variables found on %s for run %s",
instrument.name, message.run_number)
self._logger.info('Getting script and arguments')
reduction_script, arguments = self._utils.reduction_run. \
get_script_and_arguments(reduction_run)
message.reduction_script = reduction_script
message.reduction_arguments = arguments
# Make sure the RB number is valid
try:
message.validate("/queue/DataReady")
except RuntimeError as validation_err:
self._logger.error("Validation error from handler: %s",
str(validation_err))
self._client.send_message('/queue/ReductionSkipped', message)
return
if instrument.is_paused:
self._logger.info("Run %s has been skipped", message.run_number)
else:
self._client.send_message('/queue/ReductionPending', message)
self._logger.info("Run %s ready for reduction", message.run_number)
