def create_run_records(self, message: Message):
"""
Creates or gets the necessary records to construct a ReductionRun
"""
# This must be done before looking up the run version to make sure the record exists
experiment = db_access.get_experiment(message.rb_number)
run_version = db_access.find_highest_run_version(
experiment, run_number=str(message.run_number))
message.run_version = run_version
instrument = db_access.get_instrument(str(message.instrument))
script = ReductionScript(instrument.name)
script_text = script.text()
# Make the new reduction run with the information collected so far
reduction_run = db_records.create_reduction_run_record(
experiment=experiment,
instrument=instrument,
message=message,
run_version=run_version,
script_text=script_text,
status=self.status.get_queued())
reduction_run.save()
# Create a new data location entry which has a foreign key linking it to the current
# reduction run. The file path itself will point to a datafile
# (e.g. "/isis/inst$/NDXWISH/Instrument/data/cycle_17_1/WISH00038774.nxs")
data_location = self.data_model.DataLocation(
file_path=message.data, reduction_run_id=reduction_run.pk)
data_location.save()
return reduction_run, message, instrument
def _find_run_in_database(self):
"""
Find a ReductionRun record in the database
This includes a timeout to wait for several seconds to ensure the database has received
the record in question
:return: The resulting record
"""
wait_times = [0, 1, 2, 3, 5]
results = []
for timeout in wait_times:
# Wait before attempting database access
print(f"Waiting for: {timeout}")
time.sleep(timeout)
# Check database has expected values
instrument_record = db.get_instrument(self.instrument)
results = self.database_client.data_model.ReductionRun.objects \
.filter(instrument=instrument_record.id) \
.filter(run_number=self.run_number) \
.select_related() \
.all()
try:
actual = results[0]
except IndexError:
# If no results found yet then continue
continue
# verbose values = "Completed" or "Error"
if actual.status.value == 'c' or actual.status.value == 'e':
print(
f"Job reached {actual.status.value} status after {timeout} seconds"
)
break
return results
def test_get_instrument_invalid(self):
"""
Test: None is returned
When: get_instrument is called with an instrument that does not exist
"""
actual = access.get_instrument('Fake instrument')
self.assertIsNone(actual)
def show_variables_for_run(self, instrument_name, run_number=None):
"""
Look for the applicable variables for the given run number. If none are set, return an empty
list (or QuerySet) anyway.
"""
instrument = db.get_instrument(instrument_name)
variable_run_number = 0
# If we haven't been given a run number, we should try to find it.
model = db.start_database().variable_model
if not run_number:
applicable_variables = model.InstrumentVariable.objects \
.filter(instrument_id=instrument.id) \
.order_by('-start_run')
if applicable_variables:
variable_run_number = applicable_variables[0].start_run
else:
variable_run_number = run_number
# Find variable records associated with instrument variables
variables = self.find_var_records_for_inst_vars(
instrument.id, variable_run_number)
# If we have found some variables then we want to use them by first making copies of them
# and sending them back to be used. This means we don't alter the previous set of variables.
# If we haven't found any variables, just return an empty list.
if variables:
self._update_variables(variables)
new_variables = []
for variable in variables:
new_variables.append(VariableUtils().copy_variable(variable))
return new_variables
return []
def test_get_instrument_valid(self):
"""
Test: The correct instrument object is returned
When: get_instrument is called on a valid database connection
"""
actual = access.get_instrument('GEM')
self.assertIsNotNone(actual)
self.assertEqual('GEM', actual.name)
def _find_run_in_database(self):
"""
Find a ReductionRun record in the database
This includes a timeout to wait for several seconds to ensure the database has received
the record in question
:return: The resulting record
"""
instrument = db.get_instrument(self.instrument)
return instrument.reduction_runs.filter(run_number=self.run_number)
def find_runs_in_database(self, run_number):
"""
Find all run versions in the database that relate to a given instrument and run number
:param run_number: (int) The run to search for in the database
:return: The result of the query
"""
instrument_record = db.get_instrument(self.instrument)
result = self.database.data_model.ReductionRun.objects \
.filter(instrument=instrument_record.id) \
.filter(run_number=run_number) \
.order_by('-created')
self.to_delete[run_number] = result
return result
def test_get_instrument_does_not_exist(self, mock_save):
"""
Test: A new instrument record is created
When: create is true and no matching instrument can be found in the database
Note: As we do not actually save the object, we cannot assert values of it
These are only stored when it enters the database
"""
database = access.start_database()
actual = access.get_instrument('Not an instrument', create=True)
self.assertIsNotNone(actual)
self.assertIsInstance(actual, database.data_model.Instrument)
mock_save.assert_called_once()
def show_variables_for_experiment(self, instrument_name,
experiment_reference):
"""
Look for currently set variables for the experiment.
If none are set, return an empty list (or QuerySet) anyway.
"""
instrument = db.get_instrument(instrument_name)
model = db.start_database().variable_model
ins_vars = model.InstrumentVariable.objects \
.filter(instrument_id=instrument.id) \
.filter(experiment_reference=experiment_reference)
self._update_variables(ins_vars)
return [VariableUtils().copy_variable(ins_var) for ins_var in ins_vars]
def find_run_in_database(test):
"""
Find a ReductionRun record in the database
This includes a timeout to wait for several seconds to ensure the database has received
the record in question
:return: The resulting record
"""
instrument = db.get_instrument(test.instrument_name)
if isinstance(test.run_number, list):
args = {"run_number__in": test.run_number}
else:
args = {"run_number": test.run_number}
return instrument.reduction_runs.filter(**args)
def _get_and_activate_db_inst(self, instrument_name):
"""
Gets the DB instrument record from the database, if one is not
found it instead creates and saves the record to the DB, then
returns it.
"""
# Check if the instrument is active or not in the MySQL database
instrument = db_access.get_instrument(str(instrument_name),
create=True)
# Activate the instrument if it is currently set to inactive
if not instrument.is_active:
self._logger.info("Activating %s", instrument_name)
instrument.is_active = 1
db_access.save_record(instrument)
return instrument
def get_default_variables(self, instrument_name, reduce_script=None):
"""
Creates and returns a list of variables from the reduction script on disk for the
instrument. If reduce_script is supplied, return variables using that script instead of the
one on disk.
"""
if not reduce_script:
reduce_script = self._load_reduction_vars_script(instrument_name)
reduce_vars_module = self._read_script(
reduce_script,
os.path.join(self._reduction_script_location(instrument_name),
'reduce_vars.py'))
if not reduce_vars_module:
return []
instrument = db.get_instrument(instrument_name)
variables = []
# pylint:disable=no-member
if 'standard_vars' in dir(reduce_vars_module):
variables.extend(
self._create_variables(instrument, reduce_vars_module,
reduce_vars_module.standard_vars,
False))
if 'advanced_vars' in dir(reduce_vars_module):
variables.extend(
self._create_variables(instrument, reduce_vars_module,
reduce_vars_module.advanced_vars, True))
for var in variables:
var.tracks_script = True
model = db.start_database().variable_model
applicable_variables = model.InstrumentVariable.objects \
.filter(instrument_id=instrument.id) \
.order_by('-start_run')
variable_run_number = applicable_variables[0].start_run
variables = self.find_var_records_for_inst_vars(
instrument.id, variable_run_number)
return variables
def set_variables_for_runs(self,
instrument_name,
variables,
start_run=0,
end_run=None):
"""
Given a list of variables, we set them to be the variables used for subsequent runs in the
given run range. If end_run is not supplied, these variables will be ongoing indefinitely.
If start_run is not supplied, these variables will be set for all run numbers going
backwards.
"""
instrument = db.get_instrument(instrument_name)
# Ensure that the variables we set will be the only ones used for the range given.
model = db.start_database().variable_model
applicable_variables = model.InstrumentVariable.objects \
.filter(instrument_id=instrument.id) \
.filter(start_run=start_run)
final_variables = []
if end_run:
applicable_variables = applicable_variables.filter(
start_run__lte=end_run)
# pylint: disable=no-member
after_variables = model.InstrumentVariable.objects \
.filter(instrument_id=instrument.id) \
.filter(start_run=end_run + 1) \
.order_by('start_run')
# pylint: disable=no-member
previous_variables = model.InstrumentVariable.objects \
.filter(instrument_id=instrument.id) \
.filter(start_run__lt=start_run)
if applicable_variables and not after_variables:
# The last set of applicable variables extends outside our range.
# Find the last set.
final_start = applicable_variables.order_by(
'-start_run').first().start_run
final_variables = list(
applicable_variables.filter(start_run=final_start))
applicable_variables = applicable_variables.exclude(
start_run=final_start) # Don't delete the final set.
elif not applicable_variables and not after_variables and previous_variables:
# There is a previous set that applies but doesn't start or end in the range.
# Find the last set.
final_start = previous_variables.order_by(
'-start_run').first().start_run
# Set them to apply after our variables.
# pylint: disable=expression-not-assigned,no-member
final_variables = list(
previous_variables.filter(start_run=final_start))
[
VariableUtils().copy_variable(var).save()
for var in final_variables
] # Also copy them to apply before our variables.
elif not applicable_variables and not after_variables and not previous_variables:
# There are instrument defaults which apply after our range.
final_variables = self.get_default_variables(instrument_name)
# Delete all currently saved variables that apply to the range.
map(lambda var: var.delete(), applicable_variables)
# Modify the range of the final set to after the specified range, if there is one.
for var in final_variables:
var.start_run = end_run + 1
db.save_record(var)
# Then save the new ones.
for var in variables:
var.start_run = start_run
db.save_record(var)
