def setUp(self):
self.var_model = access.start_database().variable_model
self.data_model = access.start_database().data_model
self.valid_variable = self.var_model.Variable(name='test',
value='value',
type='text',
is_advanced=False,
help_text='help text')
self.valid_inst_var = self.var_model.InstrumentVariable(name='test',
value='value',
is_advanced=False,
type='text',
help_text='help test',
instrument_id=4,
experiment_reference=54321,
start_run=12345,
tracks_script=1)
self.reduction_run = self.data_model.ReductionRun(run_number=1111,
run_version=0,
run_name='run name',
hidden_in_failviewer=0,
admin_log='admin log',
reduction_log='reduction log',
created=datetime.datetime.utcnow(),
last_updated=datetime.datetime.utcnow(),
experiment_id=222,
instrument_id=3,
status_id=4,
script='script',
started_by=1)
def test_start_database(self, mock_connect):
"""
Test: The database is initialised
When: start_database is called
"""
access.start_database()
mock_connect.assert_called_once()
def reduction_complete(self, message: Message):
"""
Called when the destination queue was reduction_complete
Updates the run as complete in the database.
"""
self._logger.info("Run %s has completed reduction", message.run_number)
reduction_run = self.find_run(message)
if not reduction_run:
raise MissingReductionRunRecord(rb_number=message.rb_number,
run_number=message.run_number,
run_version=message.run_version)
if not reduction_run.status.value == 'p': # verbose value = "Processing"
raise InvalidStateException(
"An invalid attempt to complete a reduction run that wasn't"
" processing has been captured. "
f" Experiment: {message.rb_number},"
f" Run Number: {message.run_number},"
f" Run Version {message.run_version}")
reduction_run.status = self._utils.status.get_completed()
reduction_run.finished = datetime.datetime.utcnow()
reduction_run.message = message.message
reduction_run.reduction_log = message.reduction_log
reduction_run.admin_log = message.admin_log
if message.reduction_data is not None:
for location in message.reduction_data:
model = db_access.start_database().data_model
reduction_location = model \
.ReductionLocation(file_path=location,
reduction_run=reduction_run)
db_access.save_record(reduction_location)
db_access.save_record(reduction_run)
def _make_pending_msg(reduction_run):
"""
Creates a dict message from the given run, ready to be sent to ReductionPending.
:param reduction_run: (ReductionRun) database object representing a reduction run / job
:return: (Message) A constructed Message object from the meta data in the reduction_run
"""
# Deferred import to avoid circular dependencies
# pylint:disable=import-outside-toplevel
from queue_processors.queue_processor.queueproc_utils \
.reduction_run_utils import ReductionRunUtils
script, arguments = ReductionRunUtils().get_script_and_arguments(
reduction_run)
# Currently only support single location
model = access.start_database().data_model
data_location = model.DataLocation.filter_by(
reduction_run_id=reduction_run.id).first()
if data_location:
data_path = data_location.file_path
else:
raise RuntimeError("No data path found for reduction run")
message = Message(run_number=reduction_run.run_number,
instrument=reduction_run.instrument.name,
rb_number=str(
reduction_run.experiment.reference_number),
data=data_path,
reduction_script=script,
reduction_arguments=arguments,
run_version=reduction_run.run_version,
facility=FACILITY)
return message
def get_script_and_arguments(reduction_run):
"""
Fetch the reduction script from the given run and return it as a string, along with a
dictionary of arguments.
"""
model = access.start_database().variable_model
run_variable_records = model.RunVariable.objects \
.filter(reduction_run_id=reduction_run.id)
standard_vars, advanced_vars = {}, {}
for run_variable in run_variable_records:
variable = model.Variable.objects.filter(
id=run_variable.variable_ptr_id).first()
value = VariableUtils().convert_variable_to_type(
variable.value, variable.type)
if variable.is_advanced:
advanced_vars[variable.name] = value
else:
standard_vars[variable.name] = value
arguments = {
'standard_vars': standard_vars,
'advanced_vars': advanced_vars
}
return reduction_run.script, arguments
def show_variables_for_run(self, instrument_name, run_number=None):
"""
Look for the applicable variables for the given run number. If none are set, return an empty
list (or QuerySet) anyway.
"""
instrument = db.get_instrument(instrument_name)
variable_run_number = 0
# If we haven't been given a run number, we should try to find it.
model = db.start_database().variable_model
if not run_number:
applicable_variables = model.InstrumentVariable.objects \
.filter(instrument_id=instrument.id) \
.order_by('-start_run')
if applicable_variables:
variable_run_number = applicable_variables[0].start_run
else:
variable_run_number = run_number
# Find variable records associated with instrument variables
variables = self.find_var_records_for_inst_vars(
instrument.id, variable_run_number)
# If we have found some variables then we want to use them by first making copies of them
# and sending them back to be used. This means we don't alter the previous set of variables.
# If we haven't found any variables, just return an empty list.
if variables:
self._update_variables(variables)
new_variables = []
for variable in variables:
new_variables.append(VariableUtils().copy_variable(variable))
return new_variables
return []
def _create_variables(self, instrument, script, variable_dict,
is_advanced):
""" Create variables in the database. """
variables = []
for key, value in list(variable_dict.items()):
str_value = str(value).replace('[', '').replace(']', '')
if len(str_value) > 300:
raise DataTooLong
model = db.start_database().variable_model
help_text = self._get_help_text('standard_vars', key,
instrument.name, script)
var_type = VariableUtils().get_type_string(value)
# Please note: As instrument_variable inherits from Variable, the below creates BOTH an
# an InstrumentVariable and Variable record in the database when saved. As such,
# both sets of fields are required for initialisation.
instrument_variable = model.InstrumentVariable(
name=key,
value=str_value,
type=var_type,
is_advanced=is_advanced,
help_text=help_text,
start_run=0,
instrument_id=instrument.id,
tracks_script=1)
db.save_record(instrument_variable)
variables.append(instrument_variable)
return variables
def _database(self):
"""
Gets a handle to the database, starting it if required
"""
if not self._cached_db:
self._cached_db = db_access.start_database()
return self._cached_db
def derive_run_variable(instrument_var, reduction_run):
""" Returns a RunJoin record for creation in the database. """
model = access.start_database().variable_model
return model.RunVariable(name=instrument_var.name,
value=instrument_var.value,
is_advanced=instrument_var.is_advanced,
type=instrument_var.type,
help_text=instrument_var.help_text,
reduction_run=reduction_run)
def test_get_experiment_create(self, mock_save):
"""
Test: An Experiment record is created
When: get_experiment is called with create option True
"""
database = access.start_database()
actual = access.get_experiment(rb_number=9999999, create=True)
self.assertIsInstance(actual, database.data_model.Experiment)
mock_save.assert_called_once()
def __init__(self, queue_listener):
self._client = queue_listener
self.status = StatusUtils()
self.instrument_variable = InstrumentVariablesUtils()
self._logger = logging.getLogger("handle_queue_message")
self.database = db_access.start_database()
self.data_model = self.database.data_model
def test_get_software_create(self, mock_save):
"""
Test: A Software record is created
When: get_software is called with create option True
"""
database = access.start_database()
actual = access.get_software(name='Fake', version='test', create=True)
self.assertIsNotNone(actual)
self.assertIsInstance(actual, database.data_model.Software)
mock_save.assert_called_once()
def create_experiment_and_instrument():
"Creates a test experiment and instrument"
db_handle = access.start_database()
data_model = db_handle.data_model
experiment, _ = data_model.Experiment.objects.get_or_create(
reference_number=1231231)
instrument, _ = data_model.Instrument.objects.get_or_create(name="ARMI",
is_active=1,
is_paused=0)
return experiment, instrument
def log_error_and_notify(message):
"""
Helper method to log an error and save a notification
"""
logger.error(message)
model = db.start_database().data_model
notification = model.Notification(is_active=True,
is_staff_only=True,
severity='e',
message=message)
db.save_record(notification)
def save_run_variables(variables, reduction_run):
""" Save reduction run variables in the database. """
model = access.start_database().variable_model
logging.info('Saving run variables for %s',
str(reduction_run.run_number))
run_variables = []
for variable in variables:
run_var = model.RunVariable(variable=variable,
reduction_run=reduction_run)
run_var.save()
run_variables.append(run_var)
return run_variables
def show_variables_for_experiment(self, instrument_name,
experiment_reference):
"""
Look for currently set variables for the experiment.
If none are set, return an empty list (or QuerySet) anyway.
"""
instrument = db.get_instrument(instrument_name)
model = db.start_database().variable_model
ins_vars = model.InstrumentVariable.objects \
.filter(instrument_id=instrument.id) \
.filter(experiment_reference=experiment_reference)
self._update_variables(ins_vars)
return [VariableUtils().copy_variable(ins_var) for ins_var in ins_vars]
def setUp(self):
self.manual_remove = ManualRemove(instrument='ARMI')
# Setup database connection so it is possible to use
# ReductionRun objects with valid meta data
db_handle = access.start_database()
self.data_model = db_handle.data_model
self.variable_model = db_handle.variable_model
self.experiment, self.instrument = create_experiment_and_instrument()
self.run1 = make_test_run(self.experiment, self.instrument, "1")
self.run2 = make_test_run(self.experiment, self.instrument, "2")
self.run3 = make_test_run(self.experiment, self.instrument, "3")
def test_get_instrument_does_not_exist(self, mock_save):
"""
Test: A new instrument record is created
When: create is true and no matching instrument can be found in the database
Note: As we do not actually save the object, we cannot assert values of it
These are only stored when it enters the database
"""
database = access.start_database()
actual = access.get_instrument('Not an instrument', create=True)
self.assertIsNotNone(actual)
self.assertIsInstance(actual, database.data_model.Instrument)
mock_save.assert_called_once()
def copy_metadata(new_var):
""" Copy the source variable's metadata to the new one. """
source_var = variables[0]
model = db.start_database().variable_model
if isinstance(source_var, model.InstrumentVariable):
map(
lambda name: setattr(new_var, name,
getattr(source_var, name)),
["instrument", "experiment_reference", "start_run"])
elif isinstance(source_var, model.RunVariable):
# Create a run variable.
VariableUtils().derive_run_variable(new_var,
source_var.reduction_run)
else:
return
db.save_record(new_var)
def find_run(self, message: Message):
""" Find a reduction run in the database. """
experiment = db_access.get_experiment(message.rb_number)
if not experiment:
raise MissingExperimentRecord(rb_number=message.rb_number,
run_number=message.run_number,
run_version=message.run_version)
self._logger.info(
'Finding a run with an experiment ID %s, run number %s and run '
'version %s', experiment.id, int(message.run_number),
int(message.run_version))
model = db_access.start_database().data_model
reduction_run = model.ReductionRun.objects \
.filter(experiment_id=experiment.id) \
.filter(run_number=int(message.run_number)) \
.filter(run_version=int(message.run_version)) \
.first()
return reduction_run
def get_default_variables(self, instrument_name, reduce_script=None):
"""
Creates and returns a list of variables from the reduction script on disk for the
instrument. If reduce_script is supplied, return variables using that script instead of the
one on disk.
"""
if not reduce_script:
reduce_script = self._load_reduction_vars_script(instrument_name)
reduce_vars_module = self._read_script(
reduce_script,
os.path.join(self._reduction_script_location(instrument_name),
'reduce_vars.py'))
if not reduce_vars_module:
return []
instrument = db.get_instrument(instrument_name)
variables = []
# pylint:disable=no-member
if 'standard_vars' in dir(reduce_vars_module):
variables.extend(
self._create_variables(instrument, reduce_vars_module,
reduce_vars_module.standard_vars,
False))
if 'advanced_vars' in dir(reduce_vars_module):
variables.extend(
self._create_variables(instrument, reduce_vars_module,
reduce_vars_module.advanced_vars, True))
for var in variables:
var.tracks_script = True
model = db.start_database().variable_model
applicable_variables = model.InstrumentVariable.objects \
.filter(instrument_id=instrument.id) \
.order_by('-start_run')
variable_run_number = applicable_variables[0].start_run
variables = self.find_var_records_for_inst_vars(
instrument.id, variable_run_number)
return variables
def make_variable_like_dict(variables: dict,
help_dict: dict) -> Dict[str, object]:
"""
Returns a dict with unsaved Variable objects.
Not ideal but better than returning a dict that needs to be kept up to date with the
Variable interface. The right solution would be to remove all of this, and is captured in
https://github.com/ISISScientificComputing/autoreduce/issues/1137
"""
variable_model = access.start_database().variable_model.Variable
result = {}
for name, value in variables.items():
result[name] = variable_model(
name=name,
value=value,
type=VariableUtils.get_type_string(value),
help_text=help_dict["name"] if name in help_dict else "")
return result
def find_var_records_for_inst_vars(instrument_id, start_run):
"""
Returns a list of the variables that are associated with a given instrument instance
e.g. return all the Variable objects that are associated with a given set of
InstrumentVariables with a start run of 123
:param instrument_id: (int) The id of the instrument to use
:param start_run: (int) The start run of the instrument variable
:return: (list) of all Variable objects matching the criteria
"""
model = db.start_database().variable_model
instrument_var_records = model.InstrumentVariable.objects \
.filter(instrument_id=instrument_id) \
.filter(start_run=start_run)
variables = []
for record in instrument_var_records:
variables.append(
model.Variable.objects.filter(
id=record.variable_ptr_id).first())
return variables
def set_variables_for_runs(self,
instrument_name,
variables,
start_run=0,
end_run=None):
"""
Given a list of variables, we set them to be the variables used for subsequent runs in the
given run range. If end_run is not supplied, these variables will be ongoing indefinitely.
If start_run is not supplied, these variables will be set for all run numbers going
backwards.
"""
instrument = db.get_instrument(instrument_name)
# Ensure that the variables we set will be the only ones used for the range given.
model = db.start_database().variable_model
applicable_variables = model.InstrumentVariable.objects \
.filter(instrument_id=instrument.id) \
.filter(start_run=start_run)
final_variables = []
if end_run:
applicable_variables = applicable_variables.filter(
start_run__lte=end_run)
# pylint: disable=no-member
after_variables = model.InstrumentVariable.objects \
.filter(instrument_id=instrument.id) \
.filter(start_run=end_run + 1) \
.order_by('start_run')
# pylint: disable=no-member
previous_variables = model.InstrumentVariable.objects \
.filter(instrument_id=instrument.id) \
.filter(start_run__lt=start_run)
if applicable_variables and not after_variables:
# The last set of applicable variables extends outside our range.
# Find the last set.
final_start = applicable_variables.order_by(
'-start_run').first().start_run
final_variables = list(
applicable_variables.filter(start_run=final_start))
applicable_variables = applicable_variables.exclude(
start_run=final_start) # Don't delete the final set.
elif not applicable_variables and not after_variables and previous_variables:
# There is a previous set that applies but doesn't start or end in the range.
# Find the last set.
final_start = previous_variables.order_by(
'-start_run').first().start_run
# Set them to apply after our variables.
# pylint: disable=expression-not-assigned,no-member
final_variables = list(
previous_variables.filter(start_run=final_start))
[
VariableUtils().copy_variable(var).save()
for var in final_variables
] # Also copy them to apply before our variables.
elif not applicable_variables and not after_variables and not previous_variables:
# There are instrument defaults which apply after our range.
final_variables = self.get_default_variables(instrument_name)
# Delete all currently saved variables that apply to the range.
map(lambda var: var.delete(), applicable_variables)
# Modify the range of the final set to after the specified range, if there is one.
for var in final_variables:
var.start_run = end_run + 1
db.save_record(var)
# Then save the new ones.
for var in variables:
var.start_run = start_run
db.save_record(var)
def setUp(self) -> None:
self.database = access.start_database()
def create_retry_run(user_id,
reduction_run,
script=None,
variables=None,
delay=0):
"""
Create a run ready for re-running based on the run provided. If variables (RunVariable) are
provided, copy them and associate them with the new one, otherwise use the previous run's.
If a script (as a string) is supplied then use it, otherwise use the previous run's.
"""
model = access.start_database().data_model
# find the previous run version, so we don't create a duplicate
last_version = access.find_highest_run_version(
reduction_run.experiment, reduction_run.run_number)
# get the script to use:
script_text = script if script is not None else reduction_run.script
# create the run object and save it
new_job = model.ReductionRun(run_number=reduction_run.run_number,
run_version=last_version + 1,
run_name="",
experiment=reduction_run.experiment,
instrument=reduction_run.instrument,
script=script_text,
status=StatusUtils().get_queued(),
created=datetime.datetime.utcnow(),
last_updated=datetime.datetime.utcnow(),
message="",
started_by=user_id,
cancel=0,
hidden_in_failviewer=0,
admin_log="",
reduction_log="")
try:
access.save_record(new_job)
reduction_run.retry_run = new_job
reduction_run.retry_when = \
datetime.datetime.utcnow() + datetime.timedelta(seconds=delay if delay else 0)
access.save_record(reduction_run)
data_locations = model.DataLocation.objects \
.filter(reduction_run_id=reduction_run.id)
# copy the previous data locations
for data_location in data_locations:
new_data_location = model.DataLocation(
file_path=data_location.file_path, reduction_run=new_job)
access.save_record(new_data_location)
if variables is not None:
# associate the variables with the new run
for var in variables:
var.reduction_run = new_job
access.save_record(var)
else:
# provide variables if they aren't already
InstrumentVariablesUtils().create_variables_for_run(new_job)
return new_job
except:
new_job.delete()
raise
def __init__(self) -> None:
self.model = db.start_database()
