def reduction_complete(self, message: Message):
"""
Called when the destination queue was reduction_complete
Updates the run as complete in the database.
"""
self._logger.info("Run %s has completed reduction", message.run_number)
reduction_run = self.find_run(message)
if not reduction_run:
raise MissingReductionRunRecord(rb_number=message.rb_number,
run_number=message.run_number,
run_version=message.run_version)
if not reduction_run.status.value == 'p': # verbose value = "Processing"
raise InvalidStateException(
"An invalid attempt to complete a reduction run that wasn't"
" processing has been captured. "
f" Experiment: {message.rb_number},"
f" Run Number: {message.run_number},"
f" Run Version {message.run_version}")
reduction_run.status = self._utils.status.get_completed()
reduction_run.finished = datetime.datetime.utcnow()
reduction_run.message = message.message
reduction_run.reduction_log = message.reduction_log
reduction_run.admin_log = message.admin_log
if message.reduction_data is not None:
for location in message.reduction_data:
model = db_access.start_database().data_model
reduction_location = model \
.ReductionLocation(file_path=location,
reduction_run=reduction_run)
db_access.save_record(reduction_location)
db_access.save_record(reduction_run)
def set_cancelled(run):
""" Set a run as canceled """
run.message = "Run cancelled by user"
run.status = StatusUtils().get_error()
run.finished = datetime.datetime.utcnow()
run.retry_when = None
access.save_record(run)
def reduction_skipped(self, message: Message):
"""
Called when the destination was reduction skipped
Updates the run to Skipped status in database
Will NOT attempt re-run
"""
if message.message is not None:
self._logger.info("Run %s has been skipped - %s",
message.run_number, message.message)
else:
self._logger.info(
"Run %s has been skipped - No error message was found",
message.run_number)
reduction_run = self.find_run(message)
if not reduction_run:
raise MissingReductionRunRecord(rb_number=message.rb_number,
run_number=message.run_number,
run_version=message.run_version)
reduction_run.status = self._utils.status.get_skipped()
reduction_run.finished = datetime.datetime.utcnow()
reduction_run.message = message.message
reduction_run.reduction_log = message.reduction_log
reduction_run.admin_log = message.admin_log
db_access.save_record(reduction_run)
def _create_variables(self, instrument, script, variable_dict,
is_advanced):
""" Create variables in the database. """
variables = []
for key, value in list(variable_dict.items()):
str_value = str(value).replace('[', '').replace(']', '')
if len(str_value) > 300:
raise DataTooLong
model = db.start_database().variable_model
help_text = self._get_help_text('standard_vars', key,
instrument.name, script)
var_type = VariableUtils().get_type_string(value)
# Please note: As instrument_variable inherits from Variable, the below creates BOTH an
# an InstrumentVariable and Variable record in the database when saved. As such,
# both sets of fields are required for initialisation.
instrument_variable = model.InstrumentVariable(
name=key,
value=str_value,
type=var_type,
is_advanced=is_advanced,
help_text=help_text,
start_run=0,
instrument_id=instrument.id,
tracks_script=1)
db.save_record(instrument_variable)
variables.append(instrument_variable)
return variables
def reduction_started(self, message: Message):
"""
Called when destination queue was reduction_started.
Updates the run as started in the database.
"""
self._logger.info("Run %s has started reduction", message.run_number)
reduction_run = self.find_run(message=message)
if not reduction_run:
raise MissingReductionRunRecord(rb_number=message.rb_number,
run_number=message.run_number,
run_version=message.run_version)
if reduction_run.status.value not in [
'e', 'q'
]: # verbose values = ["Error", "Queued"]
raise InvalidStateException(
"An invalid attempt to re-start a reduction run was captured."
f" Experiment: {message.rb_number},"
f" Run Number: {message.run_number},"
f" Run Version {message.run_version}")
reduction_run.status = self._utils.status.get_processing()
reduction_run.started = datetime.datetime.utcnow()
db_access.save_record(reduction_run)
def update_variable(old_var):
""" Update the existing variables. """
old_var.keep = True
# Find the new variable from the script.
matching_vars = [
variable for variable in defaults
if old_var.name == variable.name
]
# Check whether we should and can update the old one.
if matching_vars and old_var.tracks_script:
new_var = matching_vars[0]
map(
lambda name: setattr(old_var, name, getattr(new_var, name)
),
["value", "type", "is_advanced", "help_text"]
) # Copy the new one's important attributes onto the old variable.
if save:
db.save_record(old_var)
elif not matching_vars:
# Or remove the variable if it doesn't exist any more.
if save:
db.save_record(old_var)
old_var.keep = False
return old_var
def test_save_record(self):
"""
Test: .save() is called on the provided object
When: Calling save_record()
"""
mock_record = Mock()
access.save_record(mock_record)
mock_record.save.assert_called_once()
def save_run_variables(self, instrument_vars, reduction_run):
""" Save reduction run variables in the database. """
logger.info('Saving run variables for %s',
str(reduction_run.run_number))
run_variables = map(
lambda ins_var: self.derive_run_variable(ins_var, reduction_run),
instrument_vars)
for run_variable in run_variables:
access.save_record(run_variable)
return run_variables
def log_error_and_notify(message):
"""
Helper method to log an error and save a notification
"""
logger.error(message)
model = db.start_database().data_model
notification = model.Notification(is_active=True,
is_staff_only=True,
severity='e',
message=message)
db.save_record(notification)
def _get_and_activate_db_inst(self, instrument_name):
"""
Gets the DB instrument record from the database, if one is not
found it instead creates and saves the record to the DB, then
returns it.
"""
# Check if the instrument is active or not in the MySQL database
instrument = db_access.get_instrument(str(instrument_name),
create=True)
# Activate the instrument if it is currently set to inactive
if not instrument.is_active:
self._logger.info("Activating %s", instrument_name)
instrument.is_active = 1
db_access.save_record(instrument)
return instrument
def copy_metadata(new_var):
""" Copy the source variable's metadata to the new one. """
source_var = variables[0]
model = db.start_database().variable_model
if isinstance(source_var, model.InstrumentVariable):
map(
lambda name: setattr(new_var, name,
getattr(source_var, name)),
["instrument", "experiment_reference", "start_run"])
elif isinstance(source_var, model.RunVariable):
# Create a run variable.
VariableUtils().derive_run_variable(new_var,
source_var.reduction_run)
else:
return
db.save_record(new_var)
def cancel_run(reduction_run):
"""
Try to cancel the run given, or the run that was scheduled as the next retry of the run.
When we cancel, we send a message to the backend queue processor, telling it to ignore this
run if it arrives. This is most likely through a delayed message through ActiveMQ's. We
also set statuses and error messages. If we can't do any of the above, we set the variable
(retry_run.cancel) that tells the frontend to not schedule another retry if the next run
fails.
"""
def set_cancelled(run):
""" Set a run as canceled """
run.message = "Run cancelled by user"
run.status = StatusUtils().get_error()
run.finished = datetime.datetime.utcnow()
run.retry_when = None
access.save_record(run)
# This is the queued run, send the message to queueProcessor to cancel it
if reduction_run.status == StatusUtils().get_queued():
MessagingUtils().send_cancel(reduction_run)
set_cancelled(reduction_run)
# Otherwise this run has already failed, and we're looking at a scheduled rerun of it
# We don't actually have a rerun, so just ensure the retry time is set to "Never" (None)
elif not reduction_run.retry_run:
reduction_run.retry_when = None
# This run is being queued to retry, so send the message to queueProcessor to cancel it,
# and set it as cancelled
elif reduction_run.retry_run.status == StatusUtils().get_queued():
MessagingUtils().send_cancel(reduction_run.retry_run)
set_cancelled(reduction_run.retry_run)
# We have a run that's retrying, so just make sure it doesn't retry next time
elif reduction_run.retry_run.status == StatusUtils().get_processing():
reduction_run.cancel = True
reduction_run.retry_run.cancel = True
# The retry run already completed, so do nothing
else:
pass
# save the run states we modified
access.save_record(reduction_run)
if reduction_run.retry_run:
access.save_record(reduction_run.retry_run)
def reduction_error(self, message: Message):
"""
Called when the destination was reduction_error.
Updates the run as complete in the database.
"""
if message.message:
self._logger.info("Run %s has encountered an error - %s",
message.run_number, message.message)
else:
self._logger.info(
"Run %s has encountered an error - No error message was found",
message.run_number)
reduction_run = self.find_run(message)
if not reduction_run:
raise MissingReductionRunRecord(rb_number=message.rb_number,
run_number=message.run_number,
run_version=message.run_version)
reduction_run.status = self._utils.status.get_error()
reduction_run.finished = datetime.datetime.utcnow()
reduction_run.message = message.message
reduction_run.reduction_log = message.reduction_log
reduction_run.admin_log = message.admin_log
db_access.save_record(reduction_run)
if message.retry_in is not None:
experiment = db_access.get_experiment(message.rb_number)
max_version = db_access.find_highest_run_version(
run_number=message.run_number, experiment=experiment)
# If we have already tried more than 5 times, we want to give up
# and we don't want
# to retry the run
if max_version <= 4:
self.retry_run(message.started_by, reduction_run,
message.retry_in)
else:
# Need to delete the retry_in entry from the dictionary so
# that the front end
# doesn't report a false retry instance.
message.retry_in = None
def create_retry_run(user_id,
reduction_run,
script=None,
variables=None,
delay=0):
"""
Create a run ready for re-running based on the run provided. If variables (RunVariable) are
provided, copy them and associate them with the new one, otherwise use the previous run's.
If a script (as a string) is supplied then use it, otherwise use the previous run's.
"""
model = access.start_database().data_model
# find the previous run version, so we don't create a duplicate
last_version = access.find_highest_run_version(
reduction_run.experiment, reduction_run.run_number)
# get the script to use:
script_text = script if script is not None else reduction_run.script
# create the run object and save it
new_job = model.ReductionRun(run_number=reduction_run.run_number,
run_version=last_version + 1,
run_name="",
experiment=reduction_run.experiment,
instrument=reduction_run.instrument,
script=script_text,
status=StatusUtils().get_queued(),
created=datetime.datetime.utcnow(),
last_updated=datetime.datetime.utcnow(),
message="",
started_by=user_id,
cancel=0,
hidden_in_failviewer=0,
admin_log="",
reduction_log="")
try:
access.save_record(new_job)
reduction_run.retry_run = new_job
reduction_run.retry_when = \
datetime.datetime.utcnow() + datetime.timedelta(seconds=delay if delay else 0)
access.save_record(reduction_run)
data_locations = model.DataLocation.objects \
.filter(reduction_run_id=reduction_run.id)
# copy the previous data locations
for data_location in data_locations:
new_data_location = model.DataLocation(
file_path=data_location.file_path, reduction_run=new_job)
access.save_record(new_data_location)
if variables is not None:
# associate the variables with the new run
for var in variables:
var.reduction_run = new_job
access.save_record(var)
else:
# provide variables if they aren't already
InstrumentVariablesUtils().create_variables_for_run(new_job)
return new_job
except:
new_job.delete()
raise
def data_ready(self, message: Message):
"""
Called when destination queue was data_ready.
Updates the reduction run in the database.
"""
self._logger.info("Data ready for processing run %s on %s",
message.run_number, message.instrument)
if not validate_rb_number(message.rb_number):
# rb_number is invalid so send message to skip queue and early return
message.message = f"Found non-integer RB number: {message.rb_number}"
self._logger.warning("%s. Skipping %s%s.", message.message,
message.instrument, message.run_number)
message.rb_number = 0
run_no = str(message.run_number)
instrument = self._get_and_activate_db_inst(message.instrument)
status = self._utils.status.get_skipped() if instrument.is_paused \
else self._utils.status.get_queued()
# This must be done before looking up the run version to make sure
# the record exists
experiment = db_access.get_experiment(message.rb_number, create=True)
run_version = db_access.find_highest_run_version(run_number=run_no,
experiment=experiment)
run_version += 1
message.run_version = run_version
# Get the script text for the current instrument. If the script text
# is null then send to
# error queue
script_text = self._utils. \
instrument_variable.get_current_script_text(instrument.name)[0]
if script_text is None:
self.reduction_error(message)
raise InvalidStateException(
"Script text for current instrument is null")
# Make the new reduction run with the information collected so far
# and add it into the database
reduction_run = db_records.create_reduction_run_record(
experiment=experiment,
instrument=instrument,
message=message,
run_version=run_version,
script_text=script_text,
status=status)
db_access.save_record(reduction_run)
# Create a new data location entry which has a foreign key linking
# it to the current
# reduction run. The file path itself will point to a datafile
# (e.g. "\isis\inst$\NDXWISH\Instrument\data\cycle_17_1\WISH00038774
# .nxs")
data_location = self._data_model.DataLocation(
file_path=message.data, reduction_run_id=reduction_run.id)
db_access.save_record(data_location)
# We now need to create all of the variables for the run such that
# the script can run
# through in the desired way
self._logger.info('Creating variables for run')
variables = self._utils.instrument_variable.create_variables_for_run(
reduction_run)
if not variables:
self._logger.warning(
"No instrument variables found on %s for run %s",
instrument.name, message.run_number)
self._logger.info('Getting script and arguments')
reduction_script, arguments = self._utils.reduction_run. \
get_script_and_arguments(reduction_run)
message.reduction_script = reduction_script
message.reduction_arguments = arguments
# Make sure the RB number is valid
try:
message.validate("/queue/DataReady")
except RuntimeError as validation_err:
self._logger.error("Validation error from handler: %s",
str(validation_err))
self._client.send_message('/queue/ReductionSkipped', message)
return
if instrument.is_paused:
self._logger.info("Run %s has been skipped", message.run_number)
else:
self._client.send_message('/queue/ReductionPending', message)
self._logger.info("Run %s ready for reduction", message.run_number)
def set_variables_for_runs(self,
instrument_name,
variables,
start_run=0,
end_run=None):
"""
Given a list of variables, we set them to be the variables used for subsequent runs in the
given run range. If end_run is not supplied, these variables will be ongoing indefinitely.
If start_run is not supplied, these variables will be set for all run numbers going
backwards.
"""
instrument = db.get_instrument(instrument_name)
# Ensure that the variables we set will be the only ones used for the range given.
model = db.start_database().variable_model
applicable_variables = model.InstrumentVariable.objects \
.filter(instrument_id=instrument.id) \
.filter(start_run=start_run)
final_variables = []
if end_run:
applicable_variables = applicable_variables.filter(
start_run__lte=end_run)
# pylint: disable=no-member
after_variables = model.InstrumentVariable.objects \
.filter(instrument_id=instrument.id) \
.filter(start_run=end_run + 1) \
.order_by('start_run')
# pylint: disable=no-member
previous_variables = model.InstrumentVariable.objects \
.filter(instrument_id=instrument.id) \
.filter(start_run__lt=start_run)
if applicable_variables and not after_variables:
# The last set of applicable variables extends outside our range.
# Find the last set.
final_start = applicable_variables.order_by(
'-start_run').first().start_run
final_variables = list(
applicable_variables.filter(start_run=final_start))
applicable_variables = applicable_variables.exclude(
start_run=final_start) # Don't delete the final set.
elif not applicable_variables and not after_variables and previous_variables:
# There is a previous set that applies but doesn't start or end in the range.
# Find the last set.
final_start = previous_variables.order_by(
'-start_run').first().start_run
# Set them to apply after our variables.
# pylint: disable=expression-not-assigned,no-member
final_variables = list(
previous_variables.filter(start_run=final_start))
[
VariableUtils().copy_variable(var).save()
for var in final_variables
] # Also copy them to apply before our variables.
elif not applicable_variables and not after_variables and not previous_variables:
# There are instrument defaults which apply after our range.
final_variables = self.get_default_variables(instrument_name)
# Delete all currently saved variables that apply to the range.
map(lambda var: var.delete(), applicable_variables)
# Modify the range of the final set to after the specified range, if there is one.
for var in final_variables:
var.start_run = end_run + 1
db.save_record(var)
# Then save the new ones.
for var in variables:
var.start_run = start_run
db.save_record(var)