def data_ready(self):
# Import within method to prevent cylindrical imports
from reduction_variables.utils import InstrumentVariablesUtils, VariableUtils
logger.info("Data ready for processing run %s on %s" % (str(self._data_dict['run_number']), self._data_dict['instrument']))
instrument = InstrumentUtils().get_instrument(self._data_dict['instrument'])
# Activate the instrument if it is currently set to inactive
if not instrument.is_active:
instrument.is_active = True
instrument.save()
status = StatusUtils().get_skipped() if instrument.is_paused else StatusUtils().get_queued()
last_run = ReductionRun.objects.filter(run_number=self._data_dict['run_number']).order_by('-run_version').first()
highest_version = last_run.run_version if last_run is not None else -1
experiment, experiment_created = Experiment.objects.get_or_create(reference_number=self._data_dict['rb_number'])
if experiment_created:
experiment.save()
script_text = InstrumentVariablesUtils().get_current_script_text(instrument.name)[0]
run_version = highest_version+1
reduction_run = ReductionRun( run_number=self._data_dict['run_number']
, run_version=run_version
, experiment=experiment
, instrument=instrument
, status=status
, script=script_text
)
reduction_run.save()
self._data_dict['run_version'] = reduction_run.run_version
data_location = DataLocation(file_path=self._data_dict['data'], reduction_run=reduction_run)
data_location.save()
variables = InstrumentVariablesUtils().create_variables_for_run(reduction_run)
if not variables:
logger.warning("No instrument variables found on %s for run %s" % (instrument.name, self._data_dict['run_number']))
reduction_script, arguments = ReductionRunUtils().get_script_and_arguments(reduction_run)
self._data_dict['reduction_script'] = reduction_script
self._data_dict['reduction_arguments'] = arguments
if instrument.is_paused:
logger.info("Run %s has been skipped" % self._data_dict['run_number'])
else:
self._client.send('/queue/ReductionPending', json.dumps(self._data_dict), priority=self._priority)
logger.info("Run %s ready for reduction" % self._data_dict['run_number'])
def get_current_and_upcoming_variables(self, instrument_name):
"""
Fetches the instrument variables for:
- The next run number
- Upcoming run numbers
- Upcoming known experiments
as a tuple of (current_variables, upcoming_variables_by_run, upcoming_variables_by_experiment)
"""
instrument = InstrumentUtils().get_instrument(instrument_name)
completed_status = StatusUtils().get_completed()
# First, we find the latest run number to determine what's upcoming.
try:
latest_completed_run_number = ReductionRun.objects.filter(
instrument=instrument, run_version=0,
status=completed_status).order_by(
'-run_number').first().run_number
except AttributeError:
latest_completed_run_number = 1
# Then we find all the upcoming runs and force updating of all subsequent variables.
upcoming_run_variables = InstrumentVariable.objects.filter(
instrument=instrument,
start_run__isnull=False,
start_run__gt=latest_completed_run_number +
1).order_by('start_run')
upcoming_run_numbers = set(
[var.start_run for var in upcoming_run_variables])
[
self.show_variables_for_run(instrument_name, run_number)
for run_number in upcoming_run_numbers
]
# Get the most recent run variables.
current_variables = self.show_variables_for_run(
instrument_name, latest_completed_run_number)
if not current_variables:
# If no variables are saved, we'll use the default ones, and set them while we're at it.
current_variables = self.get_default_variables(instrument_name)
self.set_variables_for_runs(instrument_name, current_variables)
# And then select the variables for all subsequent run numbers; collect the immediate upcoming variables and all subsequent sets.
upcoming_variables_by_run = self.show_variables_for_run(
instrument_name, latest_completed_run_number + 1)
upcoming_variables_by_run += list(
InstrumentVariable.objects.filter(
instrument=instrument,
start_run__in=upcoming_run_numbers).order_by('start_run'))
# Get the upcoming experiments, and then select all variables for these experiments.
upcoming_experiments = []
with ICATCommunication() as icat:
upcoming_experiments = list(
icat.get_upcoming_experiments_for_instrument(instrument_name))
upcoming_variables_by_experiment = InstrumentVariable.objects.filter(
instrument=instrument,
experiment_reference__in=upcoming_experiments).order_by(
'experiment_reference')
return current_variables, upcoming_variables_by_run, upcoming_variables_by_experiment
def show_variables_for_run(self, instrument_name, run_number=None):
"""
Look for the applicable variables for the given run number. If none are set, return an empty list (or QuerySet) anyway.
If run_number isn't given, we'll look for variables for the last run number.
"""
instrument = InstrumentUtils().get_instrument(instrument_name)
# Find the run number of the latest set of variables that apply to this run; descending order, so the first will be the most recent run number.
if run_number:
applicable_variables = InstrumentVariable.objects.filter(
instrument=instrument,
start_run__lte=run_number).order_by('-start_run')
else:
applicable_variables = InstrumentVariable.objects.filter(
instrument=instrument).order_by('-start_run')
if len(applicable_variables) != 0:
variable_run_number = applicable_variables.first().start_run
# Select all variables with that run number.
vars = list(
InstrumentVariable.objects.filter(
instrument=instrument, start_run=variable_run_number))
self._update_variables(vars)
return [VariableUtils().copy_variable(var) for var in vars]
else:
return []
def test_set_default_instrument_variables_empty(self):
variables = InstrumentVariablesUtils().set_default_instrument_variables("empty_script", 1)
instrument = InstrumentUtils().get_instrument("empty_script")
saved_variables = list(InstrumentVariable.objects.filter(instrument=instrument, start_run=1))
self.assertEqual(variables, [], 'Expecting no variables returned')
self.assertEqual(saved_variables, [], 'Expecting no variables saved')
def get_default_variables(self, instrument_name, reduce_script=None):
"""
Creates and returns a list of variables from the reduction script on disk for the instrument.
If reduce_script is supplied, return variables using that script instead of the one on disk.
"""
if not reduce_script:
reduce_script = self._load_reduction_vars_script(instrument_name)
reduce_vars_module = self._read_script(
reduce_script,
os.path.join(self._reduction_script_location(instrument_name),
'reduce_vars.py'))
if not reduce_vars_module:
return []
instrument = InstrumentUtils().get_instrument(instrument_name)
variables = []
if 'standard_vars' in dir(reduce_vars_module):
variables.extend(
self._create_variables(instrument, reduce_vars_module,
reduce_vars_module.standard_vars,
False))
if 'advanced_vars' in dir(reduce_vars_module):
variables.extend(
self._create_variables(instrument, reduce_vars_module,
reduce_vars_module.advanced_vars, True))
for var in variables:
var.tracks_script = True
return variables
def test_get_instrument_existing_instrument(self):
newName = "new_instrument_7219834839"
newInstrument, created = Instrument.objects.get_or_create(name=newName)
instrumentObject = InstrumentUtils().get_instrument(newName)
self.assertEqual(instrumentObject, newInstrument,
"Expected instrument objects to match")
def test_get_instrument_existing_instrument_case_insensitive(self):
newName = "new_instrument_3128908328"
casedName = "neW_InstrumEnt_3128908328"
newInstrument, created = Instrument.objects.get_or_create(name=newName)
instrumentObject = InstrumentUtils().get_instrument(casedName)
self.assertEqual(instrumentObject, newInstrument,
"Expected instrument objects to match")
def test_get_instrument_create_new_instrument(self):
newName = "new_instrument_4785937498"
instrumentObject = InstrumentUtils().get_instrument(newName)
self.assertNotEqual(instrumentObject, None,
"Expected an instrument object")
self.assertEqual(
instrumentObject.name, newName,
"Expected the instrument to be named %s, was %s" %
(newName, instrumentObject.name))
def show_variables_for_experiment(self, instrument_name,
experiment_reference):
""" Look for currently set variables for the experiment. If none are set, return an empty list (or QuerySet) anyway. """
instrument = InstrumentUtils().get_instrument(instrument_name)
vars = list(
InstrumentVariable.objects.filter(
instrument=instrument,
experiment_reference=experiment_reference))
self._update_variables(vars)
return [VariableUtils().copy_variable(var) for var in vars]
def createReductionRun(self):
instrument = InstrumentUtils().get_instrument("valid")
instrument.save()
experiment = Experiment(reference_number=1)
experiment.save()
reduction_run = ReductionRun(run_number=self.run_number,
instrument=instrument,
experiment=experiment,
run_version=1,
status=StatusUtils().get_queued(),
script=getValidScript('reduce.py'))
self.run_number += 1
reduction_run.save()
variables = InstrumentVariablesUtils().get_variables_for_run(
reduction_run)
VariableUtils().save_run_variables(variables, reduction_run)
return reduction_run
def test_set_default_instrument_variables_successful(self):
variables = InstrumentVariablesUtils().set_default_instrument_variables("valid", 1)
instrument = InstrumentUtils().get_instrument("valid")
saved_variables = list(InstrumentVariable.objects.filter(instrument=instrument, start_run=1))
self.assertNotEqual(variables, None, 'Expecting some variables returned')
self.assertNotEqual(saved_variables, None, 'Expecting some variables saved')
self.assertNotEqual(variables, [], 'Expecting some variables returned')
self.assertNotEqual(saved_variables, [], 'Expecting some variables saved')
self.assertTrue(len(variables) > 0, 'Expecting at least 1 variable returned')
self.assertEqual(variables[0].instrument.name, testInstrument, 'Expecting instrument to be "valid" but was %s' % variables[0].instrument)
self.assertEqual(len(variables), len(saved_variables), "Expecting all returned variables to have been saved")
def createReductionRun(self):
instrument = InstrumentUtils().get_instrument("valid")
instrument.save()
experiment = Experiment(reference_number=1)
experiment.save()
reduction_run = ReductionRun(
run_number=self.run_number,
instrument=instrument,
experiment=experiment,
run_version=1,
status=StatusUtils().get_queued(),
script=getValidScript("reduce.py"),
)
self.run_number += 1
reduction_run.save()
variables = InstrumentVariablesUtils().get_variables_for_run(reduction_run)
VariableUtils().save_run_variables(variables, reduction_run)
return reduction_run
def test_get_variables_for_run_default_variables(self):
instrument = InstrumentUtils().get_instrument("valid")
experiment = Experiment(reference_number=1)
reduction_run = ReductionRun(run_number=123, instrument=instrument, experiment=experiment, run_version=1, status=StatusUtils().get_queued())
variables = InstrumentVariablesUtils().get_variables_for_run(reduction_run)
self.assertNotEqual(variables, None, "Expecting some variables to be returned")
self.assertNotEqual(variables, [], "Expecting some variables to be returned")
self.assertTrue(len(variables) > 0, 'Expecting at least 1 variable returned')
self.assertEqual(variables[0].experiment_reference, None, "Not expecting experiment_reference")
self.assertEqual(variables[0].start_run, 1, "Expecting start run to be 1 but was %s" % variables[0].start_run)
def test_get_current_and_upcoming_variables_test_upcomming_by_experiment_not_in_ICAT(self, mock_icat_call):
mock_icat_call.return_value = []
instrument = InstrumentUtils().get_instrument("valid")
variable = InstrumentVariable(
instrument=instrument,
name="test",
value="test",
is_advanced=False,
type="text",
experiment_reference=99999,
)
variable.save()
current_variables, upcoming_variables_by_run, upcoming_variables_by_experiment = InstrumentVariablesUtils().get_current_and_upcoming_variables(testInstrument)
self.assertTrue(len(upcoming_variables_by_experiment) == 0, "Expecting no upcoming experiment variables to be returned")
def test_get_current_and_upcoming_variables_test_upcomming_by_experiment(self, mock_icat_call):
mock_icat_call.return_value = [99999]
instrument = InstrumentUtils().get_instrument("valid")
variable = InstrumentVariable(
instrument=instrument,
name="test",
value="test",
is_advanced=False,
type="text",
experiment_reference=99999,
)
variable.save()
current_variables, upcoming_variables_by_run, upcoming_variables_by_experiment = InstrumentVariablesUtils().get_current_and_upcoming_variables(testInstrument)
self.assertNotEqual(upcoming_variables_by_experiment, None, "Expecting some upcoming experiment variables to be returned")
self.assertNotEqual(upcoming_variables_by_experiment, [], "Expecting some upcoming experiment variables to be returned")
self.assertTrue(len(upcoming_variables_by_experiment) > 0, 'Expecting at least 1 upcoming experiment variable returned')
def getReductionRun(with_variables=True):
instrument = InstrumentUtils().get_instrument('valid')
experiment = Experiment(reference_number=1)
experiment.save()
reduction_run = ReductionRun(instrument=instrument, run_number=1, experiment=experiment, run_version=0, status=StatusUtils().get_queued(), script=getValidScript('reduce.py'))
reduction_run.save()
if with_variables:
variable = RunVariable(reduction_run=reduction_run,name='test',value='testvalue1',type='text',is_advanced=False)
variable.save()
reduction_run.run_variables.add(variable)
variable = RunVariable(reduction_run=reduction_run,name='advanced_test',value='testvalue2',type='text',is_advanced=True)
variable.save()
reduction_run.run_variables.add(variable)
reduction_run.save()
return reduction_run
def test_get_variables_for_run_experiment_reference(self):
instrument = InstrumentUtils().get_instrument("valid")
variable = InstrumentVariable(
instrument=instrument,
name="test",
value="test",
is_advanced=False,
type="text",
experiment_reference=99999,
)
variable.save()
experiment = Experiment(reference_number=99999)
reduction_run = ReductionRun(run_number=1, instrument=instrument, experiment=experiment, run_version=1, status=StatusUtils().get_queued())
variables = InstrumentVariablesUtils().get_variables_for_run(reduction_run)
self.assertNotEqual(variables, None, "Expecting some variables to be returned")
self.assertNotEqual(variables, [], "Expecting some variables to be returned")
self.assertTrue(len(variables) > 0, 'Expecting at least 1 variable returned')
self.assertEqual(variables[0].experiment_reference, 99999, "Expecting instrument variables to match with experiment reference number")
def instrument_variables(request,
instrument=None,
start=0,
end=0,
experiment_reference=0):
instrument_name = instrument
start, end = int(start), int(end)
if request.method == 'POST':
# Submission to modify variables.
varList = [
t for t in request.POST.items() if t[0].startswith("var-")
] # [("var-standard-"+name, value) or ("var-advanced-"+name, value)]
newVarDict = {
"".join(t[0].split("-")[2:]): t[1]
for t in varList
} # Remove the first two prefixes from the names to give {name: value}
tracks_script = request.POST.get("track_script_checkbox") == "on"
# Which variables should we modify?
is_run_range = request.POST.get("variable-range-toggle-value",
"True") == "True"
start = int(request.POST.get("run_start", 1))
end = int(request.POST.get("run_end", None)) if request.POST.get(
"run_end", None) else None
experiment_reference = request.POST.get("experiment_reference_number",
1)
def modifyVars(oldVars, newValues):
for var in oldVars:
if var.name in newValues:
var.value = newValues[var.name]
var.tracks_script = tracks_script
if is_run_range:
# Get the variables for the first run, modify them, and set them for the given range.
instrVars = InstrumentVariablesUtils().show_variables_for_run(
instrument_name, start)
modifyVars(instrVars, newVarDict)
InstrumentVariablesUtils().set_variables_for_runs(
instrument_name, instrVars, start, end)
else:
# Get the variables for the experiment, modify them, and set them for the experiment.
instrVars = InstrumentVariablesUtils(
).show_variables_for_experiment(instrument_name,
experiment_reference)
if not instrVars:
instrVars = InstrumentVariablesUtils().get_default_variables(
instrument_name)
modifyVars(instrVars, newVarDict)
InstrumentVariablesUtils().set_variables_for_experiment(
instrument_name, instrVars, experiment_reference)
return redirect('instrument_summary', instrument=instrument_name)
else:
instrument = InstrumentUtils().get_instrument(instrument_name)
editing = (start > 0 or experiment_reference > 0)
completed_status = StatusUtils().get_completed()
processing_status = StatusUtils().get_processing()
queued_status = StatusUtils().get_queued()
try:
latest_completed_run = ReductionRun.objects.filter(
instrument=instrument, run_version=0,
status=completed_status).order_by(
'-run_number').first().run_number
except AttributeError:
latest_completed_run = 0
try:
latest_processing_run = ReductionRun.objects.filter(
instrument=instrument, run_version=0,
status=processing_status).order_by(
'-run_number').first().run_number
except AttributeError:
latest_processing_run = 0
if experiment_reference > 0:
variables = InstrumentVariablesUtils(
).show_variables_for_experiment(instrument_name,
experiment_reference)
else:
variables = InstrumentVariablesUtils().show_variables_for_run(
instrument_name, start)
if not editing or not variables:
variables = InstrumentVariablesUtils().show_variables_for_run(
instrument.name)
if not variables:
variables = InstrumentVariablesUtils().get_default_variables(
instrument.name)
editing = False
standard_vars = {}
advanced_vars = {}
for variable in variables:
if variable.is_advanced:
advanced_vars[variable.name] = variable
else:
standard_vars[variable.name] = variable
current_variables, upcoming_variables_by_run, upcoming_variables_by_experiment = InstrumentVariablesUtils(
).get_current_and_upcoming_variables(instrument.name)
upcoming_run_variables = ','.join(
list(set([str(var.start_run)
for var in upcoming_variables_by_run]))
) # Unique, comma-joined list of all start runs belonging to the upcoming variables.
default_variables = InstrumentVariablesUtils().get_default_variables(
instrument.name)
default_standard_variables = {}
default_advanced_variables = {}
for variable in default_variables:
if variable.is_advanced:
default_advanced_variables[variable.name] = variable
else:
default_standard_variables[variable.name] = variable
context_dictionary = {
'instrument':
instrument,
'last_instrument_run':
ReductionRun.objects.filter(instrument=instrument).exclude(
status=StatusUtils().get_skipped()).order_by('-run_number')[0],
'processing':
ReductionRun.objects.filter(instrument=instrument,
status=processing_status),
'queued':
ReductionRun.objects.filter(instrument=instrument,
status=queued_status),
'standard_variables':
standard_vars,
'advanced_variables':
advanced_vars,
'default_standard_variables':
default_standard_variables,
'default_advanced_variables':
default_advanced_variables,
'run_start':
start,
'run_end':
end,
'experiment_reference':
experiment_reference,
'minimum_run_start':
max(latest_completed_run, latest_processing_run),
'upcoming_run_variables':
upcoming_run_variables,
'editing':
editing,
'tracks_script':
variables[0].tracks_script,
}
return context_dictionary
def get_current_and_upcoming_variables(self,
instrument_name,
last_run_object=None):
"""
:param instrument_name: The name of the instrument
:param last_run_object: Optionally provide an object of the last run on the instrument
Fetches the instrument variables for:
- The next run number
- Upcoming run numbers
- Upcoming known experiments
as a tuple of
(current_variables, upcoming_variables_by_run, upcoming_variables_by_experiment)
"""
instrument = InstrumentUtils().get_instrument(instrument_name)
completed_status = StatusUtils().get_completed()
# First, we find the latest run number to determine what's upcoming.
try:
if last_run_object and last_run_object.status.value_verbose(
) == 'Completed':
latest_completed_run_number = last_run_object.run_number
else:
latest_completed_run_number = ReductionRun.objects.filter(instrument=instrument,
run_version=0,
status=completed_status)\
.order_by('-run_number').first().run_number
except AttributeError:
latest_completed_run_number = 1
# Then we find all the upcoming runs and force updating of all subsequent variables.
# pylint:disable=no-member
upcoming_run_variables = InstrumentVariable.objects.\
filter(instrument=instrument,
start_run__isnull=False,
start_run__gt=latest_completed_run_number + 1).order_by('start_run')
upcoming_run_numbers = set(
[var.start_run for var in upcoming_run_variables])
# pylint:disable=expression-not-assigned
[
self.show_variables_for_run(instrument_name, run_number)
for run_number in upcoming_run_numbers
]
# Get the most recent run variables.
current_variables = self.show_variables_for_run(
instrument_name, latest_completed_run_number)
if not current_variables:
# If no variables are saved, we'll use the default ones, and set them while we're at it.
current_variables = self.get_default_variables(instrument_name)
self.set_variables_for_runs(instrument_name, current_variables)
# And then select the variables for all subsequent run numbers;
# collect the immediate upcoming variables and all subsequent sets.
upcoming_variables_by_run = self.show_variables_for_run(
instrument_name, latest_completed_run_number + 1)
# pylint:disable=no-member
upcoming_variables_by_run += list(
InstrumentVariable.objects.filter(
instrument=instrument,
start_run__in=upcoming_run_numbers).order_by('start_run'))
# Get the upcoming experiments, and then select all variables for these experiments.
upcoming_experiments = []
with ICATCommunication() as icat:
upcoming_experiments = list(
icat.get_upcoming_experiments_for_instrument(instrument_name))
# pylint:disable=line-too-long,no-member
upcoming_variables_by_experiment = InstrumentVariable.objects.\
filter(instrument=instrument,
experiment_reference__in=upcoming_experiments).order_by('experiment_reference')
return current_variables, upcoming_variables_by_run, upcoming_variables_by_experiment
def set_variables_for_runs(self,
instrument_name,
variables,
start_run=0,
end_run=None):
"""
Given a list of variables, we set them to be the variables used for subsequent runs in the given run range.
If end_run is not supplied, these variables will be ongoing indefinitely.
If start_run is not supplied, these variables will be set for all run numbers going backwards.
"""
instrument = InstrumentUtils().get_instrument(instrument_name)
# In this case we need to make sure that the variables we set will be the only ones used for the range given.
# If there are variables which apply after the given range ends, we want to create/modify a set to have a start_run after this end_run, with the right values.
# First, find all variables that are in the range.
applicable_variables = InstrumentVariable.objects.filter(
instrument=instrument, start_run__gte=start_run)
final_variables = []
if end_run:
applicable_variables = applicable_variables.filter(
start_run__lte=end_run)
after_variables = InstrumentVariable.objects.filter(
instrument=instrument,
start_run=end_run + 1).order_by('start_run')
previous_variables = InstrumentVariable.objects.filter(
instrument=instrument, start_run__lt=start_run)
if applicable_variables and not after_variables:
# The last set of applicable variables extends outside our range.
final_start = applicable_variables.order_by(
'-start_run').first().start_run # Find the last set.
final_variables = list(
applicable_variables.filter(start_run=final_start))
applicable_variables = applicable_variables.exclude(
start_run=final_start) # Don't delete the final set.
elif not applicable_variables and not after_variables and previous_variables:
# There is a previous set that applies but doesn't start or end in the range.
final_start = previous_variables.order_by(
'-start_run').first().start_run # Find the last set.
final_variables = list(
previous_variables.filter(start_run=final_start)
) # Set them to apply after our variables.
[
VariableUtils().copy_variable(var).save()
for var in final_variables
] # Also copy them to apply before our variables.
elif not applicable_variables and not after_variables and not previous_variables:
# There are instrument defaults which apply after our range.
final_variables = self.get_default_variables(instrument_name)
# Delete all currently saved variables that apply to the range.
map(lambda var: var.delete(), applicable_variables)
# Modify the range of the final set to after the specified range, if there is one.
for var in final_variables:
var.start_run = end_run + 1
var.save()
# Then save the new ones.
for var in variables:
var.start_run = start_run
var.save()