def test_get_default_variables_duplicate_var_name(self):
variables = InstrumentVariablesUtils().get_default_variables('duplicate_var_reduce')
self.assertNotEqual(variables, None, 'Expecting some variables returned')
self.assertNotEqual(variables, [], 'Expecting some variables returned')
self.assertTrue(len(variables) == 2, 'Expecting 2 variables returned')
self.assertEqual(variables[0].instrument.name, 'duplicate_var_reduce', 'Expecting instrument to be "duplicate_var_reduce" but was %s' % variables[0].instrument)
def test_set_default_instrument_variables_empty(self):
variables = InstrumentVariablesUtils().set_default_instrument_variables("empty_script", 1)
instrument = InstrumentUtils().get_instrument("empty_script")
saved_variables = list(InstrumentVariable.objects.filter(instrument=instrument, start_run=1))
self.assertEqual(variables, [], 'Expecting no variables returned')
self.assertEqual(saved_variables, [], 'Expecting no variables saved')
def test_get_default_variables_successfull(self):
variables = InstrumentVariablesUtils().get_default_variables(testInstrument)
self.assertNotEqual(variables, None, 'Expecting some variables returned')
self.assertNotEqual(variables, [], 'Expecting some variables returned')
self.assertTrue(len(variables) > 0, 'Expecting at least 1 variable returned')
self.assertEqual(variables[0].instrument.name, testInstrument, 'Expecting instrument to be "valid" but was %s' % variables[0].instrument)
def test_set_default_instrument_variables_no_start_run(self):
variables = InstrumentVariablesUtils().set_default_instrument_variables("valid", None)
self.assertNotEqual(variables, None, 'Expecting some variables returned')
self.assertNotEqual(variables, [], 'Expecting some variables returned')
self.assertTrue(len(variables) > 0, 'Expecting at least 1 variable returned')
self.assertEqual(variables[0].start_run, 1, 'Expcting start_run to be set to 1 but was instead %s' % variables[0].start_run)
def run_summary(request, run_number, run_version=0):
reduction_run = ReductionRun.objects.get(run_number=run_number,
run_version=run_version)
variables = reduction_run.run_variables.all()
standard_vars = {}
advanced_vars = {}
for variable in variables:
if variable.is_advanced:
advanced_vars[variable.name] = variable
else:
standard_vars[variable.name] = variable
current_variables = InstrumentVariablesUtils().get_default_variables(
reduction_run.instrument.name)
current_standard_variables = {}
current_advanced_variables = {}
for variable in current_variables:
if variable.is_advanced:
current_advanced_variables[variable.name] = variable
else:
current_standard_variables[variable.name] = variable
context_dictionary = {
'run_number': run_number,
'run_version': run_version,
'standard_variables': standard_vars,
'advanced_variables': advanced_vars,
'default_standard_variables': standard_vars,
'default_advanced_variables': advanced_vars,
'current_standard_variables': current_standard_variables,
'current_advanced_variables': current_advanced_variables,
'instrument': reduction_run.instrument,
}
return render(request, 'snippets/run_variables.html', context_dictionary)
def test_get_default_variables_missing(self):
initial_notification = list(Notification.objects.filter(is_active=True, is_staff_only=True))
variables = InstrumentVariablesUtils().get_default_variables('missing')
updated_notification = list(Notification.objects.filter(is_active=True, is_staff_only=True))
self.assertNotEqual(variables, None, 'Expecting some variables returned')
self.assertEqual(variables, [], 'Expecting an empty array returned')
self.assertTrue(len(updated_notification) > len(initial_notification), 'Expecting a notification to be created')
def delete_instrument_variables(request,
instrument=None,
start=0,
end=0,
experiment_reference=None):
instrument_name = instrument
start, end = int(start), int(end)
# We "save" an empty list to delete the previous variables.
if experiment_reference is not None:
InstrumentVariablesUtils().set_variables_for_experiment(
instrument_name, [], experiment_reference)
else:
InstrumentVariablesUtils().set_variables_for_runs(
instrument_name, [], start, end)
return redirect('instrument_summary', instrument=instrument_name)
def test_get_current_script_text_successful(self):
script_file = os.path.join(REDUCTION_SCRIPT_BASE % testInstrument, 'reduce.py')
f = open(script_file, 'rb')
script_binary = f.read()
script, script_vars = InstrumentVariablesUtils().get_current_script_text(testInstrument)
self.assertNotEqual(script, None, "Expecting a script to be returned")
self.assertEqual(script, script_binary, "Expecting files to match")
def test_get_default_variables_pass_in_reduce_script(self):
reduction_file = os.path.join(REDUCTION_SCRIPT_BASE % testInstrument, 'reduce_vars.py')
reduce_script = open(reduction_file, "r").read()
variables = InstrumentVariablesUtils().get_default_variables(testInstrument, reduce_script)
self.assertNotEqual(variables, None, 'Expecting some variables returned')
self.assertNotEqual(variables, [], 'Expecting some variables returned')
self.assertTrue(len(variables) > 0, 'Expecting at least 1 variable returned')
self.assertEqual(variables[0].instrument.name, testInstrument, 'Expecting instrument to be "valid" but was %s' % variables[0].instrument)
def test_get_current_and_upcoming_variables_test_upcoming(self):
upcoming = InstrumentVariablesUtils().set_default_instrument_variables(testInstrument, 99999)
current_variables, upcoming_variables_by_run, upcoming_variables_by_experiment = InstrumentVariablesUtils().get_current_and_upcoming_variables(testInstrument)
self.assertNotEqual(upcoming_variables_by_run, None, "Expecting some upcoming variables to be returned")
self.assertNotEqual(upcoming_variables_by_run, [], "Expecting some upcoming variables to be returned")
self.assertTrue(len(upcoming_variables_by_run) > 0, 'Expecting at least 1 upcoming variable returned')
self.assertEqual(len(upcoming), len(upcoming_variables_by_run), "Expecting same variables to be returned as created")
def submit_runs(request, instrument=None):
"""
Handles run submission request
"""
LOGGER.info('Submitting runs')
# pylint:disable=no-member
instrument = Instrument.objects.get(name=instrument)
if request.method == 'GET':
processing_status = StatusUtils().get_processing()
queued_status = StatusUtils().get_queued()
skipped_status = StatusUtils().get_skipped()
# pylint:disable=no-member
last_run = ReductionRun.\
objects.filter(instrument=instrument).\
exclude(status=skipped_status).order_by('-run_number').first()
standard_vars = {}
advanced_vars = {}
default_variables = InstrumentVariablesUtils().get_default_variables(
instrument.name)
default_standard_variables = {}
default_advanced_variables = {}
for variable in default_variables:
if variable.is_advanced:
advanced_vars[variable.name] = variable
default_advanced_variables[variable.name] = variable
else:
standard_vars[variable.name] = variable
default_standard_variables[variable.name] = variable
# pylint:disable=no-member
context_dictionary = {
'instrument':
instrument,
'last_instrument_run':
last_run,
'processing':
ReductionRun.objects.filter(instrument=instrument,
status=processing_status),
'queued':
ReductionRun.objects.filter(instrument=instrument,
status=queued_status),
'standard_variables':
standard_vars,
'advanced_variables':
advanced_vars,
'default_standard_variables':
default_standard_variables,
'default_advanced_variables':
default_advanced_variables,
}
return context_dictionary
def test_set_default_instrument_variables_successful(self):
variables = InstrumentVariablesUtils().set_default_instrument_variables("valid", 1)
instrument = InstrumentUtils().get_instrument("valid")
saved_variables = list(InstrumentVariable.objects.filter(instrument=instrument, start_run=1))
self.assertNotEqual(variables, None, 'Expecting some variables returned')
self.assertNotEqual(saved_variables, None, 'Expecting some variables saved')
self.assertNotEqual(variables, [], 'Expecting some variables returned')
self.assertNotEqual(saved_variables, [], 'Expecting some variables saved')
self.assertTrue(len(variables) > 0, 'Expecting at least 1 variable returned')
self.assertEqual(variables[0].instrument.name, testInstrument, 'Expecting instrument to be "valid" but was %s' % variables[0].instrument)
self.assertEqual(len(variables), len(saved_variables), "Expecting all returned variables to have been saved")
def test_get_variables_for_run_default_variables(self):
instrument = InstrumentUtils().get_instrument("valid")
experiment = Experiment(reference_number=1)
reduction_run = ReductionRun(run_number=123, instrument=instrument, experiment=experiment, run_version=1, status=StatusUtils().get_queued())
variables = InstrumentVariablesUtils().get_variables_for_run(reduction_run)
self.assertNotEqual(variables, None, "Expecting some variables to be returned")
self.assertNotEqual(variables, [], "Expecting some variables to be returned")
self.assertTrue(len(variables) > 0, 'Expecting at least 1 variable returned')
self.assertEqual(variables[0].experiment_reference, None, "Not expecting experiment_reference")
self.assertEqual(variables[0].start_run, 1, "Expecting start run to be 1 but was %s" % variables[0].start_run)
def current_default_variables(request, instrument=None):
variables = InstrumentVariablesUtils().get_default_variables(instrument)
standard_vars = {}
advanced_vars = {}
for variable in variables:
if variable.is_advanced:
advanced_vars[variable.name] = variable
else:
standard_vars[variable.name] = variable
context_dictionary = {
'instrument': instrument,
'standard_variables': standard_vars,
'advanced_variables': advanced_vars,
}
return context_dictionary
def run_summary(request, instrument_name, run_number, run_version=0):
"""
Handles request to view the summary of a run
"""
# pylint:disable=no-member
instrument = Instrument.objects.get(name=instrument_name)
# pylint:disable=no-member
reduction_run = ReductionRun.objects.get(instrument=instrument,
run_number=run_number,
run_version=run_version)
variables = reduction_run.run_variables.all()
standard_vars = {}
advanced_vars = {}
for variable in variables:
if variable.is_advanced:
advanced_vars[variable.name] = variable
else:
standard_vars[variable.name] = variable
current_variables = InstrumentVariablesUtils().\
get_default_variables(reduction_run.instrument.name)
current_standard_variables = {}
current_advanced_variables = {}
for variable in current_variables:
if variable.is_advanced:
current_advanced_variables[variable.name] = variable
else:
current_standard_variables[variable.name] = variable
context_dictionary = {
'run_number': run_number,
'run_version': run_version,
'standard_variables': standard_vars,
'advanced_variables': advanced_vars,
'default_standard_variables': standard_vars,
'default_advanced_variables': advanced_vars,
'current_standard_variables': current_standard_variables,
'current_advanced_variables': current_advanced_variables,
'instrument': reduction_run.instrument,
}
return render(request, 'snippets/run_variables.html', context_dictionary)
def createReductionRun(self):
instrument = InstrumentUtils().get_instrument("valid")
instrument.save()
experiment = Experiment(reference_number=1)
experiment.save()
reduction_run = ReductionRun(run_number=self.run_number,
instrument=instrument,
experiment=experiment,
run_version=1,
status=StatusUtils().get_queued(),
script=getValidScript('reduce.py'))
self.run_number += 1
reduction_run.save()
variables = InstrumentVariablesUtils().get_variables_for_run(
reduction_run)
VariableUtils().save_run_variables(variables, reduction_run)
return reduction_run
def test_get_variables_for_run_experiment_reference(self):
instrument = InstrumentUtils().get_instrument("valid")
variable = InstrumentVariable(
instrument=instrument,
name="test",
value="test",
is_advanced=False,
type="text",
experiment_reference=99999,
)
variable.save()
experiment = Experiment(reference_number=99999)
reduction_run = ReductionRun(run_number=1, instrument=instrument, experiment=experiment, run_version=1, status=StatusUtils().get_queued())
variables = InstrumentVariablesUtils().get_variables_for_run(reduction_run)
self.assertNotEqual(variables, None, "Expecting some variables to be returned")
self.assertNotEqual(variables, [], "Expecting some variables to be returned")
self.assertTrue(len(variables) > 0, 'Expecting at least 1 variable returned')
self.assertEqual(variables[0].experiment_reference, 99999, "Expecting instrument variables to match with experiment reference number")
def instrument_summary(request, instrument):
instrument = Instrument.objects.get(name=instrument)
current_variables, upcoming_variables_by_run, upcoming_variables_by_experiment = InstrumentVariablesUtils(
).get_current_and_upcoming_variables(instrument.name)
# Create a nested dictionary for by-run
upcoming_variables_by_run_dict = {}
for variable in upcoming_variables_by_run:
if variable.start_run not in upcoming_variables_by_run_dict:
upcoming_variables_by_run_dict[variable.start_run] = {
'run_start': variable.start_run,
'run_end': 0, # We'll fill this in after
'tracks_script': variable.tracks_script,
'variables': [],
'instrument': instrument,
}
upcoming_variables_by_run_dict[variable.start_run]['variables'].append(
variable)
# Fill in the run end numbers
run_end = 0
for run_number in sorted(upcoming_variables_by_run_dict.iterkeys(),
reverse=True):
upcoming_variables_by_run_dict[run_number]['run_end'] = run_end
run_end = max(run_number - 1, 0)
current_start = current_variables[0].start_run
next_run_starts = filter(lambda start: start > current_start,
sorted(upcoming_variables_by_run_dict.keys()))
current_end = next_run_starts[0] - 1 if next_run_starts else 0
current_vars = {
'run_start': current_start,
'run_end': current_end,
'tracks_script': current_variables[0].tracks_script,
'variables': current_variables,
'instrument': instrument,
}
# Move the upcoming vars into an ordered list
upcoming_variables_by_run_ordered = []
for key in sorted(upcoming_variables_by_run_dict):
upcoming_variables_by_run_ordered.append(
upcoming_variables_by_run_dict[key])
# Create a nested dictionary for by-experiment
upcoming_variables_by_experiment_dict = {}
for variables in upcoming_variables_by_experiment:
if variables.experiment_reference not in upcoming_variables_by_experiment_dict:
upcoming_variables_by_experiment_dict[
variables.experiment_reference] = {
'experiment': variables.experiment_reference,
'variables': [],
'instrument': instrument,
}
upcoming_variables_by_experiment_dict[
variables.experiment_reference]['variables'].append(variables)
# Move the upcoming vars into an ordered list
upcoming_variables_by_experiment_ordered = []
for key in sorted(upcoming_variables_by_experiment_dict):
upcoming_variables_by_experiment_ordered.append(
upcoming_variables_by_experiment_dict[key])
sorted(upcoming_variables_by_experiment_ordered,
key=lambda r: r['experiment'])
context_dictionary = {
'instrument':
instrument,
'current_variables':
current_vars,
'upcoming_variables_by_run':
upcoming_variables_by_run_ordered,
'upcoming_variables_by_experiment':
upcoming_variables_by_experiment_ordered,
}
return render(request, 'snippets/instrument_summary_variables.html',
context_dictionary)
def instrument_variables(request,
instrument=None,
start=0,
end=0,
experiment_reference=0):
instrument_name = instrument
start, end = int(start), int(end)
if request.method == 'POST':
# Submission to modify variables.
varList = [
t for t in request.POST.items() if t[0].startswith("var-")
] # [("var-standard-"+name, value) or ("var-advanced-"+name, value)]
newVarDict = {
"".join(t[0].split("-")[2:]): t[1]
for t in varList
} # Remove the first two prefixes from the names to give {name: value}
tracks_script = request.POST.get("track_script_checkbox") == "on"
# Which variables should we modify?
is_run_range = request.POST.get("variable-range-toggle-value",
"True") == "True"
start = int(request.POST.get("run_start", 1))
end = int(request.POST.get("run_end", None)) if request.POST.get(
"run_end", None) else None
experiment_reference = request.POST.get("experiment_reference_number",
1)
def modifyVars(oldVars, newValues):
for var in oldVars:
if var.name in newValues:
var.value = newValues[var.name]
var.tracks_script = tracks_script
if is_run_range:
# Get the variables for the first run, modify them, and set them for the given range.
instrVars = InstrumentVariablesUtils().show_variables_for_run(
instrument_name, start)
modifyVars(instrVars, newVarDict)
InstrumentVariablesUtils().set_variables_for_runs(
instrument_name, instrVars, start, end)
else:
# Get the variables for the experiment, modify them, and set them for the experiment.
instrVars = InstrumentVariablesUtils(
).show_variables_for_experiment(instrument_name,
experiment_reference)
if not instrVars:
instrVars = InstrumentVariablesUtils().get_default_variables(
instrument_name)
modifyVars(instrVars, newVarDict)
InstrumentVariablesUtils().set_variables_for_experiment(
instrument_name, instrVars, experiment_reference)
return redirect('instrument_summary', instrument=instrument_name)
else:
instrument = InstrumentUtils().get_instrument(instrument_name)
editing = (start > 0 or experiment_reference > 0)
completed_status = StatusUtils().get_completed()
processing_status = StatusUtils().get_processing()
queued_status = StatusUtils().get_queued()
try:
latest_completed_run = ReductionRun.objects.filter(
instrument=instrument, run_version=0,
status=completed_status).order_by(
'-run_number').first().run_number
except AttributeError:
latest_completed_run = 0
try:
latest_processing_run = ReductionRun.objects.filter(
instrument=instrument, run_version=0,
status=processing_status).order_by(
'-run_number').first().run_number
except AttributeError:
latest_processing_run = 0
if experiment_reference > 0:
variables = InstrumentVariablesUtils(
).show_variables_for_experiment(instrument_name,
experiment_reference)
else:
variables = InstrumentVariablesUtils().show_variables_for_run(
instrument_name, start)
if not editing or not variables:
variables = InstrumentVariablesUtils().show_variables_for_run(
instrument.name)
if not variables:
variables = InstrumentVariablesUtils().get_default_variables(
instrument.name)
editing = False
standard_vars = {}
advanced_vars = {}
for variable in variables:
if variable.is_advanced:
advanced_vars[variable.name] = variable
else:
standard_vars[variable.name] = variable
current_variables, upcoming_variables_by_run, upcoming_variables_by_experiment = InstrumentVariablesUtils(
).get_current_and_upcoming_variables(instrument.name)
upcoming_run_variables = ','.join(
list(set([str(var.start_run)
for var in upcoming_variables_by_run]))
) # Unique, comma-joined list of all start runs belonging to the upcoming variables.
default_variables = InstrumentVariablesUtils().get_default_variables(
instrument.name)
default_standard_variables = {}
default_advanced_variables = {}
for variable in default_variables:
if variable.is_advanced:
default_advanced_variables[variable.name] = variable
else:
default_standard_variables[variable.name] = variable
context_dictionary = {
'instrument':
instrument,
'last_instrument_run':
ReductionRun.objects.filter(instrument=instrument).exclude(
status=StatusUtils().get_skipped()).order_by('-run_number')[0],
'processing':
ReductionRun.objects.filter(instrument=instrument,
status=processing_status),
'queued':
ReductionRun.objects.filter(instrument=instrument,
status=queued_status),
'standard_variables':
standard_vars,
'advanced_variables':
advanced_vars,
'default_standard_variables':
default_standard_variables,
'default_advanced_variables':
default_advanced_variables,
'run_start':
start,
'run_end':
end,
'experiment_reference':
experiment_reference,
'minimum_run_start':
max(latest_completed_run, latest_processing_run),
'upcoming_run_variables':
upcoming_run_variables,
'editing':
editing,
'tracks_script':
variables[0].tracks_script,
}
return context_dictionary
def run_confirmation(request, instrument):
"""
Handles request for user to confirm re-run
"""
if request.method != 'POST':
return redirect('instrument_summary', instrument=instrument.name)
# POST
# pylint:disable=no-member
instrument = Instrument.objects.get(name=instrument)
range_string = request.POST.get('run_range')
queued_status = StatusUtils().get_queued()
# pylint:disable=no-member
queue_count = ReductionRun.objects.filter(instrument=instrument,
status=queued_status).count()
context_dictionary = {
'runs': [],
'variables': None,
'queued': queue_count,
}
try:
run_numbers = input_processing.parse_user_run_numbers(range_string)
except SyntaxError as exception:
context_dictionary['error'] = exception.msg
return context_dictionary
# Determine user level to set a maximum limit to the number of runs that can be re-queued
if request.user.is_superuser:
max_runs = 500
elif request.user.is_staff:
max_runs = 50
else:
max_runs = 20
if len(run_numbers) > max_runs:
context_dictionary["error"] = "{0} runs were requested, but only {1} runs can be " \
"queued at a time".format(len(run_numbers), max_runs)
return context_dictionary
# Check that RB numbers are the same for the range entered
# pylint:disable=no-member
rb_number = ReductionRun.objects.filter(instrument=instrument, run_number__in=run_numbers) \
.values_list('experiment__reference_number', flat=True).distinct()
if len(rb_number) > 1:
context_dictionary['error'] = 'Runs span multiple experiment numbers ' \
'(' + ','.join(str(i) for i in rb_number) + ')' \
' please select a different range.'
return context_dictionary
for run_number in run_numbers:
# pylint:disable=no-member
matching_previous_runs_queryset = ReductionRun.objects.\
filter(instrument=instrument,
run_number=run_number).order_by('-run_version')
most_recent_previous_run = matching_previous_runs_queryset.first()
# Check old run exists
if most_recent_previous_run is None:
context_dictionary['error'] = "Run number %s hasn't been" \
"ran by autoreduction yet." % str(run_number)
# Check it is not currently queued
queued_runs = matching_previous_runs_queryset.filter(
status=queued_status).first()
if queued_runs is not None:
context_dictionary['error'] = "Run number {0} is already queued to run".\
format(queued_runs.run_number)
return context_dictionary
use_current_script = request.POST.get('use_current_script',
u"true").lower() == u"true"
if use_current_script:
script_text = InstrumentVariablesUtils().get_current_script_text(
instrument.name)[0]
default_variables = InstrumentVariablesUtils(
).get_default_variables(instrument.name)
else:
script_text = most_recent_previous_run.script
default_variables = most_recent_previous_run.run_variables.all()
new_variables = []
for key, value in list(request.POST.items()):
if 'var-' in key:
name = None
if 'var-advanced-' in key:
name = key.replace('var-advanced-', '').replace('-', ' ')
is_advanced = True
if 'var-standard-' in key:
name = key.replace('var-standard-', '').replace('-', ' ')
is_advanced = False
if name is not None:
default_var = next(
(x for x in default_variables if x.name == name), None)
if not default_var:
continue
# pylint:disable=protected-access,no-member
if len(value) > InstrumentVariable._meta.get_field(
'value').max_length:
context_dictionary['error'] = 'Value given in {} is too long.'\
.format(str(name))
variable = RunVariable(name=default_var.name,
value=value,
is_advanced=is_advanced,
type=default_var.type,
help_text=default_var.help_text)
new_variables.append(variable)
if not new_variables:
context_dictionary[
'error'] = 'No variables were found to be submitted.'
# User can choose whether to overwrite with the re-run or create new data
overwrite_previous_data = bool(
request.POST.get('overwrite_checkbox') == 'on')
if 'error' in context_dictionary:
return context_dictionary
run_description = request.POST.get('run_description')
max_desc_len = 200
if len(run_description) > max_desc_len:
context_dictionary["error"] = "The description contains {0} characters, " \
"a maximum of {1} are allowed".\
format(len(run_description), max_desc_len)
return context_dictionary
new_job = ReductionRunUtils().createRetryRun(
user_id=request.user.id,
reduction_run=most_recent_previous_run,
script=script_text,
overwrite=overwrite_previous_data,
variables=new_variables,
description=run_description)
try:
MessagingUtils().send_pending(new_job)
context_dictionary['runs'].append(new_job)
context_dictionary['variables'] = new_variables
# pylint:disable=broad-except
except Exception as exception:
new_job.delete()
context_dictionary['error'] = 'Failed to send new job. (%s)' % str(
exception)
return context_dictionary
def createRetryRun(self,
reductionRun,
overwrite=None,
script=None,
variables=None,
delay=0,
username=None,
description=''):
"""
Create a run ready for re-running based on the run provided.
If variables (RunVariable) are provided, copy them and associate them with the new one, otherwise use the previous run's.
If a script (as a string) is supplied then use it, otherwise use the previous run's.
"""
from reduction_variables.utils import InstrumentVariablesUtils, VariableUtils
run_last_updated = reductionRun.last_updated
if username == 'super':
username = 1
# find the previous run version, so we don't create a duplicate
last_version = -1
for run in ReductionRun.objects.filter(
experiment=reductionRun.experiment,
run_number=reductionRun.run_number):
last_version = max(last_version, run.run_version)
try:
# get the script to use:
script_text = script if script is not None else reductionRun.script
# create the run object and save it
new_job = ReductionRun(instrument=reductionRun.instrument,
run_number=reductionRun.run_number,
run_name=description,
run_version=last_version + 1,
experiment=reductionRun.experiment,
started_by=username,
status=StatusUtils().get_queued(),
script=script_text,
overwrite=overwrite)
new_job.save()
reductionRun.retry_run = new_job
reductionRun.retry_when = timezone.now().replace(
microsecond=0) + datetime.timedelta(
seconds=delay if delay else 0)
reductionRun.save()
ReductionRun.objects.filter(id=reductionRun.id).update(
last_updated=run_last_updated)
# copy the previous data locations
for data_location in reductionRun.data_location.all():
new_data_location = DataLocation(
file_path=data_location.file_path, reduction_run=new_job)
new_data_location.save()
new_job.data_location.add(new_data_location)
if variables is not None:
# associate the variables with the new run
for var in variables:
var.reduction_run = new_job
var.save()
else:
# provide variables if they aren't already
InstrumentVariablesUtils().create_variables_for_run(new_job)
return new_job
except Exception as e:
import traceback
logger.error(traceback.format_exc())
logger.error(e)
new_job.delete()
raise
def test_get_current_and_upcoming_variables_test_upcomming_by_experiment_not_in_ICAT(self, mock_icat_call):
mock_icat_call.return_value = []
instrument = InstrumentUtils().get_instrument("valid")
variable = InstrumentVariable(
instrument=instrument,
name="test",
value="test",
is_advanced=False,
type="text",
experiment_reference=99999,
)
variable.save()
current_variables, upcoming_variables_by_run, upcoming_variables_by_experiment = InstrumentVariablesUtils().get_current_and_upcoming_variables(testInstrument)
self.assertTrue(len(upcoming_variables_by_experiment) == 0, "Expecting no upcoming experiment variables to be returned")
def run_confirmation(request, instrument=None):
if request.method != 'POST':
return redirect('instrument_summary', instrument=instrument.name)
# POST
instrument = Instrument.objects.get(name=instrument)
run_numbers = []
if 'run_number' in request.POST:
run_numbers.append(int(request.POST.get('run_number')))
else:
range_string = request.POST.get('run_range').split(',')
# Expand list
for item in range_string:
if '-' in item:
split_range = item.split('-')
run_numbers.extend(
range(int(split_range[0]),
int(split_range[1]) + 1)
) # because this is a range, the end bound is exclusive!
else:
run_numbers.append(int(item))
# Make sure run numbers are distinct
run_numbers = set(run_numbers)
queued_status = StatusUtils().get_queued()
queue_count = ReductionRun.objects.filter(instrument=instrument,
status=queued_status).count()
context_dictionary = {
'runs': [],
'variables': None,
'queued': queue_count,
}
# Check that RB numbers are the same
rb_number = ReductionRun.objects.filter(
instrument=instrument,
run_number__in=run_numbers).values_list('experiment__reference_number',
flat=True).distinct()
if len(rb_number) > 1:
context_dictionary[
'error'] = 'Runs span multiple experiment numbers (' + ','.join(
str(i)
for i in rb_number) + ') please select a different range.'
for run_number in run_numbers:
old_reduction_run = ReductionRun.objects.filter(
run_number=run_number).order_by('-run_version').first()
# Check old run exists
if old_reduction_run is None:
context_dictionary['error'] = "Run number " + str(
run_number) + " doesn't exist."
use_current_script = request.POST.get('use_current_script',
u"true").lower() == u"true"
if use_current_script:
script_text = InstrumentVariablesUtils().get_current_script_text(
instrument.name)[0]
default_variables = InstrumentVariablesUtils(
).get_default_variables(instrument.name)
else:
script_text = old_reduction_run.script
default_variables = old_reduction_run.run_variables.all()
new_variables = []
for key, value in request.POST.iteritems():
if 'var-' in key:
name = None
if 'var-advanced-' in key:
name = key.replace('var-advanced-', '').replace('-', ' ')
is_advanced = True
if 'var-standard-' in key:
name = key.replace('var-standard-', '').replace('-', ' ')
is_advanced = False
if name is not None:
default_var = next(
(x for x in default_variables if x.name == name), None)
if not default_var:
continue
if len(value) > InstrumentVariable._meta.get_field(
'value').max_length:
context_dictionary['error'] = 'Value given in ' + str(
name) + ' is too long.'
variable = RunVariable(name=default_var.name,
value=value,
is_advanced=is_advanced,
type=default_var.type,
help_text=default_var.help_text)
new_variables.append(variable)
if len(new_variables) == 0:
context_dictionary[
'error'] = 'No variables were found to be submitted.'
# User can choose whether to overwrite with the re-run or create new data
if request.POST.get('overwrite_checkbox') == 'on':
overwrite_previous_data = True
else:
overwrite_previous_data = False
if 'error' in context_dictionary:
return context_dictionary
run_description = request.POST.get('run_description')
new_job = ReductionRunUtils().createRetryRun(
old_reduction_run,
script=script_text,
overwrite=overwrite_previous_data,
variables=new_variables,
username=request.user.username,
description=run_description)
try:
MessagingUtils().send_pending(new_job)
context_dictionary['runs'].append(new_job)
context_dictionary['variables'] = new_variables
except Exception as e:
new_job.delete()
context_dictionary['error'] = 'Failed to send new job. (%s)' % str(
e)
return context_dictionary
def test_get_current_and_upcoming_variables_test_current(self):
current_variables, upcoming_variables_by_run, upcoming_variables_by_experiment = InstrumentVariablesUtils().get_current_and_upcoming_variables(testInstrument)
self.assertNotEqual(current_variables, None, "Expecting some current variables to be returned")
self.assertNotEqual(current_variables, [], "Expecting some current variables to be returned")
self.assertTrue(len(current_variables) > 0, 'Expecting at least 1 current variable returned')
def test_get_default_variables_empty(self):
variables = InstrumentVariablesUtils().get_default_variables('empty_script')
self.assertNotEqual(variables, None, 'Expecting some variables returned')
self.assertTrue(len(variables) == 0, 'Expecting an empty array returned')
def test_get_current_script_text_missing(self):
script, script_vars = InstrumentVariablesUtils().get_current_script_text('missing')
self.assertEqual(script, None, "Expecting script to be None")
self.assertEqual(script_vars, None, "Expecting script_vars to be None")
def test_get_current_and_upcoming_variables_test_upcomming_by_experiment(self, mock_icat_call):
mock_icat_call.return_value = [99999]
instrument = InstrumentUtils().get_instrument("valid")
variable = InstrumentVariable(
instrument=instrument,
name="test",
value="test",
is_advanced=False,
type="text",
experiment_reference=99999,
)
variable.save()
current_variables, upcoming_variables_by_run, upcoming_variables_by_experiment = InstrumentVariablesUtils().get_current_and_upcoming_variables(testInstrument)
self.assertNotEqual(upcoming_variables_by_experiment, None, "Expecting some upcoming experiment variables to be returned")
self.assertNotEqual(upcoming_variables_by_experiment, [], "Expecting some upcoming experiment variables to be returned")
self.assertTrue(len(upcoming_variables_by_experiment) > 0, 'Expecting at least 1 upcoming experiment variable returned')
def preview_script(request,
instrument=None,
run_number=0,
experiment_reference=0):
# Can't use login decorator as this is requested via AJAX; need to return an error message on failure.
if not has_valid_login(request):
redirect_response = handle_redirect(request)
if request.method == 'GET':
return redirect_response
else:
error = {'redirect_url': redirect_response.url}
return HttpResponseForbidden(json.dumps(error))
# Make our own little function to use the permissions decorator on; if we catch a PermissionDenied, we should give a 403 error.
# We also don't want to check the instrument in this case, since run-specific scripts ought to be viewable without owning the instrument.
@check_permissions
def permission_test(request, run_number=0, experiment_reference=0):
pass
try:
permission_test(request, run_number, experiment_reference)
except PermissionDenied:
return HttpResponseForbidden()
# Find the reduction run to get the script for.
if request.method == 'GET':
reduction_run = ReductionRun.objects.filter(run_number=run_number)
use_current_script = False
elif request.method == 'POST':
lookup_run_number = request.POST.get('run_number', None)
lookup_run_version = request.POST.get('run_version', None)
use_current_script = request.POST.get(
'use_current_script', default=u"false").lower() == u"true"
reduction_run = ReductionRun.objects.filter(
run_number=lookup_run_number, run_version=lookup_run_version)
# Get the script text and variables from the given run; note the differing types of the variables in the two cases, which don't matter (we only access the parent Variable attributes).
if reduction_run and not use_current_script:
script_text = reduction_run[0].script
script_variables = reduction_run[0].run_variables.all(
) # [InstrumentVariable]
else:
script_text = InstrumentVariablesUtils().get_current_script_text(
instrument)[0]
script_variables = InstrumentVariablesUtils().show_variables_for_run(
instrument) # [RunVariable]
def format_header(string):
# Gives a #-enclosed string that looks header-y
lines = ["#" * (len(string) + 8)] * 4
lines[2:2] = ["##" + " " * (len(string) + 4) + "##"] * 3
lines[3] = lines[3][:4] + string + lines[3][-4:]
lines += [""] * 1
return lines
def format_class(variables, name, indent):
# Gives a Python class declaration with variable dicts as required.
lines = ["class " + name + ":"]
standard_vars, advanced_vars = filter(lambda var: not var.is_advanced,
variables), filter(
lambda var: var.is_advanced,
variables)
def make_dict(variables, name):
var_dict = {
v.name:
VariableUtils().convert_variable_to_type(v.value, v.type)
for v in variables
}
return indent + name + " = " + str(var_dict)
lines.append(make_dict(standard_vars, "standard_vars"))
lines.append(make_dict(advanced_vars, "advanced_vars"))
lines += [""] * 3
return lines
def replace_variables(text):
# Find the import/s for the reduction variables and remove them.
lines = text.split("\n")
imports = filter(
lambda line: line.lstrip().startswith("import") or line.lstrip().
startswith("from"), lines)
import_vars = filter(lambda line: "reduce_vars" in line, imports)
lines = [line for line in lines if line not in import_vars]
# Print the header for the reduce.py script
lines = format_header("Reduction script - reduce.py") + lines
# Figure out space/tabs
indent = " " # Defaults to PEP 8 standard!
if filter(lambda line: line.startswith("\t"), lines):
indent = "\t"
if import_vars:
# Assume the import is of the form 'import reduce_vars as {name}' or 'import reduce_vars'
classname = import_vars[0].rstrip().split(" ")[-1]
# Print the variables and a header for them
lines = format_header("Reduction variables") + format_class(
script_variables, classname, indent) + lines
return "\n".join(lines)
new_script_text = replace_variables(script_text)
response = HttpResponse(content_type='application/x-python')
response['Content-Disposition'] = 'attachment; filename=reduce.py'
response.write(new_script_text)
return response
def createRetryRun(user_id, reduction_run, overwrite=None, script=None,
variables=None, delay=0, description=''):
"""
Create a run ready for re-running based on the run provided.
If variables (RunVariable) are provided, copy them and associate
them with the new one, otherwise use the previous run's.
If a script (as a string) is supplied then use it, otherwise use the previous run's.
"""
from reduction_variables.utils import InstrumentVariablesUtils
run_last_updated = reduction_run.last_updated
# find the previous run version, so we don't create a duplicate
last_version = -1
# pylint:disable=no-member
previous_run = ReductionRun.objects.filter(experiment=reduction_run.experiment,
run_number=reduction_run.run_number) \
.order_by("-run_version").first()
last_version = previous_run.run_version
# get the script to use:
script_text = script if script is not None else reduction_run.script
# create the run object and save it
new_job = ReductionRun(instrument=reduction_run.instrument,
run_number=reduction_run.run_number,
run_name=description,
run_version=last_version + 1,
experiment=reduction_run.experiment,
started_by=user_id,
status=StatusUtils().get_queued(),
script=script_text,
overwrite=overwrite)
# Check record is safe to save
try:
new_job.full_clean()
# pylint:disable=catching-non-exception
except django.core.exceptions as exception:
LOGGER.error(traceback.format_exc())
LOGGER.error(exception)
raise
# Attempt to save
try:
new_job.save()
except ValueError as exception:
# This usually indicates a F.K. constraint mismatch. Maybe we didn't get a record in?
LOGGER.error(traceback.format_exc())
LOGGER.error(exception)
raise
reduction_run.retry_run = new_job
reduction_run.retry_when = timezone.now().replace(microsecond=0) + datetime.timedelta(
seconds=delay if delay else 0)
reduction_run.save()
# pylint:disable=no-member
ReductionRun.objects.filter(id=reduction_run.id).update(last_updated=run_last_updated)
# copy the previous data locations
# pylint:disable=no-member
for data_location in reduction_run.data_location.all():
new_data_location = DataLocation(file_path=data_location.file_path,
reduction_run=new_job)
new_data_location.save()
new_job.data_location.add(new_data_location)
if variables is not None:
# associate the variables with the new run
for var in variables:
var.reduction_run = new_job
var.save()
else:
# provide variables if they aren't already
InstrumentVariablesUtils().create_variables_for_run(new_job)
return new_job