def run_queue(request):
"""
Render status of queue
"""
# Get all runs that should be shown
queued_status = StatusUtils().get_queued()
processing_status = StatusUtils().get_processing()
pending_jobs = ReductionRun.objects.filter(
Q(status=queued_status)
| Q(status=processing_status)).order_by('created')
# Filter those which the user shouldn't be able to see
if USER_ACCESS_CHECKS and not request.user.is_superuser:
with ICATCache(AUTH='uows',
SESSION={'sessionid':
request.session['sessionid']}) as icat:
pending_jobs = filter(
lambda job: job.experiment.reference_number in icat.
get_associated_experiments(int(request.user.username)),
pending_jobs) # check RB numbers
pending_jobs = filter(
lambda job: job.instrument.name in icat.get_owned_instruments(
int(request.user.username)),
pending_jobs) # check instrument
# Initialise list to contain the names of user/team that started runs
started_by = []
# cycle through all filtered runs and retrieve the name of the user/team that started the run
for run in pending_jobs:
started_by.append(started_by_id_to_name(run.started_by))
# zip the run information with the user/team name to enable simultaneous iteration with django
context_dictionary = {'queue': zip(pending_jobs, started_by)}
return context_dictionary
def test_get_statuses(self):
statusList = [("Error", "get_error"), ("Completed", "get_completed"),
("Processing", "get_processing"),
("Queued", "get_queued"), ("Skipped", "get_skipped")]
statuses = []
for statusValue, utilsMember in statusList:
# e.g., make sure StatusUtils().get_error exists, and that StatusUtils().get_error().value == "Error"
self.assertTrue(hasattr(StatusUtils(), utilsMember),
"Expected StatusUtils() to have %s" % utilsMember)
statusObject = getattr(StatusUtils(), utilsMember)()
self.assertNotEqual(
statusObject, None,
"Expected a status object to be returned from %s" %
utilsMember)
self.assertEqual(
statusObject.value, statusValue,
"Expected call to %s to return Status with value %s" %
(utilsMember, statusValue))
statuses.append(statusObject)
self.assertEqual(len(list(set(statuses))), len(statuses),
"Expected all statuses to be unique")
def fail_queue(request):
# render the page
error_status = StatusUtils().get_error()
failed_jobs = ReductionRun.objects.filter(Q(status=error_status) & Q(hidden_in_failviewer=False)).order_by('-created')
context_dictionary = { 'queue' : failed_jobs
, 'status_success' : StatusUtils().get_completed()
, 'status_failed' : StatusUtils().get_error()
}
if request.method == 'POST':
# perform the specified action
action = request.POST.get("action", "default")
selectedRunString = request.POST.get("selectedRuns", [])
selectedRuns = json.loads(selectedRunString)
try:
for run in selectedRuns:
runNumber = int(run[0])
runVersion = int(run[1])
RBNumber = int(run[2])
experiment = Experiment.objects.filter(reference_number=RBNumber).first()
reductionRun = ReductionRun.objects.get(experiment=experiment, run_number=runNumber, run_version=runVersion)
if action == "hide":
reductionRun.hidden_in_failviewer = True
reductionRun.save()
elif action == "rerun":
highest_version = max([int(runL[1]) for runL in selectedRuns if int(runL[0]) == runNumber])
if runVersion != highest_version:
continue # do not run multiples of the same run
ReductionRunUtils().cancelRun(reductionRun)
reductionRun.cancel = False
new_job = ReductionRunUtils().createRetryRun(reductionRun)
try:
MessagingUtils().send_pending(new_job)
except Exception as e:
new_job.delete()
raise e
elif action == "cancel":
ReductionRunUtils().cancelRun(reductionRun)
elif action == "default":
pass
except Exception as e:
failStr = "Selected action failed: %s %s" % (type(e).__name__, e)
logger.info("Failed to carry out fail_queue action - " + failStr)
context_dictionary["message"] = failStr
return context_dictionary
def submit_runs(request, instrument=None):
"""
Handles run submission request
"""
LOGGER.info('Submitting runs')
# pylint:disable=no-member
instrument = Instrument.objects.get(name=instrument)
if request.method == 'GET':
processing_status = StatusUtils().get_processing()
queued_status = StatusUtils().get_queued()
skipped_status = StatusUtils().get_skipped()
# pylint:disable=no-member
last_run = ReductionRun.\
objects.filter(instrument=instrument).\
exclude(status=skipped_status).order_by('-run_number').first()
standard_vars = {}
advanced_vars = {}
default_variables = InstrumentVariablesUtils().get_default_variables(
instrument.name)
default_standard_variables = {}
default_advanced_variables = {}
for variable in default_variables:
if variable.is_advanced:
advanced_vars[variable.name] = variable
default_advanced_variables[variable.name] = variable
else:
standard_vars[variable.name] = variable
default_standard_variables[variable.name] = variable
# pylint:disable=no-member
context_dictionary = {
'instrument':
instrument,
'last_instrument_run':
last_run,
'processing':
ReductionRun.objects.filter(instrument=instrument,
status=processing_status),
'queued':
ReductionRun.objects.filter(instrument=instrument,
status=queued_status),
'standard_variables':
standard_vars,
'advanced_variables':
advanced_vars,
'default_standard_variables':
default_standard_variables,
'default_advanced_variables':
default_advanced_variables,
}
return context_dictionary
def run_queue(request):
# Get all runs that should be shown
queued_status = StatusUtils().get_queued()
processing_status = StatusUtils().get_processing()
pending_jobs = ReductionRun.objects.filter(Q(status=queued_status) | Q(status=processing_status)).order_by('created')
# Filter those which the user shouldn't be able to see
if USER_ACCESS_CHECKS and not request.user.is_superuser:
with ICATCache(AUTH='uows', SESSION={'sessionid':request.session['sessionid']}) as icat:
pending_jobs = filter(lambda job: job.experiment.reference_number in icat.get_associated_experiments(int(request.user.username)), pending_jobs) # check RB numbers
pending_jobs = filter(lambda job: job.instrument.name in icat.get_owned_instruments(int(request.user.username)), pending_jobs) # check instrument
context_dictionary = { 'queue' : pending_jobs }
return context_dictionary
def get_current_and_upcoming_variables(self, instrument_name):
"""
Fetches the instrument variables for:
- The next run number
- Upcoming run numbers
- Upcoming known experiments
as a tuple of (current_variables, upcoming_variables_by_run, upcoming_variables_by_experiment)
"""
instrument = InstrumentUtils().get_instrument(instrument_name)
completed_status = StatusUtils().get_completed()
# First, we find the latest run number to determine what's upcoming.
try:
latest_completed_run_number = ReductionRun.objects.filter(
instrument=instrument, run_version=0,
status=completed_status).order_by(
'-run_number').first().run_number
except AttributeError:
latest_completed_run_number = 1
# Then we find all the upcoming runs and force updating of all subsequent variables.
upcoming_run_variables = InstrumentVariable.objects.filter(
instrument=instrument,
start_run__isnull=False,
start_run__gt=latest_completed_run_number +
1).order_by('start_run')
upcoming_run_numbers = set(
[var.start_run for var in upcoming_run_variables])
[
self.show_variables_for_run(instrument_name, run_number)
for run_number in upcoming_run_numbers
]
# Get the most recent run variables.
current_variables = self.show_variables_for_run(
instrument_name, latest_completed_run_number)
if not current_variables:
# If no variables are saved, we'll use the default ones, and set them while we're at it.
current_variables = self.get_default_variables(instrument_name)
self.set_variables_for_runs(instrument_name, current_variables)
# And then select the variables for all subsequent run numbers; collect the immediate upcoming variables and all subsequent sets.
upcoming_variables_by_run = self.show_variables_for_run(
instrument_name, latest_completed_run_number + 1)
upcoming_variables_by_run += list(
InstrumentVariable.objects.filter(
instrument=instrument,
start_run__in=upcoming_run_numbers).order_by('start_run'))
# Get the upcoming experiments, and then select all variables for these experiments.
upcoming_experiments = []
with ICATCommunication() as icat:
upcoming_experiments = list(
icat.get_upcoming_experiments_for_instrument(instrument_name))
upcoming_variables_by_experiment = InstrumentVariable.objects.filter(
instrument=instrument,
experiment_reference__in=upcoming_experiments).order_by(
'experiment_reference')
return current_variables, upcoming_variables_by_run, upcoming_variables_by_experiment
def test_cancel_rerun_queued(self):
reductionRun = self.createReductionRun()
reductionRun.status = StatusUtils().get_error()
retryRun = self.createReductionRun()
reductionRun.retry_run = retryRun
with patch('reduction_variables.utils.MessagingUtils',
self.createMockMessagingUtils()):
ReductionRunUtils().cancelRun(reductionRun)
self.assertEqual(
self.cancelledRun, retryRun,
"Expected that the run to be cancelled was this one")
self.assertEqual(
retryRun.status,
StatusUtils().get_error(),
"Expected the run to have Error status, was %s " %
retryRun.status.value)
def instrument_summary(request, instrument=None):
processing_status = StatusUtils().get_processing()
queued_status = StatusUtils().get_queued()
skipped_status = StatusUtils().get_skipped()
try:
instrument_obj = Instrument.objects.get(name=instrument)
context_dictionary = {
'instrument' : instrument_obj,
'last_instrument_run' : ReductionRun.objects.filter(instrument=instrument_obj).exclude(status=skipped_status).order_by('-run_number')[0],
'processing' : ReductionRun.objects.filter(instrument=instrument_obj, status=processing_status),
'queued' : ReductionRun.objects.filter(instrument=instrument_obj, status=queued_status),
}
except Exception as e:
logger.error(e.message)
context_dictionary = {}
return context_dictionary
def createReductionRun(self):
instrument = InstrumentUtils().get_instrument("valid")
instrument.save()
experiment = Experiment(reference_number=1)
experiment.save()
reduction_run = ReductionRun(run_number=self.run_number,
instrument=instrument,
experiment=experiment,
run_version=1,
status=StatusUtils().get_queued(),
script=getValidScript('reduce.py'))
self.run_number += 1
reduction_run.save()
variables = InstrumentVariablesUtils().get_variables_for_run(
reduction_run)
VariableUtils().save_run_variables(variables, reduction_run)
return reduction_run
def test_cancel_not_queued(self):
# should not cancel a run that's not queued
statusList = [
"get_error", "get_completed", "get_processing", "get_skipped"
]
for statusMember in statusList:
status = getattr(StatusUtils(), statusMember)()
reductionRun = self.createReductionRun()
reductionRun.status = status
with patch('reduction_variables.utils.MessagingUtils',
self.createMockMessagingUtils()):
ReductionRunUtils().cancelRun(reductionRun)
self.assertEqual(
self.cancelledRun, None,
"Expected that no run was cancelled when trying %s" %
statusMember)
self.assertEqual(
reductionRun.status, status,
"Expected the run to have %s status, was %s" %
(status.value, reductionRun.status.value))
def run_list(request):
context_dictionary = {}
instruments = []
owned_instruments = []
experiments = {}
# Superuser sees everything
if request.user.is_superuser or not USER_ACCESS_CHECKS:
instrument_names = Instrument.objects.values_list('name', flat=True)
is_instrument_scientist = True
if instrument_names:
for instrument_name in instrument_names:
experiments[instrument_name] = []
instrument = Instrument.objects.get(name=instrument_name)
instrument_experiments = Experiment.objects.filter(reduction_runs__instrument=instrument).values_list('reference_number', flat=True)
for experiment in instrument_experiments:
experiments[instrument_name].append(str(experiment))
else:
with ICATCache(AUTH='uows',SESSION={'sessionid':request.session.get('sessionid')}) as icat:
instrument_names = icat.get_valid_instruments(int(request.user.username))
if instrument_names:
experiments = icat.get_valid_experiments_for_instruments(int(request.user.username), instrument_names)
owned_instruments = icat.get_owned_instruments(int(request.user.username))
is_instrument_scientist = (len(owned_instruments) > 0)
# get database status labels up front to reduce queries to database
status_error = StatusUtils().get_error()
status_queued = StatusUtils().get_queued()
status_processing = StatusUtils().get_processing()
# Keep count of the total number of runs, to preload if there aren't too many.
total_runs = 0
for instrument_name in instrument_names:
try:
instrument = Instrument.objects.get(name=instrument_name)
except:
continue
instrument_queued_runs = 0
instrument_processing_runs = 0
instrument_error_runs = 0
instrument_obj = {
'name' : instrument_name,
'experiments' : [],
'is_instrument_scientist' : is_instrument_scientist,
'is_active' : instrument.is_active,
'is_paused' : instrument.is_paused
}
if instrument_name in owned_instruments:
matching_experiments = list(set(Experiment.objects.filter(reduction_runs__instrument=instrument)))
else:
experiment_references = experiments[instrument_name] if instrument_name in experiments else []
matching_experiments = Experiment.objects.filter(reference_number__in=experiment_references)
for experiment in matching_experiments:
runs = ReductionRun.objects.filter(experiment=experiment, instrument=instrument).order_by('-created')
total_runs += runs.count()
# count how many runs are in status error, queued and processing
experiment_error_runs = runs.filter(status__exact=status_error).count()
experiment_queued_runs = runs.filter(status__exact=status_queued).count()
experiment_processing_runs = runs.filter(status__exact=status_processing).count()
# Add experiment stats to instrument
instrument_queued_runs += experiment_queued_runs
instrument_processing_runs += experiment_processing_runs
instrument_error_runs += experiment_error_runs
experiment_obj = {
'reference_number' : experiment.reference_number,
'progress_summary' : {
'processing' : experiment_processing_runs,
'queued' : experiment_queued_runs,
'error' : experiment_error_runs,
}
}
instrument_obj['experiments'].append(experiment_obj)
instrument_obj['progress_summary']= {
'processing' : instrument_processing_runs,
'queued' : instrument_queued_runs,
'error' : instrument_error_runs,
}
# Sort lists before appending
instrument_obj['experiments'] = sorted(instrument_obj['experiments'], key=lambda k: k['reference_number'], reverse=True)
instruments.append(instrument_obj)
context_dictionary['instrument_list'] = instruments
context_dictionary['preload_runs'] = (total_runs < PRELOAD_RUNS_UNDER)
if is_instrument_scientist:
context_dictionary['default_tab'] = 'run_number'
else:
context_dictionary['default_tab'] = 'experiment'
return context_dictionary
def run_confirmation(request, instrument=None):
if request.method != 'POST':
return redirect('instrument_summary', instrument=instrument.name)
# POST
instrument = Instrument.objects.get(name=instrument)
run_numbers = []
if 'run_number' in request.POST:
run_numbers.append(int(request.POST.get('run_number')))
else:
range_string = request.POST.get('run_range').split(',')
# Expand list
for item in range_string:
if '-' in item:
split_range = item.split('-')
run_numbers.extend(
range(int(split_range[0]),
int(split_range[1]) + 1)
) # because this is a range, the end bound is exclusive!
else:
run_numbers.append(int(item))
# Make sure run numbers are distinct
run_numbers = set(run_numbers)
queued_status = StatusUtils().get_queued()
queue_count = ReductionRun.objects.filter(instrument=instrument,
status=queued_status).count()
context_dictionary = {
'runs': [],
'variables': None,
'queued': queue_count,
}
# Check that RB numbers are the same
rb_number = ReductionRun.objects.filter(
instrument=instrument,
run_number__in=run_numbers).values_list('experiment__reference_number',
flat=True).distinct()
if len(rb_number) > 1:
context_dictionary[
'error'] = 'Runs span multiple experiment numbers (' + ','.join(
str(i)
for i in rb_number) + ') please select a different range.'
for run_number in run_numbers:
old_reduction_run = ReductionRun.objects.filter(
run_number=run_number).order_by('-run_version').first()
# Check old run exists
if old_reduction_run is None:
context_dictionary['error'] = "Run number " + str(
run_number) + " doesn't exist."
use_current_script = request.POST.get('use_current_script',
u"true").lower() == u"true"
if use_current_script:
script_text = InstrumentVariablesUtils().get_current_script_text(
instrument.name)[0]
default_variables = InstrumentVariablesUtils(
).get_default_variables(instrument.name)
else:
script_text = old_reduction_run.script
default_variables = old_reduction_run.run_variables.all()
new_variables = []
for key, value in request.POST.iteritems():
if 'var-' in key:
name = None
if 'var-advanced-' in key:
name = key.replace('var-advanced-', '').replace('-', ' ')
is_advanced = True
if 'var-standard-' in key:
name = key.replace('var-standard-', '').replace('-', ' ')
is_advanced = False
if name is not None:
default_var = next(
(x for x in default_variables if x.name == name), None)
if not default_var:
continue
if len(value) > InstrumentVariable._meta.get_field(
'value').max_length:
context_dictionary['error'] = 'Value given in ' + str(
name) + ' is too long.'
variable = RunVariable(name=default_var.name,
value=value,
is_advanced=is_advanced,
type=default_var.type,
help_text=default_var.help_text)
new_variables.append(variable)
if len(new_variables) == 0:
context_dictionary[
'error'] = 'No variables were found to be submitted.'
# User can choose whether to overwrite with the re-run or create new data
if request.POST.get('overwrite_checkbox') == 'on':
overwrite_previous_data = True
else:
overwrite_previous_data = False
if 'error' in context_dictionary:
return context_dictionary
run_description = request.POST.get('run_description')
new_job = ReductionRunUtils().createRetryRun(
old_reduction_run,
script=script_text,
overwrite=overwrite_previous_data,
variables=new_variables,
username=request.user.username,
description=run_description)
try:
MessagingUtils().send_pending(new_job)
context_dictionary['runs'].append(new_job)
context_dictionary['variables'] = new_variables
except Exception as e:
new_job.delete()
context_dictionary['error'] = 'Failed to send new job. (%s)' % str(
e)
return context_dictionary
def instrument_variables(request,
instrument=None,
start=0,
end=0,
experiment_reference=0):
instrument_name = instrument
start, end = int(start), int(end)
if request.method == 'POST':
# Submission to modify variables.
varList = [
t for t in request.POST.items() if t[0].startswith("var-")
] # [("var-standard-"+name, value) or ("var-advanced-"+name, value)]
newVarDict = {
"".join(t[0].split("-")[2:]): t[1]
for t in varList
} # Remove the first two prefixes from the names to give {name: value}
tracks_script = request.POST.get("track_script_checkbox") == "on"
# Which variables should we modify?
is_run_range = request.POST.get("variable-range-toggle-value",
"True") == "True"
start = int(request.POST.get("run_start", 1))
end = int(request.POST.get("run_end", None)) if request.POST.get(
"run_end", None) else None
experiment_reference = request.POST.get("experiment_reference_number",
1)
def modifyVars(oldVars, newValues):
for var in oldVars:
if var.name in newValues:
var.value = newValues[var.name]
var.tracks_script = tracks_script
if is_run_range:
# Get the variables for the first run, modify them, and set them for the given range.
instrVars = InstrumentVariablesUtils().show_variables_for_run(
instrument_name, start)
modifyVars(instrVars, newVarDict)
InstrumentVariablesUtils().set_variables_for_runs(
instrument_name, instrVars, start, end)
else:
# Get the variables for the experiment, modify them, and set them for the experiment.
instrVars = InstrumentVariablesUtils(
).show_variables_for_experiment(instrument_name,
experiment_reference)
if not instrVars:
instrVars = InstrumentVariablesUtils().get_default_variables(
instrument_name)
modifyVars(instrVars, newVarDict)
InstrumentVariablesUtils().set_variables_for_experiment(
instrument_name, instrVars, experiment_reference)
return redirect('instrument_summary', instrument=instrument_name)
else:
instrument = InstrumentUtils().get_instrument(instrument_name)
editing = (start > 0 or experiment_reference > 0)
completed_status = StatusUtils().get_completed()
processing_status = StatusUtils().get_processing()
queued_status = StatusUtils().get_queued()
try:
latest_completed_run = ReductionRun.objects.filter(
instrument=instrument, run_version=0,
status=completed_status).order_by(
'-run_number').first().run_number
except AttributeError:
latest_completed_run = 0
try:
latest_processing_run = ReductionRun.objects.filter(
instrument=instrument, run_version=0,
status=processing_status).order_by(
'-run_number').first().run_number
except AttributeError:
latest_processing_run = 0
if experiment_reference > 0:
variables = InstrumentVariablesUtils(
).show_variables_for_experiment(instrument_name,
experiment_reference)
else:
variables = InstrumentVariablesUtils().show_variables_for_run(
instrument_name, start)
if not editing or not variables:
variables = InstrumentVariablesUtils().show_variables_for_run(
instrument.name)
if not variables:
variables = InstrumentVariablesUtils().get_default_variables(
instrument.name)
editing = False
standard_vars = {}
advanced_vars = {}
for variable in variables:
if variable.is_advanced:
advanced_vars[variable.name] = variable
else:
standard_vars[variable.name] = variable
current_variables, upcoming_variables_by_run, upcoming_variables_by_experiment = InstrumentVariablesUtils(
).get_current_and_upcoming_variables(instrument.name)
upcoming_run_variables = ','.join(
list(set([str(var.start_run)
for var in upcoming_variables_by_run]))
) # Unique, comma-joined list of all start runs belonging to the upcoming variables.
default_variables = InstrumentVariablesUtils().get_default_variables(
instrument.name)
default_standard_variables = {}
default_advanced_variables = {}
for variable in default_variables:
if variable.is_advanced:
default_advanced_variables[variable.name] = variable
else:
default_standard_variables[variable.name] = variable
context_dictionary = {
'instrument':
instrument,
'last_instrument_run':
ReductionRun.objects.filter(instrument=instrument).exclude(
status=StatusUtils().get_skipped()).order_by('-run_number')[0],
'processing':
ReductionRun.objects.filter(instrument=instrument,
status=processing_status),
'queued':
ReductionRun.objects.filter(instrument=instrument,
status=queued_status),
'standard_variables':
standard_vars,
'advanced_variables':
advanced_vars,
'default_standard_variables':
default_standard_variables,
'default_advanced_variables':
default_advanced_variables,
'run_start':
start,
'run_end':
end,
'experiment_reference':
experiment_reference,
'minimum_run_start':
max(latest_completed_run, latest_processing_run),
'upcoming_run_variables':
upcoming_run_variables,
'editing':
editing,
'tracks_script':
variables[0].tracks_script,
}
return context_dictionary
def instrument_summary(request, instrument=None):
"""
Render instrument summary
"""
try:
filter_by = request.GET.get('filter', 'run')
instrument_obj = Instrument.objects.get(name=instrument)
except Instrument.DoesNotExist:
return {'message': "Instrument not found."}
try:
sort_by = request.GET.get('sort', 'run')
if sort_by == 'run':
runs = (ReductionRun.objects.only(
'status', 'last_updated', 'run_number',
'run_version').select_related('status').filter(
instrument=instrument_obj).order_by(
'-run_number', 'run_version'))
else:
runs = (ReductionRun.objects.only(
'status', 'last_updated', 'run_number',
'run_version').select_related('status').filter(
instrument=instrument_obj).order_by('-last_updated'))
if len(runs) == 0:
return {'message': "No runs found for instrument."}
context_dictionary = {
'instrument': instrument_obj,
'instrument_name': instrument_obj.name,
'runs': runs,
'last_instrument_run': runs[0],
'processing': runs.filter(status=StatusUtils().get_processing()),
'queued': runs.filter(status=StatusUtils().get_queued()),
'filtering': filter_by,
'sort': sort_by
}
if filter_by == 'experiment':
experiments_and_runs = {}
experiments = Experiment.objects.filter(reduction_runs__instrument=instrument_obj). \
order_by('-reference_number').distinct()
for experiment in experiments:
associated_runs = runs.filter(experiment=experiment). \
order_by('-created')
experiments_and_runs[experiment] = associated_runs
context_dictionary['experiments'] = experiments_and_runs
else:
max_items_per_page = request.GET.get('pagination', 10)
custom_paginator = CustomPaginator(
page_type=sort_by,
query_set=runs,
items_per_page=max_items_per_page,
page_tolerance=3,
current_page=request.GET.get('page', 1))
context_dictionary['paginator'] = custom_paginator
context_dictionary['last_page_index'] = len(
custom_paginator.page_list)
context_dictionary['max_items'] = max_items_per_page
# pylint:disable=broad-except
except Exception as exception:
LOGGER.error(exception)
return {
'message':
"An unexpected error has occurred when loading the instrument."
}
return context_dictionary
def run_confirmation(request, instrument):
"""
Handles request for user to confirm re-run
"""
if request.method != 'POST':
return redirect('instrument_summary', instrument=instrument.name)
# POST
# pylint:disable=no-member
instrument = Instrument.objects.get(name=instrument)
range_string = request.POST.get('run_range')
queued_status = StatusUtils().get_queued()
# pylint:disable=no-member
queue_count = ReductionRun.objects.filter(instrument=instrument,
status=queued_status).count()
context_dictionary = {
'runs': [],
'variables': None,
'queued': queue_count,
}
try:
run_numbers = input_processing.parse_user_run_numbers(range_string)
except SyntaxError as exception:
context_dictionary['error'] = exception.msg
return context_dictionary
# Determine user level to set a maximum limit to the number of runs that can be re-queued
if request.user.is_superuser:
max_runs = 500
elif request.user.is_staff:
max_runs = 50
else:
max_runs = 20
if len(run_numbers) > max_runs:
context_dictionary["error"] = "{0} runs were requested, but only {1} runs can be " \
"queued at a time".format(len(run_numbers), max_runs)
return context_dictionary
# Check that RB numbers are the same for the range entered
# pylint:disable=no-member
rb_number = ReductionRun.objects.filter(instrument=instrument, run_number__in=run_numbers) \
.values_list('experiment__reference_number', flat=True).distinct()
if len(rb_number) > 1:
context_dictionary['error'] = 'Runs span multiple experiment numbers ' \
'(' + ','.join(str(i) for i in rb_number) + ')' \
' please select a different range.'
return context_dictionary
for run_number in run_numbers:
# pylint:disable=no-member
matching_previous_runs_queryset = ReductionRun.objects.\
filter(instrument=instrument,
run_number=run_number).order_by('-run_version')
most_recent_previous_run = matching_previous_runs_queryset.first()
# Check old run exists
if most_recent_previous_run is None:
context_dictionary['error'] = "Run number %s hasn't been" \
"ran by autoreduction yet." % str(run_number)
# Check it is not currently queued
queued_runs = matching_previous_runs_queryset.filter(
status=queued_status).first()
if queued_runs is not None:
context_dictionary['error'] = "Run number {0} is already queued to run".\
format(queued_runs.run_number)
return context_dictionary
use_current_script = request.POST.get('use_current_script',
u"true").lower() == u"true"
if use_current_script:
script_text = InstrumentVariablesUtils().get_current_script_text(
instrument.name)[0]
default_variables = InstrumentVariablesUtils(
).get_default_variables(instrument.name)
else:
script_text = most_recent_previous_run.script
default_variables = most_recent_previous_run.run_variables.all()
new_variables = []
for key, value in list(request.POST.items()):
if 'var-' in key:
name = None
if 'var-advanced-' in key:
name = key.replace('var-advanced-', '').replace('-', ' ')
is_advanced = True
if 'var-standard-' in key:
name = key.replace('var-standard-', '').replace('-', ' ')
is_advanced = False
if name is not None:
default_var = next(
(x for x in default_variables if x.name == name), None)
if not default_var:
continue
# pylint:disable=protected-access,no-member
if len(value) > InstrumentVariable._meta.get_field(
'value').max_length:
context_dictionary['error'] = 'Value given in {} is too long.'\
.format(str(name))
variable = RunVariable(name=default_var.name,
value=value,
is_advanced=is_advanced,
type=default_var.type,
help_text=default_var.help_text)
new_variables.append(variable)
if not new_variables:
context_dictionary[
'error'] = 'No variables were found to be submitted.'
# User can choose whether to overwrite with the re-run or create new data
overwrite_previous_data = bool(
request.POST.get('overwrite_checkbox') == 'on')
if 'error' in context_dictionary:
return context_dictionary
run_description = request.POST.get('run_description')
max_desc_len = 200
if len(run_description) > max_desc_len:
context_dictionary["error"] = "The description contains {0} characters, " \
"a maximum of {1} are allowed".\
format(len(run_description), max_desc_len)
return context_dictionary
new_job = ReductionRunUtils().createRetryRun(
user_id=request.user.id,
reduction_run=most_recent_previous_run,
script=script_text,
overwrite=overwrite_previous_data,
variables=new_variables,
description=run_description)
try:
MessagingUtils().send_pending(new_job)
context_dictionary['runs'].append(new_job)
context_dictionary['variables'] = new_variables
# pylint:disable=broad-except
except Exception as exception:
new_job.delete()
context_dictionary['error'] = 'Failed to send new job. (%s)' % str(
exception)
return context_dictionary
def test_get_variables_for_run_run_number(self):
instrument = InstrumentUtils().get_instrument("valid")
variable = InstrumentVariable(
instrument=instrument,
name="test",
value="test",
is_advanced=False,
type="text",
start_run=99999,
)
variable.save()
experiment = Experiment(reference_number=1)
reduction_run = ReductionRun(run_number=100000, instrument=instrument, experiment=experiment, run_version=1, status=StatusUtils().get_queued())
variables = InstrumentVariablesUtils().get_variables_for_run(reduction_run)
self.assertNotEqual(variables, None, "Expecting some variables to be returned")
self.assertNotEqual(variables, [], "Expecting some variables to be returned")
self.assertTrue(len(variables) > 0, 'Expecting at least 1 variable returned')
self.assertEqual(variables[0].experiment_reference, None, "Not expecting experiment_reference")
self.assertEqual(variables[0].start_run, 99999, "Expecting start run to be 99999 but was %s" % variables[0].start_run)
def test_get_variables_for_run_default_variables(self):
instrument = InstrumentUtils().get_instrument("valid")
experiment = Experiment(reference_number=1)
reduction_run = ReductionRun(run_number=123, instrument=instrument, experiment=experiment, run_version=1, status=StatusUtils().get_queued())
variables = InstrumentVariablesUtils().get_variables_for_run(reduction_run)
self.assertNotEqual(variables, None, "Expecting some variables to be returned")
self.assertNotEqual(variables, [], "Expecting some variables to be returned")
self.assertTrue(len(variables) > 0, 'Expecting at least 1 variable returned')
self.assertEqual(variables[0].experiment_reference, None, "Not expecting experiment_reference")
self.assertEqual(variables[0].start_run, 1, "Expecting start run to be 1 but was %s" % variables[0].start_run)
def get_current_and_upcoming_variables(self,
instrument_name,
last_run_object=None):
"""
:param instrument_name: The name of the instrument
:param last_run_object: Optionally provide an object of the last run on the instrument
Fetches the instrument variables for:
- The next run number
- Upcoming run numbers
- Upcoming known experiments
as a tuple of
(current_variables, upcoming_variables_by_run, upcoming_variables_by_experiment)
"""
instrument = InstrumentUtils().get_instrument(instrument_name)
completed_status = StatusUtils().get_completed()
# First, we find the latest run number to determine what's upcoming.
try:
if last_run_object and last_run_object.status.value_verbose(
) == 'Completed':
latest_completed_run_number = last_run_object.run_number
else:
latest_completed_run_number = ReductionRun.objects.filter(instrument=instrument,
run_version=0,
status=completed_status)\
.order_by('-run_number').first().run_number
except AttributeError:
latest_completed_run_number = 1
# Then we find all the upcoming runs and force updating of all subsequent variables.
# pylint:disable=no-member
upcoming_run_variables = InstrumentVariable.objects.\
filter(instrument=instrument,
start_run__isnull=False,
start_run__gt=latest_completed_run_number + 1).order_by('start_run')
upcoming_run_numbers = set(
[var.start_run for var in upcoming_run_variables])
# pylint:disable=expression-not-assigned
[
self.show_variables_for_run(instrument_name, run_number)
for run_number in upcoming_run_numbers
]
# Get the most recent run variables.
current_variables = self.show_variables_for_run(
instrument_name, latest_completed_run_number)
if not current_variables:
# If no variables are saved, we'll use the default ones, and set them while we're at it.
current_variables = self.get_default_variables(instrument_name)
self.set_variables_for_runs(instrument_name, current_variables)
# And then select the variables for all subsequent run numbers;
# collect the immediate upcoming variables and all subsequent sets.
upcoming_variables_by_run = self.show_variables_for_run(
instrument_name, latest_completed_run_number + 1)
# pylint:disable=no-member
upcoming_variables_by_run += list(
InstrumentVariable.objects.filter(
instrument=instrument,
start_run__in=upcoming_run_numbers).order_by('start_run'))
# Get the upcoming experiments, and then select all variables for these experiments.
upcoming_experiments = []
with ICATCommunication() as icat:
upcoming_experiments = list(
icat.get_upcoming_experiments_for_instrument(instrument_name))
# pylint:disable=line-too-long,no-member
upcoming_variables_by_experiment = InstrumentVariable.objects.\
filter(instrument=instrument,
experiment_reference__in=upcoming_experiments).order_by('experiment_reference')
return current_variables, upcoming_variables_by_run, upcoming_variables_by_experiment
def getReductionRun(with_variables=True):
instrument = InstrumentUtils().get_instrument('valid')
experiment = Experiment(reference_number=1)
experiment.save()
reduction_run = ReductionRun(instrument=instrument, run_number=1, experiment=experiment, run_version=0, status=StatusUtils().get_queued(), script=getValidScript('reduce.py'))
reduction_run.save()
if with_variables:
variable = RunVariable(reduction_run=reduction_run,name='test',value='testvalue1',type='text',is_advanced=False)
variable.save()
reduction_run.run_variables.add(variable)
variable = RunVariable(reduction_run=reduction_run,name='advanced_test',value='testvalue2',type='text',is_advanced=True)
variable.save()
reduction_run.run_variables.add(variable)
reduction_run.save()
return reduction_run