def fail_queue(request):
# render the page
error_status = StatusUtils().get_error()
failed_jobs = ReductionRun.objects.filter(Q(status=error_status) & Q(hidden_in_failviewer=False)).order_by('-created')
context_dictionary = { 'queue' : failed_jobs
, 'status_success' : StatusUtils().get_completed()
, 'status_failed' : StatusUtils().get_error()
}
if request.method == 'POST':
# perform the specified action
action = request.POST.get("action", "default")
selectedRunString = request.POST.get("selectedRuns", [])
selectedRuns = json.loads(selectedRunString)
try:
for run in selectedRuns:
runNumber = int(run[0])
runVersion = int(run[1])
RBNumber = int(run[2])
experiment = Experiment.objects.filter(reference_number=RBNumber).first()
reductionRun = ReductionRun.objects.get(experiment=experiment, run_number=runNumber, run_version=runVersion)
if action == "hide":
reductionRun.hidden_in_failviewer = True
reductionRun.save()
elif action == "rerun":
highest_version = max([int(runL[1]) for runL in selectedRuns if int(runL[0]) == runNumber])
if runVersion != highest_version:
continue # do not run multiples of the same run
ReductionRunUtils().cancelRun(reductionRun)
reductionRun.cancel = False
new_job = ReductionRunUtils().createRetryRun(reductionRun)
try:
MessagingUtils().send_pending(new_job)
except Exception as e:
new_job.delete()
raise e
elif action == "cancel":
ReductionRunUtils().cancelRun(reductionRun)
elif action == "default":
pass
except Exception as e:
failStr = "Selected action failed: %s %s" % (type(e).__name__, e)
logger.info("Failed to carry out fail_queue action - " + failStr)
context_dictionary["message"] = failStr
return context_dictionary
def test_createRetryRun_duplicate(self):
reductionRun = self.createReductionRun()
retryRun1 = ReductionRunUtils().createRetryRun(reductionRun)
retryRun2 = ReductionRunUtils().createRetryRun(reductionRun)
self.assertEqual(
retryRun1.run_version + 1, retryRun2.run_version,
"Expected run version to be correctly incremented, was %i vs %i" %
(retryRun1.run_version, retryRun2.run_version))
def fail_queue(request):
"""
Render status of failed queue
"""
# render the page
error_status = STATUS.get_error()
failed_jobs = ReductionRun.objects.filter(
Q(status=error_status)
& Q(hidden_in_failviewer=False)).order_by('-created')
context_dictionary = {
'queue': failed_jobs,
'status_success': STATUS.get_completed(),
'status_failed': STATUS.get_error()
}
if request.method == 'POST':
# perform the specified action
action = request.POST.get("action", "default")
selected_run_string = request.POST.get("selectedRuns", [])
selected_runs = json.loads(selected_run_string)
try:
for run in selected_runs:
run_number = int(run[0])
run_version = int(run[1])
reduction_run = failed_jobs.get(run_number=run_number,
run_version=run_version)
if action == "hide":
reduction_run.hidden_in_failviewer = True
reduction_run.save()
elif action == "rerun":
highest_version = max([
int(runL[1]) for runL in selected_runs
if int(runL[0]) == run_number
])
if run_version != highest_version:
continue # do not run multiples of the same run
ReductionRunUtils.send_retry_message_same_args(
request.user.id, reduction_run)
elif action == "default":
pass
# pylint:disable=broad-except
except Exception as exception:
fail_str = "Selected action failed: %s %s" % (
type(exception).__name__, exception)
LOGGER.info("Failed to carry out fail_queue action - %s", fail_str)
context_dictionary["message"] = fail_str
return context_dictionary
def retryRun(self, reductionRun, retryIn):
if (reductionRun.cancel):
logger.info("Cancelling run retry")
return
logger.info("Retrying run in %i seconds" % retryIn)
new_job = ReductionRunUtils().createRetryRun(reductionRun, delay=retryIn)
try:
MessagingUtils().send_pending(new_job, delay=retryIn*1000) # seconds to ms
except Exception as e:
new_job.delete()
raise e
def _make_pending_msg(self, reduction_run):
""" Creates a dict message from the given run, ready to be sent to ReductionPending. """
script, arguments = ReductionRunUtils().get_script_and_arguments(
reduction_run)
data_path = ''
# Currently only support single location
data_location = reduction_run.data_location.first()
if data_location:
data_path = data_location.file_path
else:
raise Exception("No data path found for reduction run")
data_dict = {
'run_number': reduction_run.run_number,
'instrument': reduction_run.instrument.name,
'rb_number': str(reduction_run.experiment.reference_number),
'data': data_path,
'reduction_script': script,
'reduction_arguments': arguments,
'run_version': reduction_run.run_version,
'facility': FACILITY,
'message': '',
'overwrite': reduction_run.overwrite,
}
return data_dict
def render_run_variables(request, instrument_name, run_number, run_version=0):
"""
Handles request to view the summary of a run
"""
# pylint:disable=no-member
reduction_run = ReductionRun.objects.get(instrument__name=instrument_name,
run_number=run_number,
run_version=run_version)
vars_kwargs = ReductionRunUtils.make_kwargs_from_runvariables(
reduction_run)
standard_vars = vars_kwargs["standard_vars"]
advanced_vars = vars_kwargs["advanced_vars"]
try:
current_variables = VariableUtils.get_default_variables(
instrument_name)
current_standard_variables = current_variables["standard_vars"]
current_advanced_variables = current_variables["advanced_vars"]
except (FileNotFoundError, ImportError, SyntaxError):
current_standard_variables = {}
current_advanced_variables = {}
context_dictionary = {
'run_number': run_number,
'run_version': run_version,
'standard_variables': standard_vars,
'advanced_variables': advanced_vars,
'current_standard_variables': current_standard_variables,
'current_advanced_variables': current_advanced_variables,
'instrument': reduction_run.instrument,
}
return render(request, 'snippets/run_variables.html', context_dictionary)
def test_createRetryRun(self):
reductionRun = self.createReductionRun()
retryRun = ReductionRunUtils().createRetryRun(reductionRun)
self.assertEqual(
reductionRun.retry_run, retryRun,
"Expected reductionRun.retry_run to be updated to retryRun")
shouldBeEqualFields = ["instrument", "experiment", "run_number"]
for field in shouldBeEqualFields:
self.assertEqual(getattr(reductionRun, field),
getattr(retryRun, field),
"Expected field %s to be equal" % field)
self.assertEqual(
reductionRun.run_version + 1, retryRun.run_version,
"Expected run version to be incremented, was %i vs %i" %
(reductionRun.run_version, retryRun.run_version))
self.assertEqual(
reductionRun.script, retryRun.script,
"Expected variable scripts to be the same, but were %s... and %s..."
% (reductionRun.script[:50], retryRun.script[:50]))
self.assertEqual(set(reductionRun.data_location.all()),
set(retryRun.data_location.all()),
"Expected data locations to be the same")
def fail_queue(request):
# render the page
error_status = StatusUtils().get_error()
failed_jobs = ReductionRun.objects.filter(Q(status=error_status) & Q(hidden_in_failviewer=False)).order_by('-created')
context_dictionary = { 'queue' : failed_jobs
, 'status_success' : StatusUtils().get_completed()
, 'status_failed' : StatusUtils().get_error()
}
if request.method == 'POST':
# perform the specified action
action = request.POST.get("action", "default")
selectedRunString = request.POST.get("selectedRuns", [])
selectedRuns = json.loads(selectedRunString)
try:
for run in selectedRuns:
runNumber = int(run[0])
runVersion = int(run[1])
RBNumber = int(run[2])
experiment = Experiment.objects.filter(reference_number=RBNumber).first()
reductionRun = ReductionRun.objects.get(experiment=experiment, run_number=runNumber, run_version=runVersion)
if action == "hide":
reductionRun.hidden_in_failviewer = True
reductionRun.save()
elif action == "rerun":
highest_version = max([int(runL[1]) for runL in selectedRuns if int(runL[0]) == runNumber])
if runVersion != highest_version:
continue # do not run multiples of the same run
ReductionRunUtils().cancelRun(reductionRun)
reductionRun.cancel = False
new_job = ReductionRunUtils().createRetryRun(reductionRun)
try:
MessagingUtils().send_pending(new_job)
except Exception as e:
new_job.delete()
raise e
elif action == "cancel":
ReductionRunUtils().cancelRun(reductionRun)
elif action == "default":
pass
except Exception as e:
failStr = "Selected action failed: %s %s" % (type(e).__name__, e)
logger.info("Failed to carry out fail_queue action - " + failStr)
context_dictionary["message"] = failStr
return context_dictionary
def test_cancel_queued(self):
reductionRun = self.createReductionRun()
with patch('reduction_variables.utils.MessagingUtils',
self.createMockMessagingUtils()):
ReductionRunUtils().cancelRun(reductionRun)
self.assertEqual(
self.cancelledRun, reductionRun,
"Expected that the run to be cancelled was this one")
self.assertEqual(
reductionRun.status,
StatusUtils().get_error(),
"Expected the run to have error status, was %s " %
reductionRun.status.value)
def test_get_script_and_arguments_successful(self):
run_variables = []
reduction_run = getReductionRun()
reduction_run.script = getValidScript("reduce.py")
variable = RunVariable(reduction_run=reduction_run,
name='test',
value='testvalue1',
type='text',
is_advanced=False)
variable.save()
run_variables.append(variable)
variable = RunVariable(reduction_run=reduction_run,
name='advanced_test',
value='testvalue2',
type='text',
is_advanced=True)
variable.save()
run_variables.append(variable)
script, arguments = ReductionRunUtils().get_script_and_arguments(
reduction_run)
self.assertNotEqual(script, None,
"Expecting to get a script path back.")
self.assertNotEqual(script, "", "Expecting to get a script path back.")
self.assertNotEqual(arguments, None,
"Expecting to get some arguments path back.")
self.assertNotEqual(arguments, {},
"Expecting to get some arguments path back.")
self.assertTrue('standard_vars' in arguments,
"Expecting arguments to have a 'standard_vars' key.")
self.assertTrue('advanced_vars' in arguments,
"Expecting arguments to have a 'advanced_vars' key.")
self.assertEqual(arguments['standard_vars']['test'], 'testvalue1',
"Expecting to find testvalue1 in standard_vars.")
self.assertEqual(arguments['advanced_vars']['advanced_test'],
'testvalue2',
"Expecting to find testvalue2 in advanced_vars.")
def test_cancel_not_queued(self):
# should not cancel a run that's not queued
statusList = [
"get_error", "get_completed", "get_processing", "get_skipped"
]
for statusMember in statusList:
status = getattr(StatusUtils(), statusMember)()
reductionRun = self.createReductionRun()
reductionRun.status = status
with patch('reduction_variables.utils.MessagingUtils',
self.createMockMessagingUtils()):
ReductionRunUtils().cancelRun(reductionRun)
self.assertEqual(
self.cancelledRun, None,
"Expected that no run was cancelled when trying %s" %
statusMember)
self.assertEqual(
reductionRun.status, status,
"Expected the run to have %s status, was %s" %
(status.value, reductionRun.status.value))
def _make_pending_msg(reduction_run):
""" Creates a Message from the given run, ready to be sent to ReductionPending. """
script, arguments = ReductionRunUtils().get_script_and_arguments(
reduction_run)
# Currently only support single location
data_location = reduction_run.data_location.first()
if data_location:
data_path = data_location.file_path
else:
raise Exception("No data path found for reduction run")
message = Message(run_number=reduction_run.run_number,
instrument=reduction_run.instrument.name,
rb_number=str(
reduction_run.experiment.reference_number),
data=data_path,
reduction_script=script,
reduction_arguments=arguments,
run_version=reduction_run.run_version,
facility=FACILITY,
overwrite=reduction_run.overwrite)
return message
def run_confirmation(request, instrument=None):
if request.method != 'POST':
return redirect('instrument_summary', instrument=instrument.name)
# POST
instrument = Instrument.objects.get(name=instrument)
run_numbers = []
if 'run_number' in request.POST:
run_numbers.append(int(request.POST.get('run_number')))
else:
range_string = request.POST.get('run_range').split(',')
# Expand list
for item in range_string:
if '-' in item:
split_range = item.split('-')
run_numbers.extend(
range(int(split_range[0]),
int(split_range[1]) + 1)
) # because this is a range, the end bound is exclusive!
else:
run_numbers.append(int(item))
# Make sure run numbers are distinct
run_numbers = set(run_numbers)
queued_status = StatusUtils().get_queued()
queue_count = ReductionRun.objects.filter(instrument=instrument,
status=queued_status).count()
context_dictionary = {
'runs': [],
'variables': None,
'queued': queue_count,
}
# Check that RB numbers are the same
rb_number = ReductionRun.objects.filter(
instrument=instrument,
run_number__in=run_numbers).values_list('experiment__reference_number',
flat=True).distinct()
if len(rb_number) > 1:
context_dictionary[
'error'] = 'Runs span multiple experiment numbers (' + ','.join(
str(i)
for i in rb_number) + ') please select a different range.'
for run_number in run_numbers:
old_reduction_run = ReductionRun.objects.filter(
run_number=run_number).order_by('-run_version').first()
# Check old run exists
if old_reduction_run is None:
context_dictionary['error'] = "Run number " + str(
run_number) + " doesn't exist."
use_current_script = request.POST.get('use_current_script',
u"true").lower() == u"true"
if use_current_script:
script_text = InstrumentVariablesUtils().get_current_script_text(
instrument.name)[0]
default_variables = InstrumentVariablesUtils(
).get_default_variables(instrument.name)
else:
script_text = old_reduction_run.script
default_variables = old_reduction_run.run_variables.all()
new_variables = []
for key, value in request.POST.iteritems():
if 'var-' in key:
name = None
if 'var-advanced-' in key:
name = key.replace('var-advanced-', '').replace('-', ' ')
is_advanced = True
if 'var-standard-' in key:
name = key.replace('var-standard-', '').replace('-', ' ')
is_advanced = False
if name is not None:
default_var = next(
(x for x in default_variables if x.name == name), None)
if not default_var:
continue
if len(value) > InstrumentVariable._meta.get_field(
'value').max_length:
context_dictionary['error'] = 'Value given in ' + str(
name) + ' is too long.'
variable = RunVariable(name=default_var.name,
value=value,
is_advanced=is_advanced,
type=default_var.type,
help_text=default_var.help_text)
new_variables.append(variable)
if len(new_variables) == 0:
context_dictionary[
'error'] = 'No variables were found to be submitted.'
# User can choose whether to overwrite with the re-run or create new data
if request.POST.get('overwrite_checkbox') == 'on':
overwrite_previous_data = True
else:
overwrite_previous_data = False
if 'error' in context_dictionary:
return context_dictionary
run_description = request.POST.get('run_description')
new_job = ReductionRunUtils().createRetryRun(
old_reduction_run,
script=script_text,
overwrite=overwrite_previous_data,
variables=new_variables,
username=request.user.username,
description=run_description)
try:
MessagingUtils().send_pending(new_job)
context_dictionary['runs'].append(new_job)
context_dictionary['variables'] = new_variables
except Exception as e:
new_job.delete()
context_dictionary['error'] = 'Failed to send new job. (%s)' % str(
e)
return context_dictionary
def run_confirmation(request, instrument):
"""
Handles request for user to confirm re-run
"""
if request.method != 'POST':
return redirect('instrument_summary', instrument=instrument.name)
# POST
# pylint:disable=no-member
instrument = Instrument.objects.get(name=instrument)
range_string = request.POST.get('run_range')
queued_status = StatusUtils().get_queued()
# pylint:disable=no-member
queue_count = ReductionRun.objects.filter(instrument=instrument,
status=queued_status).count()
context_dictionary = {
'runs': [],
'variables': None,
'queued': queue_count,
}
try:
run_numbers = input_processing.parse_user_run_numbers(range_string)
except SyntaxError as exception:
context_dictionary['error'] = exception.msg
return context_dictionary
# Determine user level to set a maximum limit to the number of runs that can be re-queued
if request.user.is_superuser:
max_runs = 500
elif request.user.is_staff:
max_runs = 50
else:
max_runs = 20
if len(run_numbers) > max_runs:
context_dictionary["error"] = "{0} runs were requested, but only {1} runs can be " \
"queued at a time".format(len(run_numbers), max_runs)
return context_dictionary
# Check that RB numbers are the same for the range entered
# pylint:disable=no-member
rb_number = ReductionRun.objects.filter(instrument=instrument, run_number__in=run_numbers) \
.values_list('experiment__reference_number', flat=True).distinct()
if len(rb_number) > 1:
context_dictionary['error'] = 'Runs span multiple experiment numbers ' \
'(' + ','.join(str(i) for i in rb_number) + ')' \
' please select a different range.'
return context_dictionary
for run_number in run_numbers:
# pylint:disable=no-member
matching_previous_runs_queryset = ReductionRun.objects.\
filter(instrument=instrument,
run_number=run_number).order_by('-run_version')
most_recent_previous_run = matching_previous_runs_queryset.first()
# Check old run exists
if most_recent_previous_run is None:
context_dictionary['error'] = "Run number %s hasn't been" \
"ran by autoreduction yet." % str(run_number)
# Check it is not currently queued
queued_runs = matching_previous_runs_queryset.filter(
status=queued_status).first()
if queued_runs is not None:
context_dictionary['error'] = "Run number {0} is already queued to run".\
format(queued_runs.run_number)
return context_dictionary
use_current_script = request.POST.get('use_current_script',
u"true").lower() == u"true"
if use_current_script:
script_text = InstrumentVariablesUtils().get_current_script_text(
instrument.name)[0]
default_variables = InstrumentVariablesUtils(
).get_default_variables(instrument.name)
else:
script_text = most_recent_previous_run.script
default_variables = most_recent_previous_run.run_variables.all()
new_variables = []
for key, value in list(request.POST.items()):
if 'var-' in key:
name = None
if 'var-advanced-' in key:
name = key.replace('var-advanced-', '').replace('-', ' ')
is_advanced = True
if 'var-standard-' in key:
name = key.replace('var-standard-', '').replace('-', ' ')
is_advanced = False
if name is not None:
default_var = next(
(x for x in default_variables if x.name == name), None)
if not default_var:
continue
# pylint:disable=protected-access,no-member
if len(value) > InstrumentVariable._meta.get_field(
'value').max_length:
context_dictionary['error'] = 'Value given in {} is too long.'\
.format(str(name))
variable = RunVariable(name=default_var.name,
value=value,
is_advanced=is_advanced,
type=default_var.type,
help_text=default_var.help_text)
new_variables.append(variable)
if not new_variables:
context_dictionary[
'error'] = 'No variables were found to be submitted.'
# User can choose whether to overwrite with the re-run or create new data
overwrite_previous_data = bool(
request.POST.get('overwrite_checkbox') == 'on')
if 'error' in context_dictionary:
return context_dictionary
run_description = request.POST.get('run_description')
max_desc_len = 200
if len(run_description) > max_desc_len:
context_dictionary["error"] = "The description contains {0} characters, " \
"a maximum of {1} are allowed".\
format(len(run_description), max_desc_len)
return context_dictionary
new_job = ReductionRunUtils().createRetryRun(
user_id=request.user.id,
reduction_run=most_recent_previous_run,
script=script_text,
overwrite=overwrite_previous_data,
variables=new_variables,
description=run_description)
try:
MessagingUtils().send_pending(new_job)
context_dictionary['runs'].append(new_job)
context_dictionary['variables'] = new_variables
# pylint:disable=broad-except
except Exception as exception:
new_job.delete()
context_dictionary['error'] = 'Failed to send new job. (%s)' % str(
exception)
return context_dictionary
def run_confirmation(request, instrument=None):
if request.method != 'POST':
return redirect('instrument_summary', instrument=instrument.name)
# POST
instrument = Instrument.objects.get(name=instrument)
run_numbers = []
if 'run_number' in request.POST:
run_numbers.append(int(request.POST.get('run_number')))
else:
range_string = request.POST.get('run_range').split(',')
# Expand list
for item in range_string:
if '-' in item:
split_range = item.split('-')
run_numbers.extend(range(int(split_range[0]), int(split_range[1])+1)) # because this is a range, the end bound is exclusive!
else:
run_numbers.append(int(item))
# Make sure run numbers are distinct
run_numbers = set(run_numbers)
queued_status = StatusUtils().get_queued()
queue_count = ReductionRun.objects.filter(instrument=instrument, status=queued_status).count()
context_dictionary = {
'runs' : [],
'variables' : None,
'queued' : queue_count,
}
# Check that RB numbers are the same
rb_number = ReductionRun.objects.filter(instrument=instrument, run_number__in=run_numbers).values_list('experiment__reference_number', flat=True).distinct()
if len(rb_number) > 1:
context_dictionary['error'] = 'Runs span multiple experiment numbers (' + ','.join(str(i) for i in rb_number) + ') please select a different range.'
for run_number in run_numbers:
old_reduction_run = ReductionRun.objects.filter(run_number=run_number).order_by('-run_version').first()
use_current_script = request.POST.get('use_current_script', u"true").lower() == u"true"
if use_current_script:
script_text = InstrumentVariablesUtils().get_current_script_text(instrument.name)[0]
default_variables = InstrumentVariablesUtils().get_default_variables(instrument.name)
else:
script_text = old_reduction_run.script
default_variables = old_reduction_run.run_variables.all()
new_variables = []
for key,value in request.POST.iteritems():
if 'var-' in key:
name = None
if 'var-advanced-' in key:
name = key.replace('var-advanced-', '').replace('-', ' ')
is_advanced = True
if 'var-standard-' in key:
name = key.replace('var-standard-', '').replace('-', ' ')
is_advanced = False
if name is not None:
default_var = next((x for x in default_variables if x.name == name), None)
if not default_var:
continue
if len(value) > InstrumentVariable._meta.get_field('value').max_length:
context_dictionary['error'] = 'Value given in ' + str(name) + ' is too long.'
variable = RunVariable( name = default_var.name
, value = value
, is_advanced = is_advanced
, type = default_var.type
, help_text = default_var.help_text
)
new_variables.append(variable)
if len(new_variables) == 0:
context_dictionary['error'] = 'No variables were found to be submitted.'
if 'error' in context_dictionary:
return context_dictionary
new_job = ReductionRunUtils().createRetryRun(old_reduction_run, script=script_text, variables=new_variables, username=request.user.username)
try:
MessagingUtils().send_pending(new_job)
context_dictionary['runs'].append(new_job)
context_dictionary['variables'] = new_variables
except Exception as e:
new_job.delete()
context_dictionary['error'] = 'Failed to send new job. (%s)' % str(e)
return context_dictionary
