def run_queue(request):
"""
Render status of queue
"""
# Get all runs that should be shown
queued_status = Status.get_queued()
processing_status = Status.get_processing()
pending_jobs = ReductionRun.objects.filter(
Q(status=queued_status)
| Q(status=processing_status)).order_by('created')
# Filter those which the user shouldn't be able to see
if USER_ACCESS_CHECKS and not request.user.is_superuser:
try:
with ICATCache(AUTH='uows',
SESSION={'sessionid':
request.session['sessionid']}) as icat:
pending_jobs = filter(
lambda job: job.experiment.reference_number in icat.
get_associated_experiments(int(request.user.username)),
pending_jobs) # check RB numbers
pending_jobs = filter(
lambda job: job.instrument.name in icat.
get_owned_instruments(int(request.user.username)),
pending_jobs) # check instrument
except ICATConnectionException as excep:
return render_error(request, str(excep))
# Initialise list to contain the names of user/team that started runs
started_by = []
# cycle through all filtered runs and retrieve the name of the user/team that started the run
for run in pending_jobs:
started_by.append(started_by_id_to_name(run.started_by))
# zip the run information with the user/team name to enable simultaneous iteration with django
context_dictionary = {'queue': zip(pending_jobs, started_by)}
return context_dictionary
def submit_and_wait_for_result(test, expected_runs=1):
"""
Submit after a reset button has been clicked. Then waits until the queue listener has finished processing.
Sticks the submission in a loop in case the first time doesn't work. The reason
it may not work is that resetting actually swaps out the whole form using JS, which
replaces ALL the elements and triggers a bunch of DOM re-renders/updates, and that isn't fast.
"""
test.listener._processing = True # pylint:disable=protected-access
expected_url = reverse("run_confirmation",
kwargs={"instrument": test.instrument_name})
def submit_successful(driver) -> bool:
try:
test.page.submit_button.click()
except ElementClickInterceptedException:
pass
# the submit is successful if the URL has changed
return expected_url in driver.current_url
WebDriverWait(test.driver, 30).until(submit_successful)
if expected_runs == 1:
WebDriverWait(
test.driver,
30).until(lambda _: not test.listener.is_processing_message())
else:
num_current_runs = ReductionRun.objects.filter(
status=Status.get_completed()).count()
WebDriverWait(test.driver, 30).until(
lambda _: ReductionRun.objects.filter(status=Status.get_completed(
)).count() == num_current_runs + expected_runs)
return find_run_in_database(test)
def fail_queue(request):
"""
Render status of failed queue
"""
# render the page
error_status = Status.get_error()
failed_jobs = ReductionRun.objects.filter(
Q(status=error_status)
& Q(hidden_in_failviewer=False)).order_by('-created')
context_dictionary = {
'queue': failed_jobs,
'status_success': Status.get_completed(),
'status_failed': Status.get_error()
}
if request.method == 'POST':
# perform the specified action
action = request.POST.get("action", "default")
selected_run_string = request.POST.get("selectedRuns", [])
selected_runs = json.loads(selected_run_string)
try:
for run in selected_runs:
run_number = int(run[0])
run_version = int(run[1])
reduction_run = failed_jobs.get(run_number=run_number,
run_version=run_version)
if action == "hide":
reduction_run.hidden_in_failviewer = True
reduction_run.save()
elif action == "rerun":
highest_version = max([
int(runL[1]) for runL in selected_runs
if int(runL[0]) == run_number
])
if run_version != highest_version:
continue # do not run multiples of the same run
ReductionRunUtils.send_retry_message_same_args(
request.user.id, reduction_run)
elif action == "default":
pass
# pylint:disable=broad-except
except Exception as exception:
fail_str = "Selected action failed: %s %s" % (
type(exception).__name__, exception)
LOGGER.info("Failed to carry out fail_queue action - %s", fail_str)
context_dictionary["message"] = fail_str
return context_dictionary
def reduction_started(self, reduction_run: ReductionRun, message: Message):
"""
Update the run as 'started' / 'processing' in the database. This is
called when the run is ready to start.
"""
self._logger.info("Run %s has started reduction", message.run_number)
reduction_run.status = Status.get_processing()
reduction_run.started = timezone.now()
reduction_run.save()
def submit_runs(request, instrument=None):
"""
Handles run submission request
"""
LOGGER.info('Submitting runs')
# pylint:disable=no-member
instrument = Instrument.objects.prefetch_related('reduction_runs').get(
name=instrument)
if request.method == 'GET':
processing_status = Status.get_processing()
queued_status = Status.get_queued()
# pylint:disable=no-member
runs_for_instrument = instrument.reduction_runs.all()
last_run = instrument.get_last_for_rerun(runs_for_instrument)
kwargs = ReductionRunUtils.make_kwargs_from_runvariables(last_run)
standard_vars = kwargs["standard_vars"]
advanced_vars = kwargs["advanced_vars"]
try:
default_variables = VariableUtils.get_default_variables(instrument)
except (FileNotFoundError, ImportError, SyntaxError) as err:
return {"message": str(err)}
final_standard = _combine_dicts(standard_vars,
default_variables["standard_vars"])
final_advanced = _combine_dicts(advanced_vars,
default_variables["advanced_vars"])
# pylint:disable=no-member
context_dictionary = {
'instrument': instrument,
'last_instrument_run': last_run,
'processing': runs_for_instrument.filter(status=processing_status),
'queued': runs_for_instrument.filter(status=queued_status),
'standard_variables': final_standard,
'advanced_variables': final_advanced,
}
return context_dictionary
def do_create_reduction_record(message: Message, experiment: Experiment, instrument: Instrument, run_version: int,
software: Software):
"""Create the reduction record."""
# Make the new reduction run with the information collected so far
reduction_run, message = records.create_reduction_run_record(experiment=experiment,
instrument=instrument,
message=message,
run_version=run_version,
software=software,
status=Status.get_queued())
return reduction_run, message, instrument, software
def reduction_error(self, reduction_run: ReductionRun, message: Message):
"""
Update the run as 'errored' in the database. This is called when the run
encounters an error.
"""
if message.message:
self._logger.info("Run %s has encountered an error - %s", message.run_number, message.message)
else:
self._logger.info("Run %s has encountered an error - No error message was found", message.run_number)
self._common_reduction_run_update(reduction_run, Status.get_error(), message)
reduction_run.save()
def reduction_skipped(self, reduction_run: ReductionRun, message: Message):
"""
Update the run status to 'skipped' in the database. This is called when
there was a reason to skip the run. Will NOT attempt re-run.
"""
if message.message is not None:
self._logger.info("Run %s has been skipped - %s", message.run_number, message.message)
else:
self._logger.info("Run %s has been skipped - No error message was found", message.run_number)
self._common_reduction_run_update(reduction_run, Status.get_skipped(), message)
reduction_run.save()
def reduction_complete(self, reduction_run: ReductionRun, message: Message):
"""
Update the run as 'completed' in the database. This is called when the
run has completed.
"""
self._logger.info("Run %s has completed reduction", message.run_number)
self._common_reduction_run_update(reduction_run, Status.get_completed(), message)
reduction_run.save()
if message.reduction_data is not None:
reduction_location = ReductionLocation(file_path=message.reduction_data, reduction_run=reduction_run)
reduction_location.save()
def test_retrieve_status(self):
"""Test that retrieving the status returns the expected one"""
assert len(Status._cached_statuses.values()) == 0
assert Status.get_error() is not None
assert str(Status.get_error()) == "Error"
assert len(Status._cached_statuses.values()) == 1
assert Status.get_completed() is not None
assert str(Status.get_completed()) == "Completed"
assert len(Status._cached_statuses.values()) == 2
assert Status.get_processing() is not None
assert str(Status.get_processing()) == "Processing"
assert len(Status._cached_statuses.values()) == 3
assert Status.get_queued() is not None
assert str(Status.get_queued()) == "Queued"
assert len(Status._cached_statuses.values()) == 4
assert Status.get_skipped() is not None
assert str(Status.get_skipped()) == "Skipped"
assert len(Status._cached_statuses.values()) == 5
def find_reason_to_avoid_re_run(matching_previous_runs, run_number):
"""
Check whether the most recent run exists
"""
most_recent_run = matching_previous_runs.first()
# Check old run exists - if it doesn't exist there's nothing to re-run!
if most_recent_run is None:
return False, f"Run number {run_number} hasn't been ran by autoreduction yet."
# Prevent multiple queueings of the same re-run
queued_runs = matching_previous_runs.filter(
status=Status.get_queued()).first()
if queued_runs is not None:
return False, f"Run number {queued_runs.run_number} is already queued to run"
return True, ""
def runs_list(request, instrument=None):
"""
Render instrument summary
"""
try:
filter_by = request.GET.get('filter', 'run')
instrument_obj = Instrument.objects.get(name=instrument)
except Instrument.DoesNotExist:
return {'message': "Instrument not found."}
try:
sort_by = request.GET.get('sort', 'run')
if sort_by == 'run':
runs = (ReductionRun.objects.only(
'status', 'last_updated', 'run_number', 'run_version',
'run_description').select_related('status').filter(
instrument=instrument_obj).order_by(
'-run_number', 'run_version'))
else:
runs = (ReductionRun.objects.only(
'status', 'last_updated', 'run_number', 'run_version',
'run_description').select_related('status').filter(
instrument=instrument_obj).order_by('-last_updated'))
if len(runs) == 0:
return {'message': "No runs found for instrument."}
try:
current_variables = VariableUtils.get_default_variables(
instrument_obj.name)
error_reason = ""
except FileNotFoundError:
current_variables = {}
error_reason = "reduce_vars.py is missing for this instrument"
except (ImportError, SyntaxError):
current_variables = {}
error_reason = "reduce_vars.py has an import or syntax error"
has_variables = bool(current_variables)
context_dictionary = {
'instrument': instrument_obj,
'instrument_name': instrument_obj.name,
'runs': runs,
'last_instrument_run': runs[0],
'processing': runs.filter(status=Status.get_processing()),
'queued': runs.filter(status=Status.get_queued()),
'filtering': filter_by,
'sort': sort_by,
'has_variables': has_variables,
'error_reason': error_reason
}
if filter_by == 'experiment':
experiments_and_runs = {}
experiments = Experiment.objects.filter(reduction_runs__instrument=instrument_obj). \
order_by('-reference_number').distinct()
for experiment in experiments:
associated_runs = runs.filter(experiment=experiment). \
order_by('-created')
experiments_and_runs[experiment] = associated_runs
context_dictionary['experiments'] = experiments_and_runs
else:
max_items_per_page = request.GET.get('pagination', 10)
custom_paginator = CustomPaginator(
page_type=sort_by,
query_set=runs,
items_per_page=max_items_per_page,
page_tolerance=3,
current_page=request.GET.get('page', 1))
context_dictionary['paginator'] = custom_paginator
context_dictionary['last_page_index'] = len(
custom_paginator.page_list)
context_dictionary['max_items'] = max_items_per_page
# pylint:disable=broad-except
except Exception:
LOGGER.error(traceback.format_exc())
return {
'message':
"An unexpected error has occurred when loading the instrument."
}
return context_dictionary
def run_confirmation(request, instrument: str):
"""
Handles request for user to confirm re-run
"""
range_string = request.POST.get('run_range')
run_description = request.POST.get('run_description')
# pylint:disable=no-member
queue_count = ReductionRun.objects.filter(
instrument__name=instrument, status=Status.get_queued()).count()
context_dictionary = {
# list stores (run_number, run_version)
'runs': [],
'variables': None,
'queued': queue_count,
'instrument_name': instrument,
'run_description': run_description
}
try:
run_numbers = input_processing.parse_user_run_numbers(range_string)
except SyntaxError as exception:
context_dictionary['error'] = exception.msg
return context_dictionary
if not run_numbers:
context_dictionary[
'error'] = f"Could not correctly parse range input {range_string}"
return context_dictionary
# Determine user level to set a maximum limit to the number of runs that can be re-queued
if request.user.is_superuser:
max_runs = 500
elif request.user.is_staff:
max_runs = 50
else:
max_runs = 20
if len(run_numbers) > max_runs:
context_dictionary["error"] = "{0} runs were requested, but only {1} runs can be " \
"queued at a time".format(len(run_numbers), max_runs)
return context_dictionary
related_runs: QuerySet[ReductionRun] = ReductionRun.objects.filter(
instrument__name=instrument, run_number__in=run_numbers)
# Check that RB numbers are the same for the range entered
# pylint:disable=no-member
rb_number = related_runs.values_list('experiment__reference_number',
flat=True).distinct()
if len(rb_number) > 1:
context_dictionary['error'] = 'Runs span multiple experiment numbers ' \
'(' + ','.join(str(i) for i in rb_number) + ')' \
' please select a different range.'
return context_dictionary
try:
script_text = InstrumentVariablesUtils.get_current_script_text(
instrument)
default_variables = VariableUtils.get_default_variables(instrument)
except (FileNotFoundError, ImportError, SyntaxError) as err:
context_dictionary['error'] = err
return context_dictionary
try:
new_script_arguments = make_reduction_arguments(
request.POST.items(), default_variables)
context_dictionary['variables'] = new_script_arguments
except ValueError as err:
context_dictionary['error'] = err
return context_dictionary
for run_number in run_numbers:
matching_previous_runs = related_runs.filter(
run_number=run_number).order_by('-run_version')
run_suitable, reason = find_reason_to_avoid_re_run(
matching_previous_runs, run_number)
if not run_suitable:
context_dictionary['error'] = reason
break
# run_description gets stored in run_description in the ReductionRun object
max_run_description_length = ReductionRun._meta.get_field(
'run_description').max_length
if len(run_description) > max_run_description_length:
context_dictionary["error"] = "The description contains {0} characters, " \
"a maximum of {1} are allowed".\
format(len(run_description), max_run_description_length)
return context_dictionary
most_recent_run: ReductionRun = matching_previous_runs.first()
# User can choose whether to overwrite with the re-run or create new data
ReductionRunUtils.send_retry_message(request.user.id, most_recent_run,
run_description, script_text,
new_script_arguments, False)
# list stores (run_number, run_version)
context_dictionary["runs"].append(
(run_number, most_recent_run.run_version + 1))
return context_dictionary
def configure_new_runs_get(instrument_name,
start=0,
end=0,
experiment_reference=0):
"""
GET for the configure new runs page
"""
instrument = Instrument.objects.get(name__iexact=instrument_name)
editing = (start > 0 or experiment_reference > 0)
last_run = instrument.get_last_for_rerun()
run_variables = ReductionRunUtils.make_kwargs_from_runvariables(last_run)
standard_vars = run_variables["standard_vars"]
advanced_vars = run_variables["advanced_vars"]
# if a specific start is provided, include vars upcoming for the specific start
filter_kwargs = {"start_run__gte": start if start else last_run.run_number}
if end:
# if an end run is provided - don't show variables outside the [start-end) range
filter_kwargs["start_run__lt"] = end
upcoming_variables = instrument.instrumentvariable_set.filter(
**filter_kwargs)
if experiment_reference:
upcoming_experiment_variables = instrument.instrumentvariable_set.filter(
experiment_reference=experiment_reference)
else:
upcoming_experiment_variables = []
# Updates the variables values. Experiment variables are chained second
# so they values will overwrite any changes from the run variables
for upcoming_var in chain(upcoming_variables,
upcoming_experiment_variables):
name = upcoming_var.name
if name in standard_vars or not upcoming_var.is_advanced:
standard_vars[name] = upcoming_var
elif name in advanced_vars or upcoming_var.is_advanced:
advanced_vars[name] = upcoming_var
# Unique, comma-joined list of all start runs belonging to the upcoming variables.
# This seems to be used to prevent submission if trying to resubmit variables for already
# configured future run numbers - check the checkForConflicts function
# This should probably be done by the POST method anyway.. so remove it when
if upcoming_variables:
upcoming_run_variables = ','.join(
{str(var.start_run)
for var in upcoming_variables})
else:
upcoming_run_variables = ""
try:
reduce_vars_variables = VariableUtils.get_default_variables(instrument)
except (FileNotFoundError, ImportError, SyntaxError) as err:
return {"message": str(err)}
final_standard = _combine_dicts(standard_vars,
reduce_vars_variables["standard_vars"])
final_advanced = _combine_dicts(advanced_vars,
reduce_vars_variables["advanced_vars"])
run_start = start if start else last_run.run_number + 1
context_dictionary = {
'instrument':
instrument,
'last_instrument_run':
last_run,
'processing':
ReductionRun.objects.filter(instrument=instrument,
status=Status.get_processing()),
'queued':
ReductionRun.objects.filter(instrument=instrument,
status=Status.get_queued()),
'standard_variables':
final_standard,
'advanced_variables':
final_advanced,
'run_start':
run_start,
# used to determine whether the current form is for an experiment reference
'current_experiment_reference':
experiment_reference,
# used to create the link to an experiment reference form, using this number
'submit_for_experiment_reference':
last_run.experiment.reference_number,
'minimum_run_start':
run_start,
'minimum_run_end':
run_start + 1,
'upcoming_run_variables':
upcoming_run_variables,
'editing':
editing,
'tracks_script':
'',
}
return context_dictionary