def fail_queue(request): # render the page error_status = StatusUtils().get_error() failed_jobs = ReductionRun.objects.filter(Q(status=error_status) & Q(hidden_in_failviewer=False)).order_by('-created') context_dictionary = { 'queue' : failed_jobs , 'status_success' : StatusUtils().get_completed() , 'status_failed' : StatusUtils().get_error() } if request.method == 'POST': # perform the specified action action = request.POST.get("action", "default") selectedRunString = request.POST.get("selectedRuns", []) selectedRuns = json.loads(selectedRunString) try: for run in selectedRuns: runNumber = int(run[0]) runVersion = int(run[1]) RBNumber = int(run[2]) experiment = Experiment.objects.filter(reference_number=RBNumber).first() reductionRun = ReductionRun.objects.get(experiment=experiment, run_number=runNumber, run_version=runVersion) if action == "hide": reductionRun.hidden_in_failviewer = True reductionRun.save() elif action == "rerun": highest_version = max([int(runL[1]) for runL in selectedRuns if int(runL[0]) == runNumber]) if runVersion != highest_version: continue # do not run multiples of the same run ReductionRunUtils().cancelRun(reductionRun) reductionRun.cancel = False new_job = ReductionRunUtils().createRetryRun(reductionRun) try: MessagingUtils().send_pending(new_job) except Exception as e: new_job.delete() raise e elif action == "cancel": ReductionRunUtils().cancelRun(reductionRun) elif action == "default": pass except Exception as e: failStr = "Selected action failed: %s %s" % (type(e).__name__, e) logger.info("Failed to carry out fail_queue action - " + failStr) context_dictionary["message"] = failStr return context_dictionary
def test_createRetryRun_duplicate(self): reductionRun = self.createReductionRun() retryRun1 = ReductionRunUtils().createRetryRun(reductionRun) retryRun2 = ReductionRunUtils().createRetryRun(reductionRun) self.assertEqual( retryRun1.run_version + 1, retryRun2.run_version, "Expected run version to be correctly incremented, was %i vs %i" % (retryRun1.run_version, retryRun2.run_version))
def fail_queue(request): """ Render status of failed queue """ # render the page error_status = STATUS.get_error() failed_jobs = ReductionRun.objects.filter( Q(status=error_status) & Q(hidden_in_failviewer=False)).order_by('-created') context_dictionary = { 'queue': failed_jobs, 'status_success': STATUS.get_completed(), 'status_failed': STATUS.get_error() } if request.method == 'POST': # perform the specified action action = request.POST.get("action", "default") selected_run_string = request.POST.get("selectedRuns", []) selected_runs = json.loads(selected_run_string) try: for run in selected_runs: run_number = int(run[0]) run_version = int(run[1]) reduction_run = failed_jobs.get(run_number=run_number, run_version=run_version) if action == "hide": reduction_run.hidden_in_failviewer = True reduction_run.save() elif action == "rerun": highest_version = max([ int(runL[1]) for runL in selected_runs if int(runL[0]) == run_number ]) if run_version != highest_version: continue # do not run multiples of the same run ReductionRunUtils.send_retry_message_same_args( request.user.id, reduction_run) elif action == "default": pass # pylint:disable=broad-except except Exception as exception: fail_str = "Selected action failed: %s %s" % ( type(exception).__name__, exception) LOGGER.info("Failed to carry out fail_queue action - %s", fail_str) context_dictionary["message"] = fail_str return context_dictionary
def retryRun(self, reductionRun, retryIn): if (reductionRun.cancel): logger.info("Cancelling run retry") return logger.info("Retrying run in %i seconds" % retryIn) new_job = ReductionRunUtils().createRetryRun(reductionRun, delay=retryIn) try: MessagingUtils().send_pending(new_job, delay=retryIn*1000) # seconds to ms except Exception as e: new_job.delete() raise e
def _make_pending_msg(self, reduction_run): """ Creates a dict message from the given run, ready to be sent to ReductionPending. """ script, arguments = ReductionRunUtils().get_script_and_arguments( reduction_run) data_path = '' # Currently only support single location data_location = reduction_run.data_location.first() if data_location: data_path = data_location.file_path else: raise Exception("No data path found for reduction run") data_dict = { 'run_number': reduction_run.run_number, 'instrument': reduction_run.instrument.name, 'rb_number': str(reduction_run.experiment.reference_number), 'data': data_path, 'reduction_script': script, 'reduction_arguments': arguments, 'run_version': reduction_run.run_version, 'facility': FACILITY, 'message': '', 'overwrite': reduction_run.overwrite, } return data_dict
def render_run_variables(request, instrument_name, run_number, run_version=0): """ Handles request to view the summary of a run """ # pylint:disable=no-member reduction_run = ReductionRun.objects.get(instrument__name=instrument_name, run_number=run_number, run_version=run_version) vars_kwargs = ReductionRunUtils.make_kwargs_from_runvariables( reduction_run) standard_vars = vars_kwargs["standard_vars"] advanced_vars = vars_kwargs["advanced_vars"] try: current_variables = VariableUtils.get_default_variables( instrument_name) current_standard_variables = current_variables["standard_vars"] current_advanced_variables = current_variables["advanced_vars"] except (FileNotFoundError, ImportError, SyntaxError): current_standard_variables = {} current_advanced_variables = {} context_dictionary = { 'run_number': run_number, 'run_version': run_version, 'standard_variables': standard_vars, 'advanced_variables': advanced_vars, 'current_standard_variables': current_standard_variables, 'current_advanced_variables': current_advanced_variables, 'instrument': reduction_run.instrument, } return render(request, 'snippets/run_variables.html', context_dictionary)
def test_createRetryRun(self): reductionRun = self.createReductionRun() retryRun = ReductionRunUtils().createRetryRun(reductionRun) self.assertEqual( reductionRun.retry_run, retryRun, "Expected reductionRun.retry_run to be updated to retryRun") shouldBeEqualFields = ["instrument", "experiment", "run_number"] for field in shouldBeEqualFields: self.assertEqual(getattr(reductionRun, field), getattr(retryRun, field), "Expected field %s to be equal" % field) self.assertEqual( reductionRun.run_version + 1, retryRun.run_version, "Expected run version to be incremented, was %i vs %i" % (reductionRun.run_version, retryRun.run_version)) self.assertEqual( reductionRun.script, retryRun.script, "Expected variable scripts to be the same, but were %s... and %s..." % (reductionRun.script[:50], retryRun.script[:50])) self.assertEqual(set(reductionRun.data_location.all()), set(retryRun.data_location.all()), "Expected data locations to be the same")
def test_cancel_queued(self): reductionRun = self.createReductionRun() with patch('reduction_variables.utils.MessagingUtils', self.createMockMessagingUtils()): ReductionRunUtils().cancelRun(reductionRun) self.assertEqual( self.cancelledRun, reductionRun, "Expected that the run to be cancelled was this one") self.assertEqual( reductionRun.status, StatusUtils().get_error(), "Expected the run to have error status, was %s " % reductionRun.status.value)
def test_get_script_and_arguments_successful(self): run_variables = [] reduction_run = getReductionRun() reduction_run.script = getValidScript("reduce.py") variable = RunVariable(reduction_run=reduction_run, name='test', value='testvalue1', type='text', is_advanced=False) variable.save() run_variables.append(variable) variable = RunVariable(reduction_run=reduction_run, name='advanced_test', value='testvalue2', type='text', is_advanced=True) variable.save() run_variables.append(variable) script, arguments = ReductionRunUtils().get_script_and_arguments( reduction_run) self.assertNotEqual(script, None, "Expecting to get a script path back.") self.assertNotEqual(script, "", "Expecting to get a script path back.") self.assertNotEqual(arguments, None, "Expecting to get some arguments path back.") self.assertNotEqual(arguments, {}, "Expecting to get some arguments path back.") self.assertTrue('standard_vars' in arguments, "Expecting arguments to have a 'standard_vars' key.") self.assertTrue('advanced_vars' in arguments, "Expecting arguments to have a 'advanced_vars' key.") self.assertEqual(arguments['standard_vars']['test'], 'testvalue1', "Expecting to find testvalue1 in standard_vars.") self.assertEqual(arguments['advanced_vars']['advanced_test'], 'testvalue2', "Expecting to find testvalue2 in advanced_vars.")
def test_cancel_not_queued(self): # should not cancel a run that's not queued statusList = [ "get_error", "get_completed", "get_processing", "get_skipped" ] for statusMember in statusList: status = getattr(StatusUtils(), statusMember)() reductionRun = self.createReductionRun() reductionRun.status = status with patch('reduction_variables.utils.MessagingUtils', self.createMockMessagingUtils()): ReductionRunUtils().cancelRun(reductionRun) self.assertEqual( self.cancelledRun, None, "Expected that no run was cancelled when trying %s" % statusMember) self.assertEqual( reductionRun.status, status, "Expected the run to have %s status, was %s" % (status.value, reductionRun.status.value))
def _make_pending_msg(reduction_run): """ Creates a Message from the given run, ready to be sent to ReductionPending. """ script, arguments = ReductionRunUtils().get_script_and_arguments( reduction_run) # Currently only support single location data_location = reduction_run.data_location.first() if data_location: data_path = data_location.file_path else: raise Exception("No data path found for reduction run") message = Message(run_number=reduction_run.run_number, instrument=reduction_run.instrument.name, rb_number=str( reduction_run.experiment.reference_number), data=data_path, reduction_script=script, reduction_arguments=arguments, run_version=reduction_run.run_version, facility=FACILITY, overwrite=reduction_run.overwrite) return message
def run_confirmation(request, instrument=None): if request.method != 'POST': return redirect('instrument_summary', instrument=instrument.name) # POST instrument = Instrument.objects.get(name=instrument) run_numbers = [] if 'run_number' in request.POST: run_numbers.append(int(request.POST.get('run_number'))) else: range_string = request.POST.get('run_range').split(',') # Expand list for item in range_string: if '-' in item: split_range = item.split('-') run_numbers.extend( range(int(split_range[0]), int(split_range[1]) + 1) ) # because this is a range, the end bound is exclusive! else: run_numbers.append(int(item)) # Make sure run numbers are distinct run_numbers = set(run_numbers) queued_status = StatusUtils().get_queued() queue_count = ReductionRun.objects.filter(instrument=instrument, status=queued_status).count() context_dictionary = { 'runs': [], 'variables': None, 'queued': queue_count, } # Check that RB numbers are the same rb_number = ReductionRun.objects.filter( instrument=instrument, run_number__in=run_numbers).values_list('experiment__reference_number', flat=True).distinct() if len(rb_number) > 1: context_dictionary[ 'error'] = 'Runs span multiple experiment numbers (' + ','.join( str(i) for i in rb_number) + ') please select a different range.' for run_number in run_numbers: old_reduction_run = ReductionRun.objects.filter( run_number=run_number).order_by('-run_version').first() # Check old run exists if old_reduction_run is None: context_dictionary['error'] = "Run number " + str( run_number) + " doesn't exist." use_current_script = request.POST.get('use_current_script', u"true").lower() == u"true" if use_current_script: script_text = InstrumentVariablesUtils().get_current_script_text( instrument.name)[0] default_variables = InstrumentVariablesUtils( ).get_default_variables(instrument.name) else: script_text = old_reduction_run.script default_variables = old_reduction_run.run_variables.all() new_variables = [] for key, value in request.POST.iteritems(): if 'var-' in key: name = None if 'var-advanced-' in key: name = key.replace('var-advanced-', '').replace('-', ' ') is_advanced = True if 'var-standard-' in key: name = key.replace('var-standard-', '').replace('-', ' ') is_advanced = False if name is not None: default_var = next( (x for x in default_variables if x.name == name), None) if not default_var: continue if len(value) > InstrumentVariable._meta.get_field( 'value').max_length: context_dictionary['error'] = 'Value given in ' + str( name) + ' is too long.' variable = RunVariable(name=default_var.name, value=value, is_advanced=is_advanced, type=default_var.type, help_text=default_var.help_text) new_variables.append(variable) if len(new_variables) == 0: context_dictionary[ 'error'] = 'No variables were found to be submitted.' # User can choose whether to overwrite with the re-run or create new data if request.POST.get('overwrite_checkbox') == 'on': overwrite_previous_data = True else: overwrite_previous_data = False if 'error' in context_dictionary: return context_dictionary run_description = request.POST.get('run_description') new_job = ReductionRunUtils().createRetryRun( old_reduction_run, script=script_text, overwrite=overwrite_previous_data, variables=new_variables, username=request.user.username, description=run_description) try: MessagingUtils().send_pending(new_job) context_dictionary['runs'].append(new_job) context_dictionary['variables'] = new_variables except Exception as e: new_job.delete() context_dictionary['error'] = 'Failed to send new job. (%s)' % str( e) return context_dictionary
def run_confirmation(request, instrument): """ Handles request for user to confirm re-run """ if request.method != 'POST': return redirect('instrument_summary', instrument=instrument.name) # POST # pylint:disable=no-member instrument = Instrument.objects.get(name=instrument) range_string = request.POST.get('run_range') queued_status = StatusUtils().get_queued() # pylint:disable=no-member queue_count = ReductionRun.objects.filter(instrument=instrument, status=queued_status).count() context_dictionary = { 'runs': [], 'variables': None, 'queued': queue_count, } try: run_numbers = input_processing.parse_user_run_numbers(range_string) except SyntaxError as exception: context_dictionary['error'] = exception.msg return context_dictionary # Determine user level to set a maximum limit to the number of runs that can be re-queued if request.user.is_superuser: max_runs = 500 elif request.user.is_staff: max_runs = 50 else: max_runs = 20 if len(run_numbers) > max_runs: context_dictionary["error"] = "{0} runs were requested, but only {1} runs can be " \ "queued at a time".format(len(run_numbers), max_runs) return context_dictionary # Check that RB numbers are the same for the range entered # pylint:disable=no-member rb_number = ReductionRun.objects.filter(instrument=instrument, run_number__in=run_numbers) \ .values_list('experiment__reference_number', flat=True).distinct() if len(rb_number) > 1: context_dictionary['error'] = 'Runs span multiple experiment numbers ' \ '(' + ','.join(str(i) for i in rb_number) + ')' \ ' please select a different range.' return context_dictionary for run_number in run_numbers: # pylint:disable=no-member matching_previous_runs_queryset = ReductionRun.objects.\ filter(instrument=instrument, run_number=run_number).order_by('-run_version') most_recent_previous_run = matching_previous_runs_queryset.first() # Check old run exists if most_recent_previous_run is None: context_dictionary['error'] = "Run number %s hasn't been" \ "ran by autoreduction yet." % str(run_number) # Check it is not currently queued queued_runs = matching_previous_runs_queryset.filter( status=queued_status).first() if queued_runs is not None: context_dictionary['error'] = "Run number {0} is already queued to run".\ format(queued_runs.run_number) return context_dictionary use_current_script = request.POST.get('use_current_script', u"true").lower() == u"true" if use_current_script: script_text = InstrumentVariablesUtils().get_current_script_text( instrument.name)[0] default_variables = InstrumentVariablesUtils( ).get_default_variables(instrument.name) else: script_text = most_recent_previous_run.script default_variables = most_recent_previous_run.run_variables.all() new_variables = [] for key, value in list(request.POST.items()): if 'var-' in key: name = None if 'var-advanced-' in key: name = key.replace('var-advanced-', '').replace('-', ' ') is_advanced = True if 'var-standard-' in key: name = key.replace('var-standard-', '').replace('-', ' ') is_advanced = False if name is not None: default_var = next( (x for x in default_variables if x.name == name), None) if not default_var: continue # pylint:disable=protected-access,no-member if len(value) > InstrumentVariable._meta.get_field( 'value').max_length: context_dictionary['error'] = 'Value given in {} is too long.'\ .format(str(name)) variable = RunVariable(name=default_var.name, value=value, is_advanced=is_advanced, type=default_var.type, help_text=default_var.help_text) new_variables.append(variable) if not new_variables: context_dictionary[ 'error'] = 'No variables were found to be submitted.' # User can choose whether to overwrite with the re-run or create new data overwrite_previous_data = bool( request.POST.get('overwrite_checkbox') == 'on') if 'error' in context_dictionary: return context_dictionary run_description = request.POST.get('run_description') max_desc_len = 200 if len(run_description) > max_desc_len: context_dictionary["error"] = "The description contains {0} characters, " \ "a maximum of {1} are allowed".\ format(len(run_description), max_desc_len) return context_dictionary new_job = ReductionRunUtils().createRetryRun( user_id=request.user.id, reduction_run=most_recent_previous_run, script=script_text, overwrite=overwrite_previous_data, variables=new_variables, description=run_description) try: MessagingUtils().send_pending(new_job) context_dictionary['runs'].append(new_job) context_dictionary['variables'] = new_variables # pylint:disable=broad-except except Exception as exception: new_job.delete() context_dictionary['error'] = 'Failed to send new job. (%s)' % str( exception) return context_dictionary
def run_confirmation(request, instrument=None): if request.method != 'POST': return redirect('instrument_summary', instrument=instrument.name) # POST instrument = Instrument.objects.get(name=instrument) run_numbers = [] if 'run_number' in request.POST: run_numbers.append(int(request.POST.get('run_number'))) else: range_string = request.POST.get('run_range').split(',') # Expand list for item in range_string: if '-' in item: split_range = item.split('-') run_numbers.extend(range(int(split_range[0]), int(split_range[1])+1)) # because this is a range, the end bound is exclusive! else: run_numbers.append(int(item)) # Make sure run numbers are distinct run_numbers = set(run_numbers) queued_status = StatusUtils().get_queued() queue_count = ReductionRun.objects.filter(instrument=instrument, status=queued_status).count() context_dictionary = { 'runs' : [], 'variables' : None, 'queued' : queue_count, } # Check that RB numbers are the same rb_number = ReductionRun.objects.filter(instrument=instrument, run_number__in=run_numbers).values_list('experiment__reference_number', flat=True).distinct() if len(rb_number) > 1: context_dictionary['error'] = 'Runs span multiple experiment numbers (' + ','.join(str(i) for i in rb_number) + ') please select a different range.' for run_number in run_numbers: old_reduction_run = ReductionRun.objects.filter(run_number=run_number).order_by('-run_version').first() use_current_script = request.POST.get('use_current_script', u"true").lower() == u"true" if use_current_script: script_text = InstrumentVariablesUtils().get_current_script_text(instrument.name)[0] default_variables = InstrumentVariablesUtils().get_default_variables(instrument.name) else: script_text = old_reduction_run.script default_variables = old_reduction_run.run_variables.all() new_variables = [] for key,value in request.POST.iteritems(): if 'var-' in key: name = None if 'var-advanced-' in key: name = key.replace('var-advanced-', '').replace('-', ' ') is_advanced = True if 'var-standard-' in key: name = key.replace('var-standard-', '').replace('-', ' ') is_advanced = False if name is not None: default_var = next((x for x in default_variables if x.name == name), None) if not default_var: continue if len(value) > InstrumentVariable._meta.get_field('value').max_length: context_dictionary['error'] = 'Value given in ' + str(name) + ' is too long.' variable = RunVariable( name = default_var.name , value = value , is_advanced = is_advanced , type = default_var.type , help_text = default_var.help_text ) new_variables.append(variable) if len(new_variables) == 0: context_dictionary['error'] = 'No variables were found to be submitted.' if 'error' in context_dictionary: return context_dictionary new_job = ReductionRunUtils().createRetryRun(old_reduction_run, script=script_text, variables=new_variables, username=request.user.username) try: MessagingUtils().send_pending(new_job) context_dictionary['runs'].append(new_job) context_dictionary['variables'] = new_variables except Exception as e: new_job.delete() context_dictionary['error'] = 'Failed to send new job. (%s)' % str(e) return context_dictionary