def test_get_default_variables_duplicate_var_name(self): variables = InstrumentVariablesUtils().get_default_variables('duplicate_var_reduce') self.assertNotEqual(variables, None, 'Expecting some variables returned') self.assertNotEqual(variables, [], 'Expecting some variables returned') self.assertTrue(len(variables) == 2, 'Expecting 2 variables returned') self.assertEqual(variables[0].instrument.name, 'duplicate_var_reduce', 'Expecting instrument to be "duplicate_var_reduce" but was %s' % variables[0].instrument)
def test_set_default_instrument_variables_empty(self): variables = InstrumentVariablesUtils().set_default_instrument_variables("empty_script", 1) instrument = InstrumentUtils().get_instrument("empty_script") saved_variables = list(InstrumentVariable.objects.filter(instrument=instrument, start_run=1)) self.assertEqual(variables, [], 'Expecting no variables returned') self.assertEqual(saved_variables, [], 'Expecting no variables saved')
def test_get_default_variables_successfull(self): variables = InstrumentVariablesUtils().get_default_variables(testInstrument) self.assertNotEqual(variables, None, 'Expecting some variables returned') self.assertNotEqual(variables, [], 'Expecting some variables returned') self.assertTrue(len(variables) > 0, 'Expecting at least 1 variable returned') self.assertEqual(variables[0].instrument.name, testInstrument, 'Expecting instrument to be "valid" but was %s' % variables[0].instrument)
def test_set_default_instrument_variables_no_start_run(self): variables = InstrumentVariablesUtils().set_default_instrument_variables("valid", None) self.assertNotEqual(variables, None, 'Expecting some variables returned') self.assertNotEqual(variables, [], 'Expecting some variables returned') self.assertTrue(len(variables) > 0, 'Expecting at least 1 variable returned') self.assertEqual(variables[0].start_run, 1, 'Expcting start_run to be set to 1 but was instead %s' % variables[0].start_run)
def run_summary(request, run_number, run_version=0): reduction_run = ReductionRun.objects.get(run_number=run_number, run_version=run_version) variables = reduction_run.run_variables.all() standard_vars = {} advanced_vars = {} for variable in variables: if variable.is_advanced: advanced_vars[variable.name] = variable else: standard_vars[variable.name] = variable current_variables = InstrumentVariablesUtils().get_default_variables( reduction_run.instrument.name) current_standard_variables = {} current_advanced_variables = {} for variable in current_variables: if variable.is_advanced: current_advanced_variables[variable.name] = variable else: current_standard_variables[variable.name] = variable context_dictionary = { 'run_number': run_number, 'run_version': run_version, 'standard_variables': standard_vars, 'advanced_variables': advanced_vars, 'default_standard_variables': standard_vars, 'default_advanced_variables': advanced_vars, 'current_standard_variables': current_standard_variables, 'current_advanced_variables': current_advanced_variables, 'instrument': reduction_run.instrument, } return render(request, 'snippets/run_variables.html', context_dictionary)
def test_get_default_variables_missing(self): initial_notification = list(Notification.objects.filter(is_active=True, is_staff_only=True)) variables = InstrumentVariablesUtils().get_default_variables('missing') updated_notification = list(Notification.objects.filter(is_active=True, is_staff_only=True)) self.assertNotEqual(variables, None, 'Expecting some variables returned') self.assertEqual(variables, [], 'Expecting an empty array returned') self.assertTrue(len(updated_notification) > len(initial_notification), 'Expecting a notification to be created')
def delete_instrument_variables(request, instrument=None, start=0, end=0, experiment_reference=None): instrument_name = instrument start, end = int(start), int(end) # We "save" an empty list to delete the previous variables. if experiment_reference is not None: InstrumentVariablesUtils().set_variables_for_experiment( instrument_name, [], experiment_reference) else: InstrumentVariablesUtils().set_variables_for_runs( instrument_name, [], start, end) return redirect('instrument_summary', instrument=instrument_name)
def test_get_current_script_text_successful(self): script_file = os.path.join(REDUCTION_SCRIPT_BASE % testInstrument, 'reduce.py') f = open(script_file, 'rb') script_binary = f.read() script, script_vars = InstrumentVariablesUtils().get_current_script_text(testInstrument) self.assertNotEqual(script, None, "Expecting a script to be returned") self.assertEqual(script, script_binary, "Expecting files to match")
def test_get_default_variables_pass_in_reduce_script(self): reduction_file = os.path.join(REDUCTION_SCRIPT_BASE % testInstrument, 'reduce_vars.py') reduce_script = open(reduction_file, "r").read() variables = InstrumentVariablesUtils().get_default_variables(testInstrument, reduce_script) self.assertNotEqual(variables, None, 'Expecting some variables returned') self.assertNotEqual(variables, [], 'Expecting some variables returned') self.assertTrue(len(variables) > 0, 'Expecting at least 1 variable returned') self.assertEqual(variables[0].instrument.name, testInstrument, 'Expecting instrument to be "valid" but was %s' % variables[0].instrument)
def test_get_current_and_upcoming_variables_test_upcoming(self): upcoming = InstrumentVariablesUtils().set_default_instrument_variables(testInstrument, 99999) current_variables, upcoming_variables_by_run, upcoming_variables_by_experiment = InstrumentVariablesUtils().get_current_and_upcoming_variables(testInstrument) self.assertNotEqual(upcoming_variables_by_run, None, "Expecting some upcoming variables to be returned") self.assertNotEqual(upcoming_variables_by_run, [], "Expecting some upcoming variables to be returned") self.assertTrue(len(upcoming_variables_by_run) > 0, 'Expecting at least 1 upcoming variable returned') self.assertEqual(len(upcoming), len(upcoming_variables_by_run), "Expecting same variables to be returned as created")
def submit_runs(request, instrument=None): """ Handles run submission request """ LOGGER.info('Submitting runs') # pylint:disable=no-member instrument = Instrument.objects.get(name=instrument) if request.method == 'GET': processing_status = StatusUtils().get_processing() queued_status = StatusUtils().get_queued() skipped_status = StatusUtils().get_skipped() # pylint:disable=no-member last_run = ReductionRun.\ objects.filter(instrument=instrument).\ exclude(status=skipped_status).order_by('-run_number').first() standard_vars = {} advanced_vars = {} default_variables = InstrumentVariablesUtils().get_default_variables( instrument.name) default_standard_variables = {} default_advanced_variables = {} for variable in default_variables: if variable.is_advanced: advanced_vars[variable.name] = variable default_advanced_variables[variable.name] = variable else: standard_vars[variable.name] = variable default_standard_variables[variable.name] = variable # pylint:disable=no-member context_dictionary = { 'instrument': instrument, 'last_instrument_run': last_run, 'processing': ReductionRun.objects.filter(instrument=instrument, status=processing_status), 'queued': ReductionRun.objects.filter(instrument=instrument, status=queued_status), 'standard_variables': standard_vars, 'advanced_variables': advanced_vars, 'default_standard_variables': default_standard_variables, 'default_advanced_variables': default_advanced_variables, } return context_dictionary
def test_set_default_instrument_variables_successful(self): variables = InstrumentVariablesUtils().set_default_instrument_variables("valid", 1) instrument = InstrumentUtils().get_instrument("valid") saved_variables = list(InstrumentVariable.objects.filter(instrument=instrument, start_run=1)) self.assertNotEqual(variables, None, 'Expecting some variables returned') self.assertNotEqual(saved_variables, None, 'Expecting some variables saved') self.assertNotEqual(variables, [], 'Expecting some variables returned') self.assertNotEqual(saved_variables, [], 'Expecting some variables saved') self.assertTrue(len(variables) > 0, 'Expecting at least 1 variable returned') self.assertEqual(variables[0].instrument.name, testInstrument, 'Expecting instrument to be "valid" but was %s' % variables[0].instrument) self.assertEqual(len(variables), len(saved_variables), "Expecting all returned variables to have been saved")
def test_get_variables_for_run_default_variables(self): instrument = InstrumentUtils().get_instrument("valid") experiment = Experiment(reference_number=1) reduction_run = ReductionRun(run_number=123, instrument=instrument, experiment=experiment, run_version=1, status=StatusUtils().get_queued()) variables = InstrumentVariablesUtils().get_variables_for_run(reduction_run) self.assertNotEqual(variables, None, "Expecting some variables to be returned") self.assertNotEqual(variables, [], "Expecting some variables to be returned") self.assertTrue(len(variables) > 0, 'Expecting at least 1 variable returned') self.assertEqual(variables[0].experiment_reference, None, "Not expecting experiment_reference") self.assertEqual(variables[0].start_run, 1, "Expecting start run to be 1 but was %s" % variables[0].start_run)
def current_default_variables(request, instrument=None): variables = InstrumentVariablesUtils().get_default_variables(instrument) standard_vars = {} advanced_vars = {} for variable in variables: if variable.is_advanced: advanced_vars[variable.name] = variable else: standard_vars[variable.name] = variable context_dictionary = { 'instrument': instrument, 'standard_variables': standard_vars, 'advanced_variables': advanced_vars, } return context_dictionary
def run_summary(request, instrument_name, run_number, run_version=0): """ Handles request to view the summary of a run """ # pylint:disable=no-member instrument = Instrument.objects.get(name=instrument_name) # pylint:disable=no-member reduction_run = ReductionRun.objects.get(instrument=instrument, run_number=run_number, run_version=run_version) variables = reduction_run.run_variables.all() standard_vars = {} advanced_vars = {} for variable in variables: if variable.is_advanced: advanced_vars[variable.name] = variable else: standard_vars[variable.name] = variable current_variables = InstrumentVariablesUtils().\ get_default_variables(reduction_run.instrument.name) current_standard_variables = {} current_advanced_variables = {} for variable in current_variables: if variable.is_advanced: current_advanced_variables[variable.name] = variable else: current_standard_variables[variable.name] = variable context_dictionary = { 'run_number': run_number, 'run_version': run_version, 'standard_variables': standard_vars, 'advanced_variables': advanced_vars, 'default_standard_variables': standard_vars, 'default_advanced_variables': advanced_vars, 'current_standard_variables': current_standard_variables, 'current_advanced_variables': current_advanced_variables, 'instrument': reduction_run.instrument, } return render(request, 'snippets/run_variables.html', context_dictionary)
def createReductionRun(self): instrument = InstrumentUtils().get_instrument("valid") instrument.save() experiment = Experiment(reference_number=1) experiment.save() reduction_run = ReductionRun(run_number=self.run_number, instrument=instrument, experiment=experiment, run_version=1, status=StatusUtils().get_queued(), script=getValidScript('reduce.py')) self.run_number += 1 reduction_run.save() variables = InstrumentVariablesUtils().get_variables_for_run( reduction_run) VariableUtils().save_run_variables(variables, reduction_run) return reduction_run
def test_get_variables_for_run_experiment_reference(self): instrument = InstrumentUtils().get_instrument("valid") variable = InstrumentVariable( instrument=instrument, name="test", value="test", is_advanced=False, type="text", experiment_reference=99999, ) variable.save() experiment = Experiment(reference_number=99999) reduction_run = ReductionRun(run_number=1, instrument=instrument, experiment=experiment, run_version=1, status=StatusUtils().get_queued()) variables = InstrumentVariablesUtils().get_variables_for_run(reduction_run) self.assertNotEqual(variables, None, "Expecting some variables to be returned") self.assertNotEqual(variables, [], "Expecting some variables to be returned") self.assertTrue(len(variables) > 0, 'Expecting at least 1 variable returned') self.assertEqual(variables[0].experiment_reference, 99999, "Expecting instrument variables to match with experiment reference number")
def instrument_summary(request, instrument): instrument = Instrument.objects.get(name=instrument) current_variables, upcoming_variables_by_run, upcoming_variables_by_experiment = InstrumentVariablesUtils( ).get_current_and_upcoming_variables(instrument.name) # Create a nested dictionary for by-run upcoming_variables_by_run_dict = {} for variable in upcoming_variables_by_run: if variable.start_run not in upcoming_variables_by_run_dict: upcoming_variables_by_run_dict[variable.start_run] = { 'run_start': variable.start_run, 'run_end': 0, # We'll fill this in after 'tracks_script': variable.tracks_script, 'variables': [], 'instrument': instrument, } upcoming_variables_by_run_dict[variable.start_run]['variables'].append( variable) # Fill in the run end numbers run_end = 0 for run_number in sorted(upcoming_variables_by_run_dict.iterkeys(), reverse=True): upcoming_variables_by_run_dict[run_number]['run_end'] = run_end run_end = max(run_number - 1, 0) current_start = current_variables[0].start_run next_run_starts = filter(lambda start: start > current_start, sorted(upcoming_variables_by_run_dict.keys())) current_end = next_run_starts[0] - 1 if next_run_starts else 0 current_vars = { 'run_start': current_start, 'run_end': current_end, 'tracks_script': current_variables[0].tracks_script, 'variables': current_variables, 'instrument': instrument, } # Move the upcoming vars into an ordered list upcoming_variables_by_run_ordered = [] for key in sorted(upcoming_variables_by_run_dict): upcoming_variables_by_run_ordered.append( upcoming_variables_by_run_dict[key]) # Create a nested dictionary for by-experiment upcoming_variables_by_experiment_dict = {} for variables in upcoming_variables_by_experiment: if variables.experiment_reference not in upcoming_variables_by_experiment_dict: upcoming_variables_by_experiment_dict[ variables.experiment_reference] = { 'experiment': variables.experiment_reference, 'variables': [], 'instrument': instrument, } upcoming_variables_by_experiment_dict[ variables.experiment_reference]['variables'].append(variables) # Move the upcoming vars into an ordered list upcoming_variables_by_experiment_ordered = [] for key in sorted(upcoming_variables_by_experiment_dict): upcoming_variables_by_experiment_ordered.append( upcoming_variables_by_experiment_dict[key]) sorted(upcoming_variables_by_experiment_ordered, key=lambda r: r['experiment']) context_dictionary = { 'instrument': instrument, 'current_variables': current_vars, 'upcoming_variables_by_run': upcoming_variables_by_run_ordered, 'upcoming_variables_by_experiment': upcoming_variables_by_experiment_ordered, } return render(request, 'snippets/instrument_summary_variables.html', context_dictionary)
def instrument_variables(request, instrument=None, start=0, end=0, experiment_reference=0): instrument_name = instrument start, end = int(start), int(end) if request.method == 'POST': # Submission to modify variables. varList = [ t for t in request.POST.items() if t[0].startswith("var-") ] # [("var-standard-"+name, value) or ("var-advanced-"+name, value)] newVarDict = { "".join(t[0].split("-")[2:]): t[1] for t in varList } # Remove the first two prefixes from the names to give {name: value} tracks_script = request.POST.get("track_script_checkbox") == "on" # Which variables should we modify? is_run_range = request.POST.get("variable-range-toggle-value", "True") == "True" start = int(request.POST.get("run_start", 1)) end = int(request.POST.get("run_end", None)) if request.POST.get( "run_end", None) else None experiment_reference = request.POST.get("experiment_reference_number", 1) def modifyVars(oldVars, newValues): for var in oldVars: if var.name in newValues: var.value = newValues[var.name] var.tracks_script = tracks_script if is_run_range: # Get the variables for the first run, modify them, and set them for the given range. instrVars = InstrumentVariablesUtils().show_variables_for_run( instrument_name, start) modifyVars(instrVars, newVarDict) InstrumentVariablesUtils().set_variables_for_runs( instrument_name, instrVars, start, end) else: # Get the variables for the experiment, modify them, and set them for the experiment. instrVars = InstrumentVariablesUtils( ).show_variables_for_experiment(instrument_name, experiment_reference) if not instrVars: instrVars = InstrumentVariablesUtils().get_default_variables( instrument_name) modifyVars(instrVars, newVarDict) InstrumentVariablesUtils().set_variables_for_experiment( instrument_name, instrVars, experiment_reference) return redirect('instrument_summary', instrument=instrument_name) else: instrument = InstrumentUtils().get_instrument(instrument_name) editing = (start > 0 or experiment_reference > 0) completed_status = StatusUtils().get_completed() processing_status = StatusUtils().get_processing() queued_status = StatusUtils().get_queued() try: latest_completed_run = ReductionRun.objects.filter( instrument=instrument, run_version=0, status=completed_status).order_by( '-run_number').first().run_number except AttributeError: latest_completed_run = 0 try: latest_processing_run = ReductionRun.objects.filter( instrument=instrument, run_version=0, status=processing_status).order_by( '-run_number').first().run_number except AttributeError: latest_processing_run = 0 if experiment_reference > 0: variables = InstrumentVariablesUtils( ).show_variables_for_experiment(instrument_name, experiment_reference) else: variables = InstrumentVariablesUtils().show_variables_for_run( instrument_name, start) if not editing or not variables: variables = InstrumentVariablesUtils().show_variables_for_run( instrument.name) if not variables: variables = InstrumentVariablesUtils().get_default_variables( instrument.name) editing = False standard_vars = {} advanced_vars = {} for variable in variables: if variable.is_advanced: advanced_vars[variable.name] = variable else: standard_vars[variable.name] = variable current_variables, upcoming_variables_by_run, upcoming_variables_by_experiment = InstrumentVariablesUtils( ).get_current_and_upcoming_variables(instrument.name) upcoming_run_variables = ','.join( list(set([str(var.start_run) for var in upcoming_variables_by_run])) ) # Unique, comma-joined list of all start runs belonging to the upcoming variables. default_variables = InstrumentVariablesUtils().get_default_variables( instrument.name) default_standard_variables = {} default_advanced_variables = {} for variable in default_variables: if variable.is_advanced: default_advanced_variables[variable.name] = variable else: default_standard_variables[variable.name] = variable context_dictionary = { 'instrument': instrument, 'last_instrument_run': ReductionRun.objects.filter(instrument=instrument).exclude( status=StatusUtils().get_skipped()).order_by('-run_number')[0], 'processing': ReductionRun.objects.filter(instrument=instrument, status=processing_status), 'queued': ReductionRun.objects.filter(instrument=instrument, status=queued_status), 'standard_variables': standard_vars, 'advanced_variables': advanced_vars, 'default_standard_variables': default_standard_variables, 'default_advanced_variables': default_advanced_variables, 'run_start': start, 'run_end': end, 'experiment_reference': experiment_reference, 'minimum_run_start': max(latest_completed_run, latest_processing_run), 'upcoming_run_variables': upcoming_run_variables, 'editing': editing, 'tracks_script': variables[0].tracks_script, } return context_dictionary
def run_confirmation(request, instrument): """ Handles request for user to confirm re-run """ if request.method != 'POST': return redirect('instrument_summary', instrument=instrument.name) # POST # pylint:disable=no-member instrument = Instrument.objects.get(name=instrument) range_string = request.POST.get('run_range') queued_status = StatusUtils().get_queued() # pylint:disable=no-member queue_count = ReductionRun.objects.filter(instrument=instrument, status=queued_status).count() context_dictionary = { 'runs': [], 'variables': None, 'queued': queue_count, } try: run_numbers = input_processing.parse_user_run_numbers(range_string) except SyntaxError as exception: context_dictionary['error'] = exception.msg return context_dictionary # Determine user level to set a maximum limit to the number of runs that can be re-queued if request.user.is_superuser: max_runs = 500 elif request.user.is_staff: max_runs = 50 else: max_runs = 20 if len(run_numbers) > max_runs: context_dictionary["error"] = "{0} runs were requested, but only {1} runs can be " \ "queued at a time".format(len(run_numbers), max_runs) return context_dictionary # Check that RB numbers are the same for the range entered # pylint:disable=no-member rb_number = ReductionRun.objects.filter(instrument=instrument, run_number__in=run_numbers) \ .values_list('experiment__reference_number', flat=True).distinct() if len(rb_number) > 1: context_dictionary['error'] = 'Runs span multiple experiment numbers ' \ '(' + ','.join(str(i) for i in rb_number) + ')' \ ' please select a different range.' return context_dictionary for run_number in run_numbers: # pylint:disable=no-member matching_previous_runs_queryset = ReductionRun.objects.\ filter(instrument=instrument, run_number=run_number).order_by('-run_version') most_recent_previous_run = matching_previous_runs_queryset.first() # Check old run exists if most_recent_previous_run is None: context_dictionary['error'] = "Run number %s hasn't been" \ "ran by autoreduction yet." % str(run_number) # Check it is not currently queued queued_runs = matching_previous_runs_queryset.filter( status=queued_status).first() if queued_runs is not None: context_dictionary['error'] = "Run number {0} is already queued to run".\ format(queued_runs.run_number) return context_dictionary use_current_script = request.POST.get('use_current_script', u"true").lower() == u"true" if use_current_script: script_text = InstrumentVariablesUtils().get_current_script_text( instrument.name)[0] default_variables = InstrumentVariablesUtils( ).get_default_variables(instrument.name) else: script_text = most_recent_previous_run.script default_variables = most_recent_previous_run.run_variables.all() new_variables = [] for key, value in list(request.POST.items()): if 'var-' in key: name = None if 'var-advanced-' in key: name = key.replace('var-advanced-', '').replace('-', ' ') is_advanced = True if 'var-standard-' in key: name = key.replace('var-standard-', '').replace('-', ' ') is_advanced = False if name is not None: default_var = next( (x for x in default_variables if x.name == name), None) if not default_var: continue # pylint:disable=protected-access,no-member if len(value) > InstrumentVariable._meta.get_field( 'value').max_length: context_dictionary['error'] = 'Value given in {} is too long.'\ .format(str(name)) variable = RunVariable(name=default_var.name, value=value, is_advanced=is_advanced, type=default_var.type, help_text=default_var.help_text) new_variables.append(variable) if not new_variables: context_dictionary[ 'error'] = 'No variables were found to be submitted.' # User can choose whether to overwrite with the re-run or create new data overwrite_previous_data = bool( request.POST.get('overwrite_checkbox') == 'on') if 'error' in context_dictionary: return context_dictionary run_description = request.POST.get('run_description') max_desc_len = 200 if len(run_description) > max_desc_len: context_dictionary["error"] = "The description contains {0} characters, " \ "a maximum of {1} are allowed".\ format(len(run_description), max_desc_len) return context_dictionary new_job = ReductionRunUtils().createRetryRun( user_id=request.user.id, reduction_run=most_recent_previous_run, script=script_text, overwrite=overwrite_previous_data, variables=new_variables, description=run_description) try: MessagingUtils().send_pending(new_job) context_dictionary['runs'].append(new_job) context_dictionary['variables'] = new_variables # pylint:disable=broad-except except Exception as exception: new_job.delete() context_dictionary['error'] = 'Failed to send new job. (%s)' % str( exception) return context_dictionary
def createRetryRun(self, reductionRun, overwrite=None, script=None, variables=None, delay=0, username=None, description=''): """ Create a run ready for re-running based on the run provided. If variables (RunVariable) are provided, copy them and associate them with the new one, otherwise use the previous run's. If a script (as a string) is supplied then use it, otherwise use the previous run's. """ from reduction_variables.utils import InstrumentVariablesUtils, VariableUtils run_last_updated = reductionRun.last_updated if username == 'super': username = 1 # find the previous run version, so we don't create a duplicate last_version = -1 for run in ReductionRun.objects.filter( experiment=reductionRun.experiment, run_number=reductionRun.run_number): last_version = max(last_version, run.run_version) try: # get the script to use: script_text = script if script is not None else reductionRun.script # create the run object and save it new_job = ReductionRun(instrument=reductionRun.instrument, run_number=reductionRun.run_number, run_name=description, run_version=last_version + 1, experiment=reductionRun.experiment, started_by=username, status=StatusUtils().get_queued(), script=script_text, overwrite=overwrite) new_job.save() reductionRun.retry_run = new_job reductionRun.retry_when = timezone.now().replace( microsecond=0) + datetime.timedelta( seconds=delay if delay else 0) reductionRun.save() ReductionRun.objects.filter(id=reductionRun.id).update( last_updated=run_last_updated) # copy the previous data locations for data_location in reductionRun.data_location.all(): new_data_location = DataLocation( file_path=data_location.file_path, reduction_run=new_job) new_data_location.save() new_job.data_location.add(new_data_location) if variables is not None: # associate the variables with the new run for var in variables: var.reduction_run = new_job var.save() else: # provide variables if they aren't already InstrumentVariablesUtils().create_variables_for_run(new_job) return new_job except Exception as e: import traceback logger.error(traceback.format_exc()) logger.error(e) new_job.delete() raise
def test_get_current_and_upcoming_variables_test_upcomming_by_experiment_not_in_ICAT(self, mock_icat_call): mock_icat_call.return_value = [] instrument = InstrumentUtils().get_instrument("valid") variable = InstrumentVariable( instrument=instrument, name="test", value="test", is_advanced=False, type="text", experiment_reference=99999, ) variable.save() current_variables, upcoming_variables_by_run, upcoming_variables_by_experiment = InstrumentVariablesUtils().get_current_and_upcoming_variables(testInstrument) self.assertTrue(len(upcoming_variables_by_experiment) == 0, "Expecting no upcoming experiment variables to be returned")
def run_confirmation(request, instrument=None): if request.method != 'POST': return redirect('instrument_summary', instrument=instrument.name) # POST instrument = Instrument.objects.get(name=instrument) run_numbers = [] if 'run_number' in request.POST: run_numbers.append(int(request.POST.get('run_number'))) else: range_string = request.POST.get('run_range').split(',') # Expand list for item in range_string: if '-' in item: split_range = item.split('-') run_numbers.extend( range(int(split_range[0]), int(split_range[1]) + 1) ) # because this is a range, the end bound is exclusive! else: run_numbers.append(int(item)) # Make sure run numbers are distinct run_numbers = set(run_numbers) queued_status = StatusUtils().get_queued() queue_count = ReductionRun.objects.filter(instrument=instrument, status=queued_status).count() context_dictionary = { 'runs': [], 'variables': None, 'queued': queue_count, } # Check that RB numbers are the same rb_number = ReductionRun.objects.filter( instrument=instrument, run_number__in=run_numbers).values_list('experiment__reference_number', flat=True).distinct() if len(rb_number) > 1: context_dictionary[ 'error'] = 'Runs span multiple experiment numbers (' + ','.join( str(i) for i in rb_number) + ') please select a different range.' for run_number in run_numbers: old_reduction_run = ReductionRun.objects.filter( run_number=run_number).order_by('-run_version').first() # Check old run exists if old_reduction_run is None: context_dictionary['error'] = "Run number " + str( run_number) + " doesn't exist." use_current_script = request.POST.get('use_current_script', u"true").lower() == u"true" if use_current_script: script_text = InstrumentVariablesUtils().get_current_script_text( instrument.name)[0] default_variables = InstrumentVariablesUtils( ).get_default_variables(instrument.name) else: script_text = old_reduction_run.script default_variables = old_reduction_run.run_variables.all() new_variables = [] for key, value in request.POST.iteritems(): if 'var-' in key: name = None if 'var-advanced-' in key: name = key.replace('var-advanced-', '').replace('-', ' ') is_advanced = True if 'var-standard-' in key: name = key.replace('var-standard-', '').replace('-', ' ') is_advanced = False if name is not None: default_var = next( (x for x in default_variables if x.name == name), None) if not default_var: continue if len(value) > InstrumentVariable._meta.get_field( 'value').max_length: context_dictionary['error'] = 'Value given in ' + str( name) + ' is too long.' variable = RunVariable(name=default_var.name, value=value, is_advanced=is_advanced, type=default_var.type, help_text=default_var.help_text) new_variables.append(variable) if len(new_variables) == 0: context_dictionary[ 'error'] = 'No variables were found to be submitted.' # User can choose whether to overwrite with the re-run or create new data if request.POST.get('overwrite_checkbox') == 'on': overwrite_previous_data = True else: overwrite_previous_data = False if 'error' in context_dictionary: return context_dictionary run_description = request.POST.get('run_description') new_job = ReductionRunUtils().createRetryRun( old_reduction_run, script=script_text, overwrite=overwrite_previous_data, variables=new_variables, username=request.user.username, description=run_description) try: MessagingUtils().send_pending(new_job) context_dictionary['runs'].append(new_job) context_dictionary['variables'] = new_variables except Exception as e: new_job.delete() context_dictionary['error'] = 'Failed to send new job. (%s)' % str( e) return context_dictionary
def test_get_current_and_upcoming_variables_test_current(self): current_variables, upcoming_variables_by_run, upcoming_variables_by_experiment = InstrumentVariablesUtils().get_current_and_upcoming_variables(testInstrument) self.assertNotEqual(current_variables, None, "Expecting some current variables to be returned") self.assertNotEqual(current_variables, [], "Expecting some current variables to be returned") self.assertTrue(len(current_variables) > 0, 'Expecting at least 1 current variable returned')
def test_get_default_variables_empty(self): variables = InstrumentVariablesUtils().get_default_variables('empty_script') self.assertNotEqual(variables, None, 'Expecting some variables returned') self.assertTrue(len(variables) == 0, 'Expecting an empty array returned')
def test_get_current_script_text_missing(self): script, script_vars = InstrumentVariablesUtils().get_current_script_text('missing') self.assertEqual(script, None, "Expecting script to be None") self.assertEqual(script_vars, None, "Expecting script_vars to be None")
def test_get_current_and_upcoming_variables_test_upcomming_by_experiment(self, mock_icat_call): mock_icat_call.return_value = [99999] instrument = InstrumentUtils().get_instrument("valid") variable = InstrumentVariable( instrument=instrument, name="test", value="test", is_advanced=False, type="text", experiment_reference=99999, ) variable.save() current_variables, upcoming_variables_by_run, upcoming_variables_by_experiment = InstrumentVariablesUtils().get_current_and_upcoming_variables(testInstrument) self.assertNotEqual(upcoming_variables_by_experiment, None, "Expecting some upcoming experiment variables to be returned") self.assertNotEqual(upcoming_variables_by_experiment, [], "Expecting some upcoming experiment variables to be returned") self.assertTrue(len(upcoming_variables_by_experiment) > 0, 'Expecting at least 1 upcoming experiment variable returned')
def preview_script(request, instrument=None, run_number=0, experiment_reference=0): # Can't use login decorator as this is requested via AJAX; need to return an error message on failure. if not has_valid_login(request): redirect_response = handle_redirect(request) if request.method == 'GET': return redirect_response else: error = {'redirect_url': redirect_response.url} return HttpResponseForbidden(json.dumps(error)) # Make our own little function to use the permissions decorator on; if we catch a PermissionDenied, we should give a 403 error. # We also don't want to check the instrument in this case, since run-specific scripts ought to be viewable without owning the instrument. @check_permissions def permission_test(request, run_number=0, experiment_reference=0): pass try: permission_test(request, run_number, experiment_reference) except PermissionDenied: return HttpResponseForbidden() # Find the reduction run to get the script for. if request.method == 'GET': reduction_run = ReductionRun.objects.filter(run_number=run_number) use_current_script = False elif request.method == 'POST': lookup_run_number = request.POST.get('run_number', None) lookup_run_version = request.POST.get('run_version', None) use_current_script = request.POST.get( 'use_current_script', default=u"false").lower() == u"true" reduction_run = ReductionRun.objects.filter( run_number=lookup_run_number, run_version=lookup_run_version) # Get the script text and variables from the given run; note the differing types of the variables in the two cases, which don't matter (we only access the parent Variable attributes). if reduction_run and not use_current_script: script_text = reduction_run[0].script script_variables = reduction_run[0].run_variables.all( ) # [InstrumentVariable] else: script_text = InstrumentVariablesUtils().get_current_script_text( instrument)[0] script_variables = InstrumentVariablesUtils().show_variables_for_run( instrument) # [RunVariable] def format_header(string): # Gives a #-enclosed string that looks header-y lines = ["#" * (len(string) + 8)] * 4 lines[2:2] = ["##" + " " * (len(string) + 4) + "##"] * 3 lines[3] = lines[3][:4] + string + lines[3][-4:] lines += [""] * 1 return lines def format_class(variables, name, indent): # Gives a Python class declaration with variable dicts as required. lines = ["class " + name + ":"] standard_vars, advanced_vars = filter(lambda var: not var.is_advanced, variables), filter( lambda var: var.is_advanced, variables) def make_dict(variables, name): var_dict = { v.name: VariableUtils().convert_variable_to_type(v.value, v.type) for v in variables } return indent + name + " = " + str(var_dict) lines.append(make_dict(standard_vars, "standard_vars")) lines.append(make_dict(advanced_vars, "advanced_vars")) lines += [""] * 3 return lines def replace_variables(text): # Find the import/s for the reduction variables and remove them. lines = text.split("\n") imports = filter( lambda line: line.lstrip().startswith("import") or line.lstrip(). startswith("from"), lines) import_vars = filter(lambda line: "reduce_vars" in line, imports) lines = [line for line in lines if line not in import_vars] # Print the header for the reduce.py script lines = format_header("Reduction script - reduce.py") + lines # Figure out space/tabs indent = " " # Defaults to PEP 8 standard! if filter(lambda line: line.startswith("\t"), lines): indent = "\t" if import_vars: # Assume the import is of the form 'import reduce_vars as {name}' or 'import reduce_vars' classname = import_vars[0].rstrip().split(" ")[-1] # Print the variables and a header for them lines = format_header("Reduction variables") + format_class( script_variables, classname, indent) + lines return "\n".join(lines) new_script_text = replace_variables(script_text) response = HttpResponse(content_type='application/x-python') response['Content-Disposition'] = 'attachment; filename=reduce.py' response.write(new_script_text) return response
def createRetryRun(user_id, reduction_run, overwrite=None, script=None, variables=None, delay=0, description=''): """ Create a run ready for re-running based on the run provided. If variables (RunVariable) are provided, copy them and associate them with the new one, otherwise use the previous run's. If a script (as a string) is supplied then use it, otherwise use the previous run's. """ from reduction_variables.utils import InstrumentVariablesUtils run_last_updated = reduction_run.last_updated # find the previous run version, so we don't create a duplicate last_version = -1 # pylint:disable=no-member previous_run = ReductionRun.objects.filter(experiment=reduction_run.experiment, run_number=reduction_run.run_number) \ .order_by("-run_version").first() last_version = previous_run.run_version # get the script to use: script_text = script if script is not None else reduction_run.script # create the run object and save it new_job = ReductionRun(instrument=reduction_run.instrument, run_number=reduction_run.run_number, run_name=description, run_version=last_version + 1, experiment=reduction_run.experiment, started_by=user_id, status=StatusUtils().get_queued(), script=script_text, overwrite=overwrite) # Check record is safe to save try: new_job.full_clean() # pylint:disable=catching-non-exception except django.core.exceptions as exception: LOGGER.error(traceback.format_exc()) LOGGER.error(exception) raise # Attempt to save try: new_job.save() except ValueError as exception: # This usually indicates a F.K. constraint mismatch. Maybe we didn't get a record in? LOGGER.error(traceback.format_exc()) LOGGER.error(exception) raise reduction_run.retry_run = new_job reduction_run.retry_when = timezone.now().replace(microsecond=0) + datetime.timedelta( seconds=delay if delay else 0) reduction_run.save() # pylint:disable=no-member ReductionRun.objects.filter(id=reduction_run.id).update(last_updated=run_last_updated) # copy the previous data locations # pylint:disable=no-member for data_location in reduction_run.data_location.all(): new_data_location = DataLocation(file_path=data_location.file_path, reduction_run=new_job) new_data_location.save() new_job.data_location.add(new_data_location) if variables is not None: # associate the variables with the new run for var in variables: var.reduction_run = new_job var.save() else: # provide variables if they aren't already InstrumentVariablesUtils().create_variables_for_run(new_job) return new_job