def getReductionRun(with_variables=True): instrument = InstrumentUtils().get_instrument("valid") experiment = Experiment(reference_number=1) experiment.save() reduction_run = ReductionRun( instrument=instrument, run_number=1, experiment=experiment, run_version=0, status=StatusUtils().get_queued(), script=getValidScript("reduce.py"), ) reduction_run.save() if with_variables: variable = RunVariable( reduction_run=reduction_run, name="test", value="testvalue1", type="text", is_advanced=False ) variable.save() reduction_run.run_variables.add(variable) variable = RunVariable( reduction_run=reduction_run, name="advanced_test", value="testvalue2", type="text", is_advanced=True ) variable.save() reduction_run.run_variables.add(variable) reduction_run.save() return reduction_run
def createRetryRun(self, reductionRun, script=None, variables=None, delay=0, username=None): """ Create a run ready for re-running based on the run provided. If variables (RunVariable) are provided, copy them and associate them with the new one, otherwise use the previous run's. If a script (as a string) is supplied then use it, otherwise use the previous run's. """ from reduction_variables.utils import InstrumentVariablesUtils, VariableUtils # find the previous run version, so we don't create a duplicate last_version = -1 for run in ReductionRun.objects.filter(experiment=reductionRun.experiment, run_number=reductionRun.run_number): last_version = max(last_version, run.run_version) try: # get the script to use: script_text = script if script is not None else reductionRun.script # create the run object and save it new_job = ReductionRun( instrument=reductionRun.instrument, run_number=reductionRun.run_number, run_name="", run_version=last_version + 1, experiment=reductionRun.experiment, started_by=username, status=StatusUtils().get_queued(), script=script_text, ) new_job.save() reductionRun.retry_run = new_job reductionRun.retry_when = timezone.now().replace(microsecond=0) + datetime.timedelta( seconds=delay if delay else 0 ) reductionRun.save() # copy the previous data locations for data_location in reductionRun.data_location.all(): new_data_location = DataLocation(file_path=data_location.file_path, reduction_run=new_job) new_data_location.save() new_job.data_location.add(new_data_location) if variables is not None: # associate the variables with the new run for var in variables: var.reduction_run = new_job var.save() else: # provide variables if they aren't already InstrumentVariablesUtils().create_variables_for_run(new_job) return new_job except: new_job.delete() raise
def data_ready(self): # Import within method to prevent cylindrical imports from reduction_variables.utils import InstrumentVariablesUtils, VariableUtils logger.info("Data ready for processing run %s on %s" % (str(self._data_dict['run_number']), self._data_dict['instrument'])) instrument = InstrumentUtils().get_instrument(self._data_dict['instrument']) # Activate the instrument if it is currently set to inactive if not instrument.is_active: instrument.is_active = True instrument.save() status = StatusUtils().get_skipped() if instrument.is_paused else StatusUtils().get_queued() last_run = ReductionRun.objects.filter(run_number=self._data_dict['run_number']).order_by('-run_version').first() highest_version = last_run.run_version if last_run is not None else -1 experiment, experiment_created = Experiment.objects.get_or_create(reference_number=self._data_dict['rb_number']) if experiment_created: experiment.save() script_text = InstrumentVariablesUtils().get_current_script_text(instrument.name)[0] run_version = highest_version+1 reduction_run = ReductionRun( run_number=self._data_dict['run_number'] , run_version=run_version , experiment=experiment , instrument=instrument , status=status , script=script_text ) reduction_run.save() self._data_dict['run_version'] = reduction_run.run_version data_location = DataLocation(file_path=self._data_dict['data'], reduction_run=reduction_run) data_location.save() variables = InstrumentVariablesUtils().create_variables_for_run(reduction_run) if not variables: logger.warning("No instrument variables found on %s for run %s" % (instrument.name, self._data_dict['run_number'])) reduction_script, arguments = ReductionRunUtils().get_script_and_arguments(reduction_run) self._data_dict['reduction_script'] = reduction_script self._data_dict['reduction_arguments'] = arguments if instrument.is_paused: logger.info("Run %s has been skipped" % self._data_dict['run_number']) else: self._client.send('/queue/ReductionPending', json.dumps(self._data_dict), priority=self._priority) logger.info("Run %s ready for reduction" % self._data_dict['run_number'])
def createReductionRun(self): instrument = InstrumentUtils().get_instrument("valid") instrument.save() experiment = Experiment(reference_number=1) experiment.save() reduction_run = ReductionRun(run_number=self.run_number, instrument=instrument, experiment=experiment, run_version=1, status=StatusUtils().get_queued(), script=getValidScript('reduce.py')) self.run_number += 1 reduction_run.save() variables = InstrumentVariablesUtils().get_variables_for_run( reduction_run) VariableUtils().save_run_variables(variables, reduction_run) return reduction_run
def createReductionRun(self): instrument = InstrumentUtils().get_instrument("valid") instrument.save() experiment = Experiment(reference_number=1) experiment.save() reduction_run = ReductionRun( run_number=self.run_number, instrument=instrument, experiment=experiment, run_version=1, status=StatusUtils().get_queued(), script=getValidScript("reduce.py"), ) self.run_number += 1 reduction_run.save() variables = InstrumentVariablesUtils().get_variables_for_run(reduction_run) VariableUtils().save_run_variables(variables, reduction_run) return reduction_run
def test_get_variables_for_run_default_variables(self): instrument = InstrumentUtils().get_instrument("valid") experiment = Experiment(reference_number=1) reduction_run = ReductionRun(run_number=123, instrument=instrument, experiment=experiment, run_version=1, status=StatusUtils().get_queued()) variables = InstrumentVariablesUtils().get_variables_for_run(reduction_run) self.assertNotEqual(variables, None, "Expecting some variables to be returned") self.assertNotEqual(variables, [], "Expecting some variables to be returned") self.assertTrue(len(variables) > 0, 'Expecting at least 1 variable returned') self.assertEqual(variables[0].experiment_reference, None, "Not expecting experiment_reference") self.assertEqual(variables[0].start_run, 1, "Expecting start run to be 1 but was %s" % variables[0].start_run)
def test_get_variables_for_run_experiment_reference(self): instrument = InstrumentUtils().get_instrument("valid") variable = InstrumentVariable( instrument=instrument, name="test", value="test", is_advanced=False, type="text", experiment_reference=99999, ) variable.save() experiment = Experiment(reference_number=99999) reduction_run = ReductionRun(run_number=1, instrument=instrument, experiment=experiment, run_version=1, status=StatusUtils().get_queued()) variables = InstrumentVariablesUtils().get_variables_for_run(reduction_run) self.assertNotEqual(variables, None, "Expecting some variables to be returned") self.assertNotEqual(variables, [], "Expecting some variables to be returned") self.assertTrue(len(variables) > 0, 'Expecting at least 1 variable returned') self.assertEqual(variables[0].experiment_reference, 99999, "Expecting instrument variables to match with experiment reference number")
def getReductionRun(with_variables=True): instrument = InstrumentUtils().get_instrument('valid') experiment = Experiment(reference_number=1) experiment.save() reduction_run = ReductionRun(instrument=instrument, run_number=1, experiment=experiment, run_version=0, status=StatusUtils().get_queued(), script=getValidScript('reduce.py')) reduction_run.save() if with_variables: variable = RunVariable(reduction_run=reduction_run,name='test',value='testvalue1',type='text',is_advanced=False) variable.save() reduction_run.run_variables.add(variable) variable = RunVariable(reduction_run=reduction_run,name='advanced_test',value='testvalue2',type='text',is_advanced=True) variable.save() reduction_run.run_variables.add(variable) reduction_run.save() return reduction_run
def createRetryRun(self, reductionRun, overwrite=None, script=None, variables=None, delay=0, username=None, description=''): """ Create a run ready for re-running based on the run provided. If variables (RunVariable) are provided, copy them and associate them with the new one, otherwise use the previous run's. If a script (as a string) is supplied then use it, otherwise use the previous run's. """ from reduction_variables.utils import InstrumentVariablesUtils, VariableUtils run_last_updated = reductionRun.last_updated if username == 'super': username = 1 # find the previous run version, so we don't create a duplicate last_version = -1 for run in ReductionRun.objects.filter( experiment=reductionRun.experiment, run_number=reductionRun.run_number): last_version = max(last_version, run.run_version) try: # get the script to use: script_text = script if script is not None else reductionRun.script # create the run object and save it new_job = ReductionRun(instrument=reductionRun.instrument, run_number=reductionRun.run_number, run_name=description, run_version=last_version + 1, experiment=reductionRun.experiment, started_by=username, status=StatusUtils().get_queued(), script=script_text, overwrite=overwrite) new_job.save() reductionRun.retry_run = new_job reductionRun.retry_when = timezone.now().replace( microsecond=0) + datetime.timedelta( seconds=delay if delay else 0) reductionRun.save() ReductionRun.objects.filter(id=reductionRun.id).update( last_updated=run_last_updated) # copy the previous data locations for data_location in reductionRun.data_location.all(): new_data_location = DataLocation( file_path=data_location.file_path, reduction_run=new_job) new_data_location.save() new_job.data_location.add(new_data_location) if variables is not None: # associate the variables with the new run for var in variables: var.reduction_run = new_job var.save() else: # provide variables if they aren't already InstrumentVariablesUtils().create_variables_for_run(new_job) return new_job except Exception as e: import traceback logger.error(traceback.format_exc()) logger.error(e) new_job.delete() raise
def createRetryRun(user_id, reduction_run, overwrite=None, script=None, variables=None, delay=0, description=''): """ Create a run ready for re-running based on the run provided. If variables (RunVariable) are provided, copy them and associate them with the new one, otherwise use the previous run's. If a script (as a string) is supplied then use it, otherwise use the previous run's. """ from reduction_variables.utils import InstrumentVariablesUtils run_last_updated = reduction_run.last_updated # find the previous run version, so we don't create a duplicate last_version = -1 # pylint:disable=no-member previous_run = ReductionRun.objects.filter(experiment=reduction_run.experiment, run_number=reduction_run.run_number) \ .order_by("-run_version").first() last_version = previous_run.run_version # get the script to use: script_text = script if script is not None else reduction_run.script # create the run object and save it new_job = ReductionRun(instrument=reduction_run.instrument, run_number=reduction_run.run_number, run_name=description, run_version=last_version + 1, experiment=reduction_run.experiment, started_by=user_id, status=StatusUtils().get_queued(), script=script_text, overwrite=overwrite) # Check record is safe to save try: new_job.full_clean() # pylint:disable=catching-non-exception except django.core.exceptions as exception: LOGGER.error(traceback.format_exc()) LOGGER.error(exception) raise # Attempt to save try: new_job.save() except ValueError as exception: # This usually indicates a F.K. constraint mismatch. Maybe we didn't get a record in? LOGGER.error(traceback.format_exc()) LOGGER.error(exception) raise reduction_run.retry_run = new_job reduction_run.retry_when = timezone.now().replace(microsecond=0) + datetime.timedelta( seconds=delay if delay else 0) reduction_run.save() # pylint:disable=no-member ReductionRun.objects.filter(id=reduction_run.id).update(last_updated=run_last_updated) # copy the previous data locations # pylint:disable=no-member for data_location in reduction_run.data_location.all(): new_data_location = DataLocation(file_path=data_location.file_path, reduction_run=new_job) new_data_location.save() new_job.data_location.add(new_data_location) if variables is not None: # associate the variables with the new run for var in variables: var.reduction_run = new_job var.save() else: # provide variables if they aren't already InstrumentVariablesUtils().create_variables_for_run(new_job) return new_job