def setUp(self): self.var_model = access.start_database().variable_model self.data_model = access.start_database().data_model self.valid_variable = self.var_model.Variable(name='test', value='value', type='text', is_advanced=False, help_text='help text') self.valid_inst_var = self.var_model.InstrumentVariable(name='test', value='value', is_advanced=False, type='text', help_text='help test', instrument_id=4, experiment_reference=54321, start_run=12345, tracks_script=1) self.reduction_run = self.data_model.ReductionRun(run_number=1111, run_version=0, run_name='run name', hidden_in_failviewer=0, admin_log='admin log', reduction_log='reduction log', created=datetime.datetime.utcnow(), last_updated=datetime.datetime.utcnow(), experiment_id=222, instrument_id=3, status_id=4, script='script', started_by=1)
def test_start_database(self, mock_connect): """ Test: The database is initialised When: start_database is called """ access.start_database() mock_connect.assert_called_once()
def reduction_complete(self, message: Message): """ Called when the destination queue was reduction_complete Updates the run as complete in the database. """ self._logger.info("Run %s has completed reduction", message.run_number) reduction_run = self.find_run(message) if not reduction_run: raise MissingReductionRunRecord(rb_number=message.rb_number, run_number=message.run_number, run_version=message.run_version) if not reduction_run.status.value == 'p': # verbose value = "Processing" raise InvalidStateException( "An invalid attempt to complete a reduction run that wasn't" " processing has been captured. " f" Experiment: {message.rb_number}," f" Run Number: {message.run_number}," f" Run Version {message.run_version}") reduction_run.status = self._utils.status.get_completed() reduction_run.finished = datetime.datetime.utcnow() reduction_run.message = message.message reduction_run.reduction_log = message.reduction_log reduction_run.admin_log = message.admin_log if message.reduction_data is not None: for location in message.reduction_data: model = db_access.start_database().data_model reduction_location = model \ .ReductionLocation(file_path=location, reduction_run=reduction_run) db_access.save_record(reduction_location) db_access.save_record(reduction_run)
def _make_pending_msg(reduction_run): """ Creates a dict message from the given run, ready to be sent to ReductionPending. :param reduction_run: (ReductionRun) database object representing a reduction run / job :return: (Message) A constructed Message object from the meta data in the reduction_run """ # Deferred import to avoid circular dependencies # pylint:disable=import-outside-toplevel from queue_processors.queue_processor.queueproc_utils \ .reduction_run_utils import ReductionRunUtils script, arguments = ReductionRunUtils().get_script_and_arguments( reduction_run) # Currently only support single location model = access.start_database().data_model data_location = model.DataLocation.filter_by( reduction_run_id=reduction_run.id).first() if data_location: data_path = data_location.file_path else: raise RuntimeError("No data path found for reduction run") message = Message(run_number=reduction_run.run_number, instrument=reduction_run.instrument.name, rb_number=str( reduction_run.experiment.reference_number), data=data_path, reduction_script=script, reduction_arguments=arguments, run_version=reduction_run.run_version, facility=FACILITY) return message
def get_script_and_arguments(reduction_run): """ Fetch the reduction script from the given run and return it as a string, along with a dictionary of arguments. """ model = access.start_database().variable_model run_variable_records = model.RunVariable.objects \ .filter(reduction_run_id=reduction_run.id) standard_vars, advanced_vars = {}, {} for run_variable in run_variable_records: variable = model.Variable.objects.filter( id=run_variable.variable_ptr_id).first() value = VariableUtils().convert_variable_to_type( variable.value, variable.type) if variable.is_advanced: advanced_vars[variable.name] = value else: standard_vars[variable.name] = value arguments = { 'standard_vars': standard_vars, 'advanced_vars': advanced_vars } return reduction_run.script, arguments
def show_variables_for_run(self, instrument_name, run_number=None): """ Look for the applicable variables for the given run number. If none are set, return an empty list (or QuerySet) anyway. """ instrument = db.get_instrument(instrument_name) variable_run_number = 0 # If we haven't been given a run number, we should try to find it. model = db.start_database().variable_model if not run_number: applicable_variables = model.InstrumentVariable.objects \ .filter(instrument_id=instrument.id) \ .order_by('-start_run') if applicable_variables: variable_run_number = applicable_variables[0].start_run else: variable_run_number = run_number # Find variable records associated with instrument variables variables = self.find_var_records_for_inst_vars( instrument.id, variable_run_number) # If we have found some variables then we want to use them by first making copies of them # and sending them back to be used. This means we don't alter the previous set of variables. # If we haven't found any variables, just return an empty list. if variables: self._update_variables(variables) new_variables = [] for variable in variables: new_variables.append(VariableUtils().copy_variable(variable)) return new_variables return []
def _create_variables(self, instrument, script, variable_dict, is_advanced): """ Create variables in the database. """ variables = [] for key, value in list(variable_dict.items()): str_value = str(value).replace('[', '').replace(']', '') if len(str_value) > 300: raise DataTooLong model = db.start_database().variable_model help_text = self._get_help_text('standard_vars', key, instrument.name, script) var_type = VariableUtils().get_type_string(value) # Please note: As instrument_variable inherits from Variable, the below creates BOTH an # an InstrumentVariable and Variable record in the database when saved. As such, # both sets of fields are required for initialisation. instrument_variable = model.InstrumentVariable( name=key, value=str_value, type=var_type, is_advanced=is_advanced, help_text=help_text, start_run=0, instrument_id=instrument.id, tracks_script=1) db.save_record(instrument_variable) variables.append(instrument_variable) return variables
def _database(self): """ Gets a handle to the database, starting it if required """ if not self._cached_db: self._cached_db = db_access.start_database() return self._cached_db
def derive_run_variable(instrument_var, reduction_run): """ Returns a RunJoin record for creation in the database. """ model = access.start_database().variable_model return model.RunVariable(name=instrument_var.name, value=instrument_var.value, is_advanced=instrument_var.is_advanced, type=instrument_var.type, help_text=instrument_var.help_text, reduction_run=reduction_run)
def test_get_experiment_create(self, mock_save): """ Test: An Experiment record is created When: get_experiment is called with create option True """ database = access.start_database() actual = access.get_experiment(rb_number=9999999, create=True) self.assertIsInstance(actual, database.data_model.Experiment) mock_save.assert_called_once()
def __init__(self, queue_listener): self._client = queue_listener self.status = StatusUtils() self.instrument_variable = InstrumentVariablesUtils() self._logger = logging.getLogger("handle_queue_message") self.database = db_access.start_database() self.data_model = self.database.data_model
def test_get_software_create(self, mock_save): """ Test: A Software record is created When: get_software is called with create option True """ database = access.start_database() actual = access.get_software(name='Fake', version='test', create=True) self.assertIsNotNone(actual) self.assertIsInstance(actual, database.data_model.Software) mock_save.assert_called_once()
def create_experiment_and_instrument(): "Creates a test experiment and instrument" db_handle = access.start_database() data_model = db_handle.data_model experiment, _ = data_model.Experiment.objects.get_or_create( reference_number=1231231) instrument, _ = data_model.Instrument.objects.get_or_create(name="ARMI", is_active=1, is_paused=0) return experiment, instrument
def log_error_and_notify(message): """ Helper method to log an error and save a notification """ logger.error(message) model = db.start_database().data_model notification = model.Notification(is_active=True, is_staff_only=True, severity='e', message=message) db.save_record(notification)
def save_run_variables(variables, reduction_run): """ Save reduction run variables in the database. """ model = access.start_database().variable_model logging.info('Saving run variables for %s', str(reduction_run.run_number)) run_variables = [] for variable in variables: run_var = model.RunVariable(variable=variable, reduction_run=reduction_run) run_var.save() run_variables.append(run_var) return run_variables
def show_variables_for_experiment(self, instrument_name, experiment_reference): """ Look for currently set variables for the experiment. If none are set, return an empty list (or QuerySet) anyway. """ instrument = db.get_instrument(instrument_name) model = db.start_database().variable_model ins_vars = model.InstrumentVariable.objects \ .filter(instrument_id=instrument.id) \ .filter(experiment_reference=experiment_reference) self._update_variables(ins_vars) return [VariableUtils().copy_variable(ins_var) for ins_var in ins_vars]
def setUp(self): self.manual_remove = ManualRemove(instrument='ARMI') # Setup database connection so it is possible to use # ReductionRun objects with valid meta data db_handle = access.start_database() self.data_model = db_handle.data_model self.variable_model = db_handle.variable_model self.experiment, self.instrument = create_experiment_and_instrument() self.run1 = make_test_run(self.experiment, self.instrument, "1") self.run2 = make_test_run(self.experiment, self.instrument, "2") self.run3 = make_test_run(self.experiment, self.instrument, "3")
def test_get_instrument_does_not_exist(self, mock_save): """ Test: A new instrument record is created When: create is true and no matching instrument can be found in the database Note: As we do not actually save the object, we cannot assert values of it These are only stored when it enters the database """ database = access.start_database() actual = access.get_instrument('Not an instrument', create=True) self.assertIsNotNone(actual) self.assertIsInstance(actual, database.data_model.Instrument) mock_save.assert_called_once()
def copy_metadata(new_var): """ Copy the source variable's metadata to the new one. """ source_var = variables[0] model = db.start_database().variable_model if isinstance(source_var, model.InstrumentVariable): map( lambda name: setattr(new_var, name, getattr(source_var, name)), ["instrument", "experiment_reference", "start_run"]) elif isinstance(source_var, model.RunVariable): # Create a run variable. VariableUtils().derive_run_variable(new_var, source_var.reduction_run) else: return db.save_record(new_var)
def find_run(self, message: Message): """ Find a reduction run in the database. """ experiment = db_access.get_experiment(message.rb_number) if not experiment: raise MissingExperimentRecord(rb_number=message.rb_number, run_number=message.run_number, run_version=message.run_version) self._logger.info( 'Finding a run with an experiment ID %s, run number %s and run ' 'version %s', experiment.id, int(message.run_number), int(message.run_version)) model = db_access.start_database().data_model reduction_run = model.ReductionRun.objects \ .filter(experiment_id=experiment.id) \ .filter(run_number=int(message.run_number)) \ .filter(run_version=int(message.run_version)) \ .first() return reduction_run
def get_default_variables(self, instrument_name, reduce_script=None): """ Creates and returns a list of variables from the reduction script on disk for the instrument. If reduce_script is supplied, return variables using that script instead of the one on disk. """ if not reduce_script: reduce_script = self._load_reduction_vars_script(instrument_name) reduce_vars_module = self._read_script( reduce_script, os.path.join(self._reduction_script_location(instrument_name), 'reduce_vars.py')) if not reduce_vars_module: return [] instrument = db.get_instrument(instrument_name) variables = [] # pylint:disable=no-member if 'standard_vars' in dir(reduce_vars_module): variables.extend( self._create_variables(instrument, reduce_vars_module, reduce_vars_module.standard_vars, False)) if 'advanced_vars' in dir(reduce_vars_module): variables.extend( self._create_variables(instrument, reduce_vars_module, reduce_vars_module.advanced_vars, True)) for var in variables: var.tracks_script = True model = db.start_database().variable_model applicable_variables = model.InstrumentVariable.objects \ .filter(instrument_id=instrument.id) \ .order_by('-start_run') variable_run_number = applicable_variables[0].start_run variables = self.find_var_records_for_inst_vars( instrument.id, variable_run_number) return variables
def make_variable_like_dict(variables: dict, help_dict: dict) -> Dict[str, object]: """ Returns a dict with unsaved Variable objects. Not ideal but better than returning a dict that needs to be kept up to date with the Variable interface. The right solution would be to remove all of this, and is captured in https://github.com/ISISScientificComputing/autoreduce/issues/1137 """ variable_model = access.start_database().variable_model.Variable result = {} for name, value in variables.items(): result[name] = variable_model( name=name, value=value, type=VariableUtils.get_type_string(value), help_text=help_dict["name"] if name in help_dict else "") return result
def find_var_records_for_inst_vars(instrument_id, start_run): """ Returns a list of the variables that are associated with a given instrument instance e.g. return all the Variable objects that are associated with a given set of InstrumentVariables with a start run of 123 :param instrument_id: (int) The id of the instrument to use :param start_run: (int) The start run of the instrument variable :return: (list) of all Variable objects matching the criteria """ model = db.start_database().variable_model instrument_var_records = model.InstrumentVariable.objects \ .filter(instrument_id=instrument_id) \ .filter(start_run=start_run) variables = [] for record in instrument_var_records: variables.append( model.Variable.objects.filter( id=record.variable_ptr_id).first()) return variables
def set_variables_for_runs(self, instrument_name, variables, start_run=0, end_run=None): """ Given a list of variables, we set them to be the variables used for subsequent runs in the given run range. If end_run is not supplied, these variables will be ongoing indefinitely. If start_run is not supplied, these variables will be set for all run numbers going backwards. """ instrument = db.get_instrument(instrument_name) # Ensure that the variables we set will be the only ones used for the range given. model = db.start_database().variable_model applicable_variables = model.InstrumentVariable.objects \ .filter(instrument_id=instrument.id) \ .filter(start_run=start_run) final_variables = [] if end_run: applicable_variables = applicable_variables.filter( start_run__lte=end_run) # pylint: disable=no-member after_variables = model.InstrumentVariable.objects \ .filter(instrument_id=instrument.id) \ .filter(start_run=end_run + 1) \ .order_by('start_run') # pylint: disable=no-member previous_variables = model.InstrumentVariable.objects \ .filter(instrument_id=instrument.id) \ .filter(start_run__lt=start_run) if applicable_variables and not after_variables: # The last set of applicable variables extends outside our range. # Find the last set. final_start = applicable_variables.order_by( '-start_run').first().start_run final_variables = list( applicable_variables.filter(start_run=final_start)) applicable_variables = applicable_variables.exclude( start_run=final_start) # Don't delete the final set. elif not applicable_variables and not after_variables and previous_variables: # There is a previous set that applies but doesn't start or end in the range. # Find the last set. final_start = previous_variables.order_by( '-start_run').first().start_run # Set them to apply after our variables. # pylint: disable=expression-not-assigned,no-member final_variables = list( previous_variables.filter(start_run=final_start)) [ VariableUtils().copy_variable(var).save() for var in final_variables ] # Also copy them to apply before our variables. elif not applicable_variables and not after_variables and not previous_variables: # There are instrument defaults which apply after our range. final_variables = self.get_default_variables(instrument_name) # Delete all currently saved variables that apply to the range. map(lambda var: var.delete(), applicable_variables) # Modify the range of the final set to after the specified range, if there is one. for var in final_variables: var.start_run = end_run + 1 db.save_record(var) # Then save the new ones. for var in variables: var.start_run = start_run db.save_record(var)
def setUp(self) -> None: self.database = access.start_database()
def create_retry_run(user_id, reduction_run, script=None, variables=None, delay=0): """ Create a run ready for re-running based on the run provided. If variables (RunVariable) are provided, copy them and associate them with the new one, otherwise use the previous run's. If a script (as a string) is supplied then use it, otherwise use the previous run's. """ model = access.start_database().data_model # find the previous run version, so we don't create a duplicate last_version = access.find_highest_run_version( reduction_run.experiment, reduction_run.run_number) # get the script to use: script_text = script if script is not None else reduction_run.script # create the run object and save it new_job = model.ReductionRun(run_number=reduction_run.run_number, run_version=last_version + 1, run_name="", experiment=reduction_run.experiment, instrument=reduction_run.instrument, script=script_text, status=StatusUtils().get_queued(), created=datetime.datetime.utcnow(), last_updated=datetime.datetime.utcnow(), message="", started_by=user_id, cancel=0, hidden_in_failviewer=0, admin_log="", reduction_log="") try: access.save_record(new_job) reduction_run.retry_run = new_job reduction_run.retry_when = \ datetime.datetime.utcnow() + datetime.timedelta(seconds=delay if delay else 0) access.save_record(reduction_run) data_locations = model.DataLocation.objects \ .filter(reduction_run_id=reduction_run.id) # copy the previous data locations for data_location in data_locations: new_data_location = model.DataLocation( file_path=data_location.file_path, reduction_run=new_job) access.save_record(new_data_location) if variables is not None: # associate the variables with the new run for var in variables: var.reduction_run = new_job access.save_record(var) else: # provide variables if they aren't already InstrumentVariablesUtils().create_variables_for_run(new_job) return new_job except: new_job.delete() raise
def __init__(self) -> None: self.model = db.start_database()