def create_run_records(self, message: Message): """ Creates or gets the necessary records to construct a ReductionRun """ # This must be done before looking up the run version to make sure the record exists experiment = db_access.get_experiment(message.rb_number) run_version = db_access.find_highest_run_version( experiment, run_number=str(message.run_number)) message.run_version = run_version instrument = db_access.get_instrument(str(message.instrument)) script = ReductionScript(instrument.name) script_text = script.text() # Make the new reduction run with the information collected so far reduction_run = db_records.create_reduction_run_record( experiment=experiment, instrument=instrument, message=message, run_version=run_version, script_text=script_text, status=self.status.get_queued()) reduction_run.save() # Create a new data location entry which has a foreign key linking it to the current # reduction run. The file path itself will point to a datafile # (e.g. "/isis/inst$/NDXWISH/Instrument/data/cycle_17_1/WISH00038774.nxs") data_location = self.data_model.DataLocation( file_path=message.data, reduction_run_id=reduction_run.pk) data_location.save() return reduction_run, message, instrument
def _find_run_in_database(self): """ Find a ReductionRun record in the database This includes a timeout to wait for several seconds to ensure the database has received the record in question :return: The resulting record """ wait_times = [0, 1, 2, 3, 5] results = [] for timeout in wait_times: # Wait before attempting database access print(f"Waiting for: {timeout}") time.sleep(timeout) # Check database has expected values instrument_record = db.get_instrument(self.instrument) results = self.database_client.data_model.ReductionRun.objects \ .filter(instrument=instrument_record.id) \ .filter(run_number=self.run_number) \ .select_related() \ .all() try: actual = results[0] except IndexError: # If no results found yet then continue continue # verbose values = "Completed" or "Error" if actual.status.value == 'c' or actual.status.value == 'e': print( f"Job reached {actual.status.value} status after {timeout} seconds" ) break return results
def test_get_instrument_invalid(self): """ Test: None is returned When: get_instrument is called with an instrument that does not exist """ actual = access.get_instrument('Fake instrument') self.assertIsNone(actual)
def show_variables_for_run(self, instrument_name, run_number=None): """ Look for the applicable variables for the given run number. If none are set, return an empty list (or QuerySet) anyway. """ instrument = db.get_instrument(instrument_name) variable_run_number = 0 # If we haven't been given a run number, we should try to find it. model = db.start_database().variable_model if not run_number: applicable_variables = model.InstrumentVariable.objects \ .filter(instrument_id=instrument.id) \ .order_by('-start_run') if applicable_variables: variable_run_number = applicable_variables[0].start_run else: variable_run_number = run_number # Find variable records associated with instrument variables variables = self.find_var_records_for_inst_vars( instrument.id, variable_run_number) # If we have found some variables then we want to use them by first making copies of them # and sending them back to be used. This means we don't alter the previous set of variables. # If we haven't found any variables, just return an empty list. if variables: self._update_variables(variables) new_variables = [] for variable in variables: new_variables.append(VariableUtils().copy_variable(variable)) return new_variables return []
def test_get_instrument_valid(self): """ Test: The correct instrument object is returned When: get_instrument is called on a valid database connection """ actual = access.get_instrument('GEM') self.assertIsNotNone(actual) self.assertEqual('GEM', actual.name)
def _find_run_in_database(self): """ Find a ReductionRun record in the database This includes a timeout to wait for several seconds to ensure the database has received the record in question :return: The resulting record """ instrument = db.get_instrument(self.instrument) return instrument.reduction_runs.filter(run_number=self.run_number)
def find_runs_in_database(self, run_number): """ Find all run versions in the database that relate to a given instrument and run number :param run_number: (int) The run to search for in the database :return: The result of the query """ instrument_record = db.get_instrument(self.instrument) result = self.database.data_model.ReductionRun.objects \ .filter(instrument=instrument_record.id) \ .filter(run_number=run_number) \ .order_by('-created') self.to_delete[run_number] = result return result
def test_get_instrument_does_not_exist(self, mock_save): """ Test: A new instrument record is created When: create is true and no matching instrument can be found in the database Note: As we do not actually save the object, we cannot assert values of it These are only stored when it enters the database """ database = access.start_database() actual = access.get_instrument('Not an instrument', create=True) self.assertIsNotNone(actual) self.assertIsInstance(actual, database.data_model.Instrument) mock_save.assert_called_once()
def show_variables_for_experiment(self, instrument_name, experiment_reference): """ Look for currently set variables for the experiment. If none are set, return an empty list (or QuerySet) anyway. """ instrument = db.get_instrument(instrument_name) model = db.start_database().variable_model ins_vars = model.InstrumentVariable.objects \ .filter(instrument_id=instrument.id) \ .filter(experiment_reference=experiment_reference) self._update_variables(ins_vars) return [VariableUtils().copy_variable(ins_var) for ins_var in ins_vars]
def find_run_in_database(test): """ Find a ReductionRun record in the database This includes a timeout to wait for several seconds to ensure the database has received the record in question :return: The resulting record """ instrument = db.get_instrument(test.instrument_name) if isinstance(test.run_number, list): args = {"run_number__in": test.run_number} else: args = {"run_number": test.run_number} return instrument.reduction_runs.filter(**args)
def _get_and_activate_db_inst(self, instrument_name): """ Gets the DB instrument record from the database, if one is not found it instead creates and saves the record to the DB, then returns it. """ # Check if the instrument is active or not in the MySQL database instrument = db_access.get_instrument(str(instrument_name), create=True) # Activate the instrument if it is currently set to inactive if not instrument.is_active: self._logger.info("Activating %s", instrument_name) instrument.is_active = 1 db_access.save_record(instrument) return instrument
def get_default_variables(self, instrument_name, reduce_script=None): """ Creates and returns a list of variables from the reduction script on disk for the instrument. If reduce_script is supplied, return variables using that script instead of the one on disk. """ if not reduce_script: reduce_script = self._load_reduction_vars_script(instrument_name) reduce_vars_module = self._read_script( reduce_script, os.path.join(self._reduction_script_location(instrument_name), 'reduce_vars.py')) if not reduce_vars_module: return [] instrument = db.get_instrument(instrument_name) variables = [] # pylint:disable=no-member if 'standard_vars' in dir(reduce_vars_module): variables.extend( self._create_variables(instrument, reduce_vars_module, reduce_vars_module.standard_vars, False)) if 'advanced_vars' in dir(reduce_vars_module): variables.extend( self._create_variables(instrument, reduce_vars_module, reduce_vars_module.advanced_vars, True)) for var in variables: var.tracks_script = True model = db.start_database().variable_model applicable_variables = model.InstrumentVariable.objects \ .filter(instrument_id=instrument.id) \ .order_by('-start_run') variable_run_number = applicable_variables[0].start_run variables = self.find_var_records_for_inst_vars( instrument.id, variable_run_number) return variables
def set_variables_for_runs(self, instrument_name, variables, start_run=0, end_run=None): """ Given a list of variables, we set them to be the variables used for subsequent runs in the given run range. If end_run is not supplied, these variables will be ongoing indefinitely. If start_run is not supplied, these variables will be set for all run numbers going backwards. """ instrument = db.get_instrument(instrument_name) # Ensure that the variables we set will be the only ones used for the range given. model = db.start_database().variable_model applicable_variables = model.InstrumentVariable.objects \ .filter(instrument_id=instrument.id) \ .filter(start_run=start_run) final_variables = [] if end_run: applicable_variables = applicable_variables.filter( start_run__lte=end_run) # pylint: disable=no-member after_variables = model.InstrumentVariable.objects \ .filter(instrument_id=instrument.id) \ .filter(start_run=end_run + 1) \ .order_by('start_run') # pylint: disable=no-member previous_variables = model.InstrumentVariable.objects \ .filter(instrument_id=instrument.id) \ .filter(start_run__lt=start_run) if applicable_variables and not after_variables: # The last set of applicable variables extends outside our range. # Find the last set. final_start = applicable_variables.order_by( '-start_run').first().start_run final_variables = list( applicable_variables.filter(start_run=final_start)) applicable_variables = applicable_variables.exclude( start_run=final_start) # Don't delete the final set. elif not applicable_variables and not after_variables and previous_variables: # There is a previous set that applies but doesn't start or end in the range. # Find the last set. final_start = previous_variables.order_by( '-start_run').first().start_run # Set them to apply after our variables. # pylint: disable=expression-not-assigned,no-member final_variables = list( previous_variables.filter(start_run=final_start)) [ VariableUtils().copy_variable(var).save() for var in final_variables ] # Also copy them to apply before our variables. elif not applicable_variables and not after_variables and not previous_variables: # There are instrument defaults which apply after our range. final_variables = self.get_default_variables(instrument_name) # Delete all currently saved variables that apply to the range. map(lambda var: var.delete(), applicable_variables) # Modify the range of the final set to after the specified range, if there is one. for var in final_variables: var.start_run = end_run + 1 db.save_record(var) # Then save the new ones. for var in variables: var.start_run = start_run db.save_record(var)