def setUp(self): """ Start all external services """ # Get all clients self.database_client = DatabaseClient() self.database_client.connect() try: self.queue_client, self.listener = main() except ConnectionException as err: raise RuntimeError( "Could not connect to ActiveMQ - check you credentials. If running locally check that " "ActiveMQ is running and started by `python setup.py start`" ) from err # Add placeholder variables: # these are used to ensure runs are deleted even if test fails before completion self.instrument = 'ARMI' self.rb_number = 1234567 self.run_number = 101 # Create test archive and add data self.data_archive = DataArchive([self.instrument], 19, 19) self.data_archive.create() # Create and send json message to ActiveMQ self.data_ready_message = Message(rb_number=self.rb_number, instrument=self.instrument, run_number=self.run_number, description="This is a system test", facility="ISIS", started_by=0)
def test_valid_connection(self): """ Test: Access is established with a valid connection When: connect is called while valid credentials are held """ client = DatabaseClient() client.connect() self.assertTrue(client._test_connection())
def __init__(self, instrument): """ :param instrument: (str) The name of the instrument associated with runs """ self.database = DatabaseClient() self.database.connect() self.to_delete = {} self.instrument = instrument
def start_database(): """ Create and connect a database client :return: The connected database client """ database = DatabaseClient() database.connect() return database
def test_model_access_after_connect(self): """ Test: Models variables can be accessed When: connect has been called successfully """ client = DatabaseClient() client.connect() self.assertIsNotNone(client.data_model) self.assertIsNotNone(client.variable_model) self.assertIsNotNone(client.data_model.Instrument.objects.all()) self.assertIsNotNone(client.variable_model.Variable.objects.all())
def test_get_from_database(self): """ Test: Data for a given run can be retrieved from the database in the expected format When: get_location_and_rb_from_database is called and the data is present in the database """ db_client = DatabaseClient() db_client.connect() actual = ms.get_location_and_rb_from_database(db_client, 'MUSR', 2) # Values from testing database expected = ('test/file/path/2.raw', 123) self.assertEqual(expected, actual)
def login_database(): """ Log into the DatabaseClient :return: The client connected, or None if failed """ print("Logging into Database") database_client = DatabaseClient() try: database_client.connect() except ConnectionException: print( "Couldn't connect to Database. Continuing without Database connection." ) database_client = None return database_client
def test_default_init(self): """ Test: Class variables are created and set When: DatabaseClient is initialised with default credentials """ client = DatabaseClient() self.assertIsNone(client.data_model) self.assertIsNone(client.variable_model)
def test_invalid_connection(self, mock_var_query, mock_data_query): """ Test: A ConnectionException is raised When: connect is called while invalid connection are held """ mock_data_query.Instrument.objects.first.side_effect = RuntimeError mock_var_query.Instrument.objects.first.side_effect = RuntimeError client = DatabaseClient() self.assertRaises(ConnectionException, client._test_connection)
def setUp(self): """ Start all external services """ # Get all clients self.database_client = DatabaseClient() self.database_client.connect() self.queue_client = QueueClient(ACTIVEMQ_SETTINGS) self.queue_client.connect() # Create test archive and add data self.data_archive_creator = DataArchiveCreator(os.path.join( get_project_root()), overwrite=True) self.archive_explorer = ArchiveExplorer( os.path.join(get_project_root(), 'data-archive')) # Add placeholder variables: # these are used to ensure runs are deleted even if test fails before completion self.instrument = None self.rb_number = None self.run_number = None
def test_disconnect_client(self): """ Test: Connection is stopped and connection variables are set to None When: disconnect is called while a valid connection is currently established """ django_client = DatabaseClient() django_client.connect() self.assertTrue(django_client._test_connection()) django_client.disconnect() self.assertIsNone(django_client.data_model) self.assertIsNone(django_client.variable_model)
def setUpClass(cls): """ Start all external services """ super().setUpClass() cls.database_client = DatabaseClient() cls.database_client.connect() try: cls.queue_client, cls.listener = main() except ConnectionException as err: raise RuntimeError( "Could not connect to ActiveMQ - check you credentials. If running locally check that " "ActiveMQ is running and started by `python setup.py start`" ) from err cls.instrument_name = "TestInstrument" cls.rb_number = 1234567 cls.run_number = 99999 cls.data_archive = DataArchive([cls.instrument_name], 21, 21) cls.data_archive.create() cls.data_archive.add_reduce_vars_script( cls.instrument_name, """standard_vars={"variable1":"test_variable_value_123"}""")
def setUpClass(cls): super().setUpClass() cls.instrument_name = "TestInstrument" cls.data_archive = DataArchive([cls.instrument_name], 21, 21) cls.data_archive.create() cls.data_archive.add_reduction_script(cls.instrument_name, """print('some text')""") cls.data_archive.add_reduce_vars_script( cls.instrument_name, f"""standard_vars={{"variable1":"{REDUCE_VARS_DEFAULT_VALUE}"}}""") cls.database_client = DatabaseClient() cls.database_client.connect() try: cls.queue_client, cls.listener = main() except ConnectionException as err: raise RuntimeError( "Could not connect to ActiveMQ - check you credentials. If running locally check that " "ActiveMQ is running and started by `python setup.py start`" ) from err cls.instrument_name = "TestInstrument" cls.rb_number = 1234567 cls.run_number = 99999
class ManualRemove: """ Handles removing a run from the database """ def __init__(self, instrument): """ :param instrument: (str) The name of the instrument associated with runs """ self.database = DatabaseClient() self.database.connect() self.to_delete = {} self.instrument = instrument def find_runs_in_database(self, run_number): """ Find all run versions in the database that relate to a given instrument and run number :param run_number: (int) The run to search for in the database :return: The result of the query """ instrument_record = db.get_instrument(self.instrument) result = self.database.data_model.ReductionRun.objects \ .filter(instrument=instrument_record.id) \ .filter(run_number=run_number) \ .order_by('-created') self.to_delete[run_number] = result return result def process_results(self): """ Process all the results what to do with the run based on the result of database query """ copy_to_delete = self.to_delete.copy() for key, value in copy_to_delete.items(): if not value: self.run_not_found(run_number=key) if len(value) == 1: continue if len(value) > 1: self.multiple_versions_found(run_number=key) def run_not_found(self, run_number): """ Inform user and remove key from dictionary :param run_number: (int) The run to remove from the dictionary """ print('No runs found associated with {} for instrument {}'.format( run_number, self.instrument)) del self.to_delete[run_number] def multiple_versions_found(self, run_number): """ Ask the user which versions they want to remove Update the self.to_delete dictionary by removing unwanted versions :param run_number: (int) The run number with multiple versions """ # Display run_number - title - version for all matching runs print("Discovered multiple reduction versions for {}{}:".format( self.instrument, run_number)) for run in self.to_delete[run_number]: print("\tv{} - {}".format(run.run_version, run.run_name)) # Get user input for which versions they wish to delete user_input = input( "Which runs would you like to delete (e.g. 1,2,3): ") input_valid, user_input = self.validate_csv_input(user_input) while input_valid is False: user_input = input( 'Input of \'{}\' was invalid. ' 'Please provide a comma separated list of values:') input_valid, user_input = self.validate_csv_input(user_input) # Remove runs that the user does NOT want to delete from the delete list for reduction_job in self.to_delete[run_number]: if not int(reduction_job.run_version) in user_input: self.to_delete[run_number].remove(reduction_job) def delete_records(self): """ Delete all records from the database that match those found in self.to_delete """ # Make a copy to ensure dict being iterated stays same size through processing to_delete_copy = self.to_delete.copy() for run_number, job_list in to_delete_copy.items(): for version in job_list: # Delete the specified version print('{}{}:'.format(self.instrument, run_number)) self.delete_reduction_location(version.id) self.delete_data_location(version.id) self.delete_variables(version.id) self.delete_reduction_run(version.id) # Remove deleted run from dictionary del self.to_delete[run_number] def delete_reduction_location(self, reduction_run_id): """ Delete a ReductionLocation record from the database :param reduction_run_id: (int) The id of the associated reduction job """ self.database.data_model.ReductionLocation.objects \ .filter(reduction_run_id=reduction_run_id) \ .delete() def delete_data_location(self, reduction_run_id): """ Delete a DataLocation record from the database :param reduction_run_id: (int) The id of the associated reduction job """ self.database.data_model.DataLocation.objects \ .filter(reduction_run_id=reduction_run_id) \ .delete() def delete_variables(self, reduction_run_id): """ Removes all the RunVariable records associated with a given ReductionRun from the database :param reduction_run_id: (int) The id of the associated reduction job """ run_variables = self.find_variables_of_reduction(reduction_run_id) for record in run_variables: self.database.variable_model.RunVariable.objects \ .filter(variable_ptr_id=record.variable_ptr_id) \ .delete() def find_variables_of_reduction(self, reduction_run_id): """ Find all the RunVariable records in the database associated with a reduction job :param reduction_run_id: (int) The id of the reduction job to filter by :return: (QuerySet) of the associated RunVariables """ return self.database.variable_model.RunVariable.objects \ .filter(reduction_run_id=reduction_run_id) def delete_reduction_run(self, reduction_run_id): """ Delete a ReductionRun record from the database :param reduction_run_id: (int) The id of the associated reduction job """ self.database.data_model.ReductionRun.objects \ .filter(id=reduction_run_id) \ .delete() @staticmethod def validate_csv_input(user_input): """ checks if a comma separated list was provided :return: (tuple) = (bool - is valid? , list - csv as list (empty list if invalid)) """ processed_input = [] if ',' in user_input: versions_to_delete = user_input.split(',') for number in versions_to_delete: try: number = int(number) processed_input.append(number) except ValueError: return False, [] else: try: user_input = int(user_input) processed_input.append(user_input) except ValueError: return False, [] return True, processed_input
class TestEndToEnd(unittest.TestCase): """ Class to test pipelines in autoreduction""" def setUp(self): """ Start all external services """ # Get all clients self.database_client = DatabaseClient() self.database_client.connect() try: self.queue_client, self.listener = main() except ConnectionException as err: raise RuntimeError( "Could not connect to ActiveMQ - check you credentials. If running locally check that " "ActiveMQ is running and started by `python setup.py start`" ) from err # Add placeholder variables: # these are used to ensure runs are deleted even if test fails before completion self.instrument = 'ARMI' self.rb_number = 1234567 self.run_number = 101 # Create test archive and add data self.data_archive = DataArchive([self.instrument], 19, 19) self.data_archive.create() # Create and send json message to ActiveMQ self.data_ready_message = Message(rb_number=self.rb_number, instrument=self.instrument, run_number=self.run_number, description="This is a system test", facility="ISIS", started_by=0) def tearDown(self): """ Disconnect from services, stop external services and delete data archive """ self.queue_client.disconnect() self.database_client.disconnect() self._remove_run_from_database(self.instrument, self.run_number) self.data_archive.delete() self._delete_reduction_directory() def _setup_data_structures(self, reduce_script, vars_script): """ Sets up a fake archive and reduced data save location on the system :param reduce_script: The content to use in the reduce.py file :param vars_script: The content to use in the reduce_vars.py file :return: file_path to the reduced data """ raw_file = '{}{}.nxs'.format(self.instrument, self.run_number) self.data_archive.add_reduction_script(self.instrument, reduce_script) self.data_archive.add_reduce_vars_script(self.instrument, vars_script) raw_file = self.data_archive.add_data_file(self.instrument, raw_file, 19, 1) return raw_file def _find_run_in_database(self): """ Find a ReductionRun record in the database This includes a timeout to wait for several seconds to ensure the database has received the record in question :return: The resulting record """ instrument = db.get_instrument(self.instrument) return instrument.reduction_runs.filter(run_number=self.run_number) @staticmethod def _remove_run_from_database(instrument, run_number): """ Uses the scripts.manual_operations.manual_remove script to remove records added to the database """ if not isinstance(run_number, list): run_number = [run_number] for run in run_number: remove.remove(instrument, run, delete_all_versions=True) @staticmethod def _delete_reduction_directory(): """ Delete the temporary reduction directory""" path = Path(os.path.join(PROJECT_ROOT, 'reduced-data')) if path.exists(): shutil.rmtree(path.absolute()) def send_and_wait_for_result(self, message): """Sends the message to the queue and waits until the listener has finished processing it""" # forces the is_processing to return True so that the listener has time to actually start processing the message self.listener._processing = True #pylint:disable=protected-access self.queue_client.send('/queue/DataReady', message) while self.listener.is_processing_message(): time.sleep(0.5) # Get Result from database results = self._find_run_in_database() assert results return results def test_end_to_end_wish_invalid_rb_number_skipped(self): """ Test that data gets skipped when the RB Number doesn't validate """ # Set meta data for test self.rb_number = 222 self.data_ready_message.rb_number = self.rb_number # Create supporting data structures e.g. Data Archive, Reduce directory file_location = self._setup_data_structures( reduce_script=REDUCE_SCRIPT, vars_script='') self.data_ready_message.data = file_location results = self.send_and_wait_for_result(self.data_ready_message) # Validate self.assertEqual(self.instrument, results[0].instrument.name) self.assertEqual(self.rb_number, results[0].experiment.reference_number) self.assertEqual(self.run_number, results[0].run_number) self.assertEqual("This is a system test", results[0].run_name) self.assertEqual('Skipped', results[0].status.value_verbose()) def test_end_to_end_wish_completed(self): """ Test that runs gets completed when everything is OK """ # Create supporting data structures e.g. Data Archive, Reduce directory file_location = self._setup_data_structures( reduce_script=REDUCE_SCRIPT, vars_script='') self.data_ready_message.data = file_location results = self.send_and_wait_for_result(self.data_ready_message) # Validate self.assertEqual(self.instrument, results[0].instrument.name) self.assertEqual(self.rb_number, results[0].experiment.reference_number) self.assertEqual(self.run_number, results[0].run_number) self.assertEqual("This is a system test", results[0].run_name) self.assertEqual('Completed', results[0].status.value_verbose()) def test_end_to_end_wish_bad_script_syntax_error(self): """ Test that run gets marked as error when the script has a syntax error """ # Create supporting data structures e.g. Data Archive, Reduce directory file_location = self._setup_data_structures( reduce_script=SYNTAX_ERROR_REDUCE_SCRIPT, vars_script='') self.data_ready_message.data = file_location results = self.send_and_wait_for_result(self.data_ready_message) # Validate self.assertEqual(self.instrument, results[0].instrument.name) self.assertEqual(self.rb_number, results[0].experiment.reference_number) self.assertEqual(self.run_number, results[0].run_number) self.assertEqual("This is a system test", results[0].run_name) self.assertEqual('Error', results[0].status.value_verbose()) self.assertIn("REDUCTION Error", results[0].message) self.assertIn("Error encountered when running the reduction script", results[0].message) self.assertIn("SyntaxError('EOL while scanning string literal'", results[0].message) def test_end_to_end_wish_bad_script_raises_exception(self): """ Test that WISH data goes through the system without issue """ # Create supporting data structures e.g. Data Archive, Reduce directory file_location = self._setup_data_structures( reduce_script="raise ValueError('hello from the other side')", vars_script='') self.data_ready_message.data = file_location results = self.send_and_wait_for_result(self.data_ready_message) # Validate self.assertEqual(self.instrument, results[0].instrument.name) self.assertEqual(self.rb_number, results[0].experiment.reference_number) self.assertEqual(self.run_number, results[0].run_number) self.assertEqual("This is a system test", results[0].run_name) self.assertEqual('Error', results[0].status.value_verbose()) self.assertIn('ValueError', results[0].message) self.assertIn('hello from the other side', results[0].message) def test_end_to_end_wish_vars_script_gets_new_variable(self): """Test running the same run twice, but the second time the reduce_vars has a new variable""" # Create supporting data structures e.g. Data Archive, Reduce directory file_location = self._setup_data_structures( reduce_script=REDUCE_SCRIPT, vars_script='') self.data_ready_message.data = file_location result_one = self.send_and_wait_for_result(self.data_ready_message) assert len(result_one) == 1 run_without_vars = result_one[0] self.data_archive.add_reduce_vars_script(self.instrument, VARS_SCRIPT) result_two = self.send_and_wait_for_result(self.data_ready_message) assert len(result_two) == 2 assert run_without_vars == result_two[ 0] # check that the first run is queried again run_with_vars = result_two[1] assert run_without_vars.run_variables.count() == 0 assert run_with_vars.run_variables.count( ) == 1 # the one standard variable in the VARS_SCRIPT var = run_with_vars.run_variables.first().variable assert var.name == "variable1" assert var.value == "value1" def test_end_to_end_wish_vars_script_loses_variable(self): """Test running the same run twice, but the second time the reduce_vars has one less variable""" # Create supporting data structures e.g. Data Archive, Reduce directory file_location = self._setup_data_structures( reduce_script=REDUCE_SCRIPT, vars_script=VARS_SCRIPT) self.data_ready_message.data = file_location result_one = self.send_and_wait_for_result(self.data_ready_message) assert len(result_one) == 1 run_with_vars = result_one[0] assert run_with_vars.run_variables.count( ) == 1 # the one standard variable in the VARS_SCRIPT var = run_with_vars.run_variables.first().variable assert var.name == "variable1" assert var.value == "value1" self.data_archive.add_reduce_vars_script(self.instrument, "") result_two = self.send_and_wait_for_result(self.data_ready_message) assert len(result_two) == 2 assert run_with_vars == result_two[0] run_without_vars = result_two[1] assert run_without_vars.run_variables.count() == 0 def test_end_to_end_vars_script_has_variable_value_changed(self): """Test that reducing the same run after changing the reduce_vars updates the variable's value""" # Create supporting data structures e.g. Data Archive, Reduce directory file_location = self._setup_data_structures( reduce_script=REDUCE_SCRIPT, vars_script=VARS_SCRIPT) self.data_ready_message.data = file_location result_one = self.send_and_wait_for_result(self.data_ready_message) assert len(result_one) == 1 run_with_initial_var = result_one[0] assert run_with_initial_var.run_variables.count( ) == 1 # the one standard variable in the VARS_SCRIPT var = run_with_initial_var.run_variables.first().variable assert var.name == "variable1" assert var.value == "value1" self.data_archive.add_reduce_vars_script( self.instrument, 'standard_vars={"variable1": 123}') result_two = self.send_and_wait_for_result(self.data_ready_message) assert len(result_two) == 2 assert run_with_initial_var == result_two[0] run_with_changed_var = result_two[1] assert run_with_initial_var.run_variables.count() == 1 assert run_with_changed_var.run_variables.count() == 1 initial_var = run_with_initial_var.run_variables.first().variable changed_var = run_with_changed_var.run_variables.first().variable assert initial_var == changed_var def test_end_to_end_wish_vars_script_has_variable_reused_on_new_run_number( self): """Test that the variables are reused on new run numbers, IF their value has not changed""" # Create supporting data structures e.g. Data Archive, Reduce directory file_location = self._setup_data_structures( reduce_script=REDUCE_SCRIPT, vars_script=VARS_SCRIPT) self.data_ready_message.data = file_location result_one = self.send_and_wait_for_result(self.data_ready_message) run_with_initial_var = result_one[0] self.data_ready_message.run_number = 1234568 result_two = self.send_and_wait_for_result(self.data_ready_message) run_with_different_run_number = result_two[0] assert run_with_initial_var.run_variables.count() == 1 assert run_with_different_run_number.run_variables.count() == 1 initial_var = run_with_initial_var.run_variables.first().variable new_var = run_with_different_run_number.run_variables.first().variable assert initial_var == new_var def test_end_to_end_wish_vars_script_has_variable_copied_on_new_run_number_when_value_changed( self): """Test that the variable is copied for a new run WHEN it's value has been changed""" # Create supporting data structures e.g. Data Archive, Reduce directory file_location = self._setup_data_structures( reduce_script=REDUCE_SCRIPT, vars_script=VARS_SCRIPT) self.run_number = 101 self.data_ready_message.data = file_location result_one = self.send_and_wait_for_result(self.data_ready_message) assert len(result_one) == 1 run_with_initial_var = result_one[0] assert run_with_initial_var.run_variables.count( ) == 1 # the one standard variable in the VARS_SCRIPT var = run_with_initial_var.run_variables.first().variable assert var.name == "variable1" assert var.value == "value1" # update the run number in the class because it's used to query for the correct run self.data_ready_message.run_number = self.run_number = 102 self.data_archive.add_reduce_vars_script( self.instrument, 'standard_vars={"variable1": 123}') result_two = self.send_and_wait_for_result(self.data_ready_message) # making the run_number a list so that they can be deleted by the tearDown! self.run_number = [101, 102] assert len(result_two) == 1 run_with_changed_var = result_two[0] assert run_with_initial_var.run_variables.count() == 1 assert run_with_changed_var.run_variables.count() == 1 initial_var = run_with_initial_var.run_variables.first().variable changed_var = run_with_changed_var.run_variables.first().variable assert initial_var != changed_var assert initial_var.name == changed_var.name assert initial_var.value != changed_var.value assert initial_var.type != changed_var.type assert initial_var.instrumentvariable.start_run < changed_var.instrumentvariable.start_run
class ManualRemove: """ Handles removing a run from the database """ def __init__(self, instrument): """ :param instrument: (str) The name of the instrument associated with runs """ self.database = DatabaseClient() self.database.connect() self.to_delete = {} self.instrument = instrument def find_run_versions_in_database(self, run_number): """ Find all run versions in the database that relate to a given instrument and run number :param run_number: (int) The run to search for in the database :return: (QuerySet) The result of the query """ instrument_record = db.get_instrument(self.instrument) result = self.database.data_model.ReductionRun.objects \ .filter(instrument=instrument_record.id) \ .filter(run_number=run_number) \ .order_by('-created') self.to_delete[run_number] = list(result) return result def process_results(self, delete_all_versions: bool): """ Process all the results what to do with the run based on the result of database query """ copy_to_delete = self.to_delete.copy() for key, value in copy_to_delete.items(): if not value: self.run_not_found(run_number=key) if len(value) == 1: continue if len(value) > 1 and not delete_all_versions: self.multiple_versions_found(run_number=key) def run_not_found(self, run_number): """ Inform user and remove key from dictionary :param run_number: (int) The run to remove from the dictionary """ print('No runs found associated with {} for instrument {}'.format(run_number, self.instrument)) del self.to_delete[run_number] def multiple_versions_found(self, run_number): """ Ask the user which versions they want to remove Update the self.to_delete dictionary by removing unwanted versions :param run_number: (int) The run number with multiple versions """ # Display run_number - title - version for all matching runs print("Discovered multiple reduction versions for {}{}:".format(self.instrument, run_number)) for run in self.to_delete[run_number]: print("\tv{} - {}".format(run.run_version, run.run_name)) # Get user input for which versions they wish to delete user_input = input("Which runs would you like to delete (e.g. 0,1,2,3 or 0-3): ") input_valid, user_input = self.validate_csv_input(user_input) while input_valid is False: user_input = input('Input was invalid. ' 'Please provide a comma separated list or a range of values: ') input_valid, user_input = self.validate_csv_input(user_input) # Remove runs that the user does NOT want to delete from the delete list self.to_delete[run_number] = [ reduction_job for reduction_job in self.to_delete[run_number] if reduction_job.run_version in user_input ] def delete_records(self): """ Delete all records from the database that match those found in self.to_delete """ # Make a copy to ensure dict being iterated stays same size through processing to_delete_copy = self.to_delete.copy() for run_number, job_list in to_delete_copy.items(): for version in job_list: print(f'Deleting {self.instrument}{run_number} - v{version.run_version}') try: version.delete() except IntegrityError as err: print(f"Encountered integrity error: {err}\n\n" "Reverting to old behaviour - manual deletion. This can take much longer.") # For some reason some entries can throw an integrity error. # In that case we revert to the previous (much slower) way of manually # deleting everything. Perhaps there is a badly configured relation # but I am not sure why it works on _most_ self.delete_reduction_location(version.id) self.delete_data_location(version.id) self.delete_variables(version.id) self.delete_reduction_run(version.id) def delete_reduction_location(self, reduction_run_id): """ Delete a ReductionLocation record from the database :param reduction_run_id: (int) The id of the associated reduction job """ self.database.data_model.ReductionLocation.objects \ .filter(reduction_run_id=reduction_run_id) \ .delete() def delete_data_location(self, reduction_run_id): """ Delete a DataLocation record from the database :param reduction_run_id: (int) The id of the associated reduction job """ self.database.data_model.DataLocation.objects \ .filter(reduction_run_id=reduction_run_id) \ .delete() def delete_variables(self, reduction_run_id): """ Removes all the RunVariable records associated with a given ReductionRun from the database :param reduction_run_id: (int) The id of the associated reduction job """ run_variables = self.find_variables_of_reduction(reduction_run_id) for record in run_variables: self.database.variable_model.RunVariable.objects \ .filter(variable_ptr_id=record.variable_ptr_id) \ .delete() def find_variables_of_reduction(self, reduction_run_id): """ Find all the RunVariable records in the database associated with a reduction job :param reduction_run_id: (int) The id of the reduction job to filter by :return: (QuerySet) of the associated RunVariables """ return self.database.variable_model.RunVariable.objects \ .filter(reduction_run_id=reduction_run_id) def delete_reduction_run(self, reduction_run_id): """ Delete a ReductionRun record from the database :param reduction_run_id: (int) The id of the associated reduction job """ self.database.data_model.ReductionRun.objects \ .filter(id=reduction_run_id) \ .delete() @staticmethod def validate_csv_input(user_input): """ checks if a comma separated list was provided :return: (tuple) = (bool - is valid? , list - csv as list (empty list if invalid)) """ processed_input = [] if ',' in user_input: versions_to_delete = user_input.split(',') for number in versions_to_delete: try: number = int(number) processed_input.append(number) except ValueError: return False, [] elif "-" in user_input: range_of_versions_to_delete = user_input.split('-') if len(range_of_versions_to_delete) != 2: return False, [] sorted_range_of_versions = sorted(map(int, range_of_versions_to_delete)) smaller_version = int(sorted_range_of_versions[0]) larger_version = int(sorted_range_of_versions[1]) return True, list(range(smaller_version, larger_version + 1)) else: try: user_input = int(user_input) processed_input.append(user_input) except ValueError: return False, [] return True, processed_input
class TestEndToEnd(unittest.TestCase): """ Class to test pipelines in autoreduction""" @classmethod def setUpClass(cls): # Start all services external.start_queue_processors() def setUp(self): """ Start all external services """ # Get all clients self.database_client = DatabaseClient() self.database_client.connect() self.queue_client = QueueClient(ACTIVEMQ_SETTINGS) self.queue_client.connect() # Create test archive and add data self.data_archive_creator = DataArchiveCreator(os.path.join( get_project_root()), overwrite=True) self.archive_explorer = ArchiveExplorer( os.path.join(get_project_root(), 'data-archive')) # Add placeholder variables: # these are used to ensure runs are deleted even if test fails before completion self.instrument = None self.rb_number = None self.run_number = None def test_end_to_end_wish(self): """ Test that WISH data goes through the system without issue """ # Set meta data for test self.instrument = 'WISH' self.rb_number = 222 self.run_number = 101 reduce_script = \ 'def main(input_file, output_dir):\n' \ '\tprint("WISH system test")\n' \ '\n' \ 'if __name__ == "__main__":\n' \ '\tmain()\n' # Create supporting data structures e.g. Data Archive, Reduce directory file_location = self._setup_data_structures( reduce_script=reduce_script, vars_script='') # Create and send json message to ActiveMQ data_ready_message = Message(rb_number=self.rb_number, instrument=self.instrument, data=file_location, run_number=self.run_number, facility="ISIS", started_by=0) self.queue_client.send('/queue/DataReady', data_ready_message) # Get Result from database results = self._find_run_in_database() # Validate self.assertEqual(self.instrument, results[0].instrument.name) self.assertEqual(self.rb_number, results[0].experiment.reference_number) self.assertEqual(self.run_number, results[0].run_number) self.assertEqual('Completed', results[0].status.value_verbose()) def test_wish_user_script_failure(self): """ Test that WISH data goes through the system without issue """ # Set meta data for test self.instrument = 'WISH' self.rb_number = 222 self.run_number = 101 # Create supporting data structures e.g. Data Archive, Reduce directory file_location = self._setup_data_structures(reduce_script='fail', vars_script='') # Create and send json message to ActiveMQ data_ready_message = Message(rb_number=self.rb_number, instrument=self.instrument, data=file_location, run_number=self.run_number, facility="ISIS", started_by=0) self.queue_client.send('/queue/DataReady', data_ready_message) # Get Result from database results = self._find_run_in_database() # Validate self.assertEqual(self.instrument, results[0].instrument.name) self.assertEqual(self.rb_number, results[0].experiment.reference_number) self.assertEqual(self.run_number, results[0].run_number) self.assertEqual( 'e', results[0].status.value) # verbose value = "Error" def _setup_data_structures(self, reduce_script, vars_script): """ Sets up a fake archive and reduced data save location on the system :param reduce_script: The content to use in the reduce.py file :param vars_script: The content to use in the reduce_vars.py file :return: file_path to the reduced data """ raw_file = '{}{}.nxs'.format(self.instrument, self.run_number) # Create and add data to archive self.data_archive_creator.make_data_archive([self.instrument], 19, 19, 1) self.data_archive_creator.add_reduce_script( instrument=self.instrument, file_content=reduce_script) self.data_archive_creator.add_reduce_vars_script( self.instrument, vars_script) self.data_archive_creator.add_data_to_most_recent_cycle( self.instrument, raw_file) # Make temporary location to add reduced files to self._make_reduction_directory(self.instrument, self.rb_number, self.run_number) # Submit message to activemq cycle_path = self.archive_explorer.get_cycle_directory( self.instrument, 19, 1) return os.path.join(cycle_path, raw_file) @staticmethod def _make_reduction_directory(instrument, rb_number, run_number): """ Make a directory in the expected location for reduced runs to be written to """ reduced_dir = os.path.join(get_project_root(), 'reduced-data') reduced_inst = os.path.join(reduced_dir, str(instrument)) reduced_rb = os.path.join(reduced_inst, 'RB{}'.format(str(rb_number))) reduced_auto = os.path.join(reduced_rb, 'autoreduced') reduced_run = os.path.join(reduced_auto, str(run_number)) os.mkdir(reduced_dir) os.mkdir(reduced_inst) os.mkdir(reduced_rb) os.mkdir(reduced_auto) os.mkdir(reduced_run) def _find_run_in_database(self): """ Find a ReductionRun record in the database This includes a timeout to wait for several seconds to ensure the database has received the record in question :return: The resulting record """ wait_times = [0, 1, 2, 3, 5] results = [] for timeout in wait_times: # Wait before attempting database access print(f"Waiting for: {timeout}") time.sleep(timeout) # Check database has expected values instrument_record = db.get_instrument(self.instrument) results = self.database_client.data_model.ReductionRun.objects \ .filter(instrument=instrument_record.id) \ .filter(run_number=self.run_number) \ .select_related() \ .all() try: actual = results[0] except IndexError: # If no results found yet then continue continue # verbose values = "Completed" or "Error" if actual.status.value == 'c' or actual.status.value == 'e': print( f"Job reached {actual.status.value} status after {timeout} seconds" ) break return results @staticmethod def _remove_run_from_database(instrument, run_number): """ Uses the scripts.manual_operations.manual_remove script to remove records added to the database """ remove.remove(instrument, run_number) @staticmethod def _delete_reduction_directory(): """ Delete the temporary reduction directory""" shutil.rmtree(os.path.join(get_project_root(), 'reduced-data')) def tearDown(self): """ Disconnect from services, stop external services and delete data archive """ self.queue_client.disconnect() self.database_client.disconnect() self._delete_reduction_directory() del self.data_archive_creator # Done in tearDown to ensure run is removed even if test fails early self._remove_run_from_database(self.instrument, self.run_number) @classmethod def tearDownClass(cls): # Stop external services external.stop_queue_processors()