def send(message): """ Sends message to ReductionPending (with the specified delay) """ message_client = QueueClient() message_client.connect() message_client.send('/queue/DataReady', message, priority='1') message_client.disconnect()
def test_invalid_credentials(self): """ Test: A ConnectionException is raised When: _test_connection is called while invalid credentials are held """ client = QueueClient(self.incorrect_credentials) with self.assertRaises(ConnectionException): client.connect()
def test_invalid_connection_raises_on_test(self, _): """ Test: A ConnectionException is raised When: _test_connection is called while a valid connection is not held """ client = QueueClient() client.connect() self.assertRaises(ConnectionException, client._test_connection)
def test_valid_connection(self): """ Test: Access is established with a valid connection (This by proxy will also test the get_connection function) When: connect is called while valid credentials are held """ client = QueueClient() client.connect() self.assertTrue(client._test_connection())
def run(self): """ Connect to ActiveMQ via the QueueClient and listen to the /ReductionPending queue for messages. """ activemq_client = QueueClient() activemq_client.connect() activemq_client.subscribe_amq(consumer_name=self.consumer_name, listener=Listener())
def test_ack(self, mock_stomp_ack): """ Test: ack sends an ack frame using stomp.ack When: ack is called while a valid connection is held """ client = QueueClient() client.connect() client.ack("test", "subscription") mock_stomp_ack.assert_called_once_with('test', "subscription")
def test_subscribe_to_single_queue(self, mock_stomp_subscribe, mock_stomp_set_listener): """ Test: subscribe_queues handles a single queue (non-list) and calls stomp.subscribe_queues for it When: subscribe_queues is called a single queue passed as queue_list """ client = QueueClient() client.connect() client.subscribe_queues('single-queue', 'consumer', None) mock_stomp_set_listener.assert_called_once_with('consumer', None) mock_stomp_subscribe.assert_called_once()
def _send_pending_msg(message, delay=None): """ Sends data_dict to ReductionPending (with the specified delay) :param message: (Message) The message to send to the pending queue :param delay: (int) how long to delay the message sending - if at all (None) """ # To prevent circular dependencies message_client = QueueClient() message_client.connect() message_client.send('/queue/ReductionPending', message, priority='0', delay=delay) message_client.disconnect()
def login_queue(): """ Log into the QueueClient :return: The client connected, or raise exception """ print("Logging into ActiveMQ") activemq_client = QueueClient() try: activemq_client.connect() except (ConnectionException, ValueError) as exp: raise RuntimeError( "Unable to proceed. Unable to log in to ActiveMQ." "This is required to perform a manual submission") from exp return activemq_client
def _send_pending_msg(message, delay=None): """ Sends message to ReductionPending (with the specified delay) """ # To prevent circular dependencies MessagingUtils._add_project_root_to_path() from utils.clients.queue_client import QueueClient message_client = QueueClient() message_client.connect() message_client.send('/queue/ReductionPending', message, priority='0', delay=delay) message_client.disconnect()
def test_connection_failed_invalid_credentials(self): """ Test: A ConnectionException is raised When: _test_connection is called while invalid credentials are held """ incorrect_credentials = ClientSettingsFactory().create( 'queue', username='******', password='******', host='not-host', port='1234') client = QueueClient(incorrect_credentials) with self.assertRaises(ConnectionException): client.connect()
def login_queue(): """ Log into the QueueClient :return: The client connected, or raise exception """ print("Logging into ActiveMQ") activemq_client = QueueClient() try: activemq_client.connect() except (ConnectionException, ValueError) as exp: raise RuntimeError( "Cannot connect to ActiveMQ with provided credentials in credentials.ini\n" "Check that the ActiveMQ service is running, and the username, password and host are correct." ) from exp return activemq_client
def test_subscribe_to_single_queue(self, mock_stomp_subscribe, mock_stomp_set_listener): """ Test: subscribe_queues handles a single queue (non-list) and calls stomp.subscribe_queues for it When: subscribe_queues is called a single queue passed as queue_list """ client = QueueClient() client.connect() client.subscribe_queues('single-queue', 'consumer', None, 'auto') mock_stomp_set_listener.assert_called_once_with('consumer', None) test_expected_args = {'destination': 'single-queue', 'id': '1', 'ack': 'auto', 'header': {'activemq.prefetchSize': '1'}} mock_stomp_subscribe.assert_called_once_with(**test_expected_args)
def main(): """ Main method. """ queue_client = QueueClient() try: logger.info("PostProcessAdmin Connecting to ActiveMQ") queue_client.connect() logger.info("PostProcessAdmin Successfully Connected to ActiveMQ") destination, data = sys.argv[1:3] # pylint: disable=unbalanced-tuple-unpacking message = Message() message.populate(data) logger.info("destination: %s", destination) logger.info("message: %s", message.serialize(limit_reduction_script=True)) try: post_proc = PostProcessAdmin(message, queue_client) log_stream_handler = logging.StreamHandler( post_proc.admin_log_stream) logger.addHandler(log_stream_handler) if destination == '/queue/ReductionPending': post_proc.reduce() except ValueError as exp: message.message = str( exp) # Note: I believe this should be .message logger.info("Message data error: %s", message.serialize(limit_reduction_script=True)) raise except Exception as exp: logger.info("PostProcessAdmin error: %s", str(exp)) raise finally: try: logger.removeHandler(log_stream_handler) except: pass except Exception as exp: logger.info("Something went wrong: %s", str(exp)) try: queue_client.send(ACTIVEMQ_SETTINGS.reduction_error, message) logger.info("Called %s ---- %s", ACTIVEMQ_SETTINGS.reduction_error, message.serialize(limit_reduction_script=True)) finally: sys.exit()
def setup_connection(consumer_name): """ Starts the ActiveMQ connection and registers the event listener :return: (Listener) A listener instance which has subscribed to an ActiveMQ queue """ # Connect to ActiveMQ activemq_client = QueueClient() activemq_client.connect() # Register the event listener listener = QueueListener(activemq_client) # Subscribe to queues activemq_client.subscribe_autoreduce(consumer_name, listener) return activemq_client
def setup_connection(consumer_name) -> Tuple[QueueClient, QueueListener]: """ Starts the ActiveMQ connection and registers the event listener :return: A client connected and subscribed to the queue specified in credentials, and a listener instance which will handle incoming messages """ # Connect to ActiveMQ activemq_client = QueueClient() activemq_client.connect() # Register the event listener listener = QueueListener(activemq_client) # Subscribe to queues activemq_client.subscribe_autoreduce(consumer_name, listener) return activemq_client, listener
def test_subscribe_to_queue_list(self, mock_stomp_subscribe, mock_stomp_set_listener): """ Test: subscribe_queues calls stomp.subscribe_queues twice, once for each queue given When: subscribe_queues is called with a queue_list length of 2 """ client = QueueClient() client.connect() client.subscribe_queues(['test', 'queues'], 'consumer', None, 'auto') mock_stomp_set_listener.assert_called_once_with('consumer', None) test_expected_args = {'destination': 'test', 'id': '1', 'ack': 'auto', 'header': {'activemq.prefetchSize': '1'}} queue_expected_args = {'destination': 'queues', 'id': '1', 'ack': 'auto', 'header': {'activemq.prefetchSize': '1'}} mock_stomp_subscribe.assert_has_calls([call(**test_expected_args), call(**queue_expected_args)])
class TestEndToEnd(unittest.TestCase): """ Class to test pipelines in autoreduction""" @classmethod def setUpClass(cls): # Start all services external.start_queue_processors() def setUp(self): """ Start all external services """ # Get all clients self.database_client = DatabaseClient() self.database_client.connect() self.queue_client = QueueClient(ACTIVEMQ_SETTINGS) self.queue_client.connect() # Create test archive and add data self.data_archive_creator = DataArchiveCreator(os.path.join( get_project_root()), overwrite=True) self.archive_explorer = ArchiveExplorer( os.path.join(get_project_root(), 'data-archive')) # Add placeholder variables: # these are used to ensure runs are deleted even if test fails before completion self.instrument = None self.rb_number = None self.run_number = None def test_end_to_end_wish(self): """ Test that WISH data goes through the system without issue """ # Set meta data for test self.instrument = 'WISH' self.rb_number = 222 self.run_number = 101 reduce_script = \ 'def main(input_file, output_dir):\n' \ '\tprint("WISH system test")\n' \ '\n' \ 'if __name__ == "__main__":\n' \ '\tmain()\n' # Create supporting data structures e.g. Data Archive, Reduce directory file_location = self._setup_data_structures( reduce_script=reduce_script, vars_script='') # Create and send json message to ActiveMQ data_ready_message = Message(rb_number=self.rb_number, instrument=self.instrument, data=file_location, run_number=self.run_number, facility="ISIS", started_by=0) self.queue_client.send('/queue/DataReady', data_ready_message) # Get Result from database results = self._find_run_in_database() # Validate self.assertEqual(self.instrument, results[0].instrument.name) self.assertEqual(self.rb_number, results[0].experiment.reference_number) self.assertEqual(self.run_number, results[0].run_number) self.assertEqual('Completed', results[0].status.value_verbose()) def test_wish_user_script_failure(self): """ Test that WISH data goes through the system without issue """ # Set meta data for test self.instrument = 'WISH' self.rb_number = 222 self.run_number = 101 # Create supporting data structures e.g. Data Archive, Reduce directory file_location = self._setup_data_structures(reduce_script='fail', vars_script='') # Create and send json message to ActiveMQ data_ready_message = Message(rb_number=self.rb_number, instrument=self.instrument, data=file_location, run_number=self.run_number, facility="ISIS", started_by=0) self.queue_client.send('/queue/DataReady', data_ready_message) # Get Result from database results = self._find_run_in_database() # Validate self.assertEqual(self.instrument, results[0].instrument.name) self.assertEqual(self.rb_number, results[0].experiment.reference_number) self.assertEqual(self.run_number, results[0].run_number) self.assertEqual( 'e', results[0].status.value) # verbose value = "Error" def _setup_data_structures(self, reduce_script, vars_script): """ Sets up a fake archive and reduced data save location on the system :param reduce_script: The content to use in the reduce.py file :param vars_script: The content to use in the reduce_vars.py file :return: file_path to the reduced data """ raw_file = '{}{}.nxs'.format(self.instrument, self.run_number) # Create and add data to archive self.data_archive_creator.make_data_archive([self.instrument], 19, 19, 1) self.data_archive_creator.add_reduce_script( instrument=self.instrument, file_content=reduce_script) self.data_archive_creator.add_reduce_vars_script( self.instrument, vars_script) self.data_archive_creator.add_data_to_most_recent_cycle( self.instrument, raw_file) # Make temporary location to add reduced files to self._make_reduction_directory(self.instrument, self.rb_number, self.run_number) # Submit message to activemq cycle_path = self.archive_explorer.get_cycle_directory( self.instrument, 19, 1) return os.path.join(cycle_path, raw_file) @staticmethod def _make_reduction_directory(instrument, rb_number, run_number): """ Make a directory in the expected location for reduced runs to be written to """ reduced_dir = os.path.join(get_project_root(), 'reduced-data') reduced_inst = os.path.join(reduced_dir, str(instrument)) reduced_rb = os.path.join(reduced_inst, 'RB{}'.format(str(rb_number))) reduced_auto = os.path.join(reduced_rb, 'autoreduced') reduced_run = os.path.join(reduced_auto, str(run_number)) os.mkdir(reduced_dir) os.mkdir(reduced_inst) os.mkdir(reduced_rb) os.mkdir(reduced_auto) os.mkdir(reduced_run) def _find_run_in_database(self): """ Find a ReductionRun record in the database This includes a timeout to wait for several seconds to ensure the database has received the record in question :return: The resulting record """ wait_times = [0, 1, 2, 3, 5] results = [] for timeout in wait_times: # Wait before attempting database access print(f"Waiting for: {timeout}") time.sleep(timeout) # Check database has expected values instrument_record = db.get_instrument(self.instrument) results = self.database_client.data_model.ReductionRun.objects \ .filter(instrument=instrument_record.id) \ .filter(run_number=self.run_number) \ .select_related() \ .all() try: actual = results[0] except IndexError: # If no results found yet then continue continue # verbose values = "Completed" or "Error" if actual.status.value == 'c' or actual.status.value == 'e': print( f"Job reached {actual.status.value} status after {timeout} seconds" ) break return results @staticmethod def _remove_run_from_database(instrument, run_number): """ Uses the scripts.manual_operations.manual_remove script to remove records added to the database """ remove.remove(instrument, run_number) @staticmethod def _delete_reduction_directory(): """ Delete the temporary reduction directory""" shutil.rmtree(os.path.join(get_project_root(), 'reduced-data')) def tearDown(self): """ Disconnect from services, stop external services and delete data archive """ self.queue_client.disconnect() self.database_client.disconnect() self._delete_reduction_directory() del self.data_archive_creator # Done in tearDown to ensure run is removed even if test fails early self._remove_run_from_database(self.instrument, self.run_number) @classmethod def tearDownClass(cls): # Stop external services external.stop_queue_processors()