def test_invalid_credentials(self):
"""
Test: A ConnectionException is raised
When: _test_connection is called while invalid credentials are held
"""
client = QueueClient(self.incorrect_credentials)
with self.assertRaises(ConnectionException):
client.connect()
def test_invalid_connection_raises_on_test(self, _):
"""
Test: A ConnectionException is raised
When: _test_connection is called while a valid connection is not held
"""
client = QueueClient()
client.connect()
self.assertRaises(ConnectionException, client._test_connection)
def test_valid_connection(self):
"""
Test: Access is established with a valid connection
(This by proxy will also test the get_connection function)
When: connect is called while valid credentials are held
"""
client = QueueClient()
client.connect()
self.assertTrue(client._test_connection())
def test_send_with_raw_string(self, mock_stomp_send):
"""
Test: send sends the given data using stomp.send
When: send is called with a string argument for message
"""
client = QueueClient()
client.send('dataready', 'raw_json_dump')
(args, _) = mock_stomp_send.call_args
self.assertEqual(args[0], 'dataready')
self.assertEqual(args[1], 'raw_json_dump')
def test_send_with_message_instance(self, mock_stomp_send):
"""
Test: send sends the given data using stomp.send
When: send is called with a Message instance argument for message
"""
client = QueueClient()
message = Message(description="test-message")
client.send('dataready', message)
(args, _) = mock_stomp_send.call_args
self.assertEqual(args[0], 'dataready')
self.assertEqual(args[1], message.serialize())
def test_connection_failed_invalid_credentials(self):
"""
Test: A ConnectionException is raised
When: _test_connection is called while invalid credentials are held
"""
incorrect_credentials = ClientSettingsFactory().create(
'queue',
username='******',
password='******',
host='not-host',
port='1234')
client = QueueClient(incorrect_credentials)
with self.assertRaises(ConnectionException):
client.connect()
def login_queue():
"""
Log into the QueueClient
:return: The client connected, or raise exception
"""
print("Logging into ActiveMQ")
activemq_client = QueueClient()
try:
activemq_client.connect()
except (ConnectionException, ValueError) as exp:
raise RuntimeError(
"Unable to proceed. Unable to log in to ActiveMQ."
"This is required to perform a manual submission") from exp
return activemq_client
def send(message):
""" Sends message to ReductionPending (with the specified delay) """
message_client = QueueClient()
message_client.connect()
message_client.send('/queue/DataReady', message, priority='1')
message_client.disconnect()
def test_subscribe_to_pending(self, mock_subscribe):
"""
Test: subscribe_amq calls subscribe_queues with given arguments,
including a single queue (ReductionPending)
When: subscribe_amq is called once with the arguments given
"""
client = QueueClient()
client.subscribe_amq('consumer', None, 'auto')
# due to default params these have to be supplied to the mock in a dictionary
expected_args = {'queue_list': '/queue/ReductionPending',
'ack': 'auto',
'listener': None,
'consumer_name': 'consumer'}
mock_subscribe.assert_called_once_with(**expected_args)
def test_stop_connection(self):
"""
Test: Connection is stopped and connection variables are set to None
When: disconnect is called while a valid connection is currently established
"""
client = QueueClient()
mocked_connection = mock.Mock()
client._connection = mocked_connection
with mock.patch("uuid.uuid4") as patched_uuid:
patched_uuid.return_value = 1
client.disconnect()
mocked_connection.disconnect.assert_called_with(receipt=str(1))
self.assertIsNone(client._connection)
def login_queue():
"""
Log into the QueueClient
:return: The client connected, or raise exception
"""
print("Logging into ActiveMQ")
activemq_client = QueueClient()
try:
activemq_client.connect()
except (ConnectionException, ValueError) as exp:
raise RuntimeError(
"Cannot connect to ActiveMQ with provided credentials in credentials.ini\n"
"Check that the ActiveMQ service is running, and the username, password and host are correct."
) from exp
return activemq_client
def update_last_runs(csv_name):
"""
Read the last runs CSV file and bring it up to date with the
instrument lastrun.txt
:param csv_name: File name of the local last runs CSV file
"""
connection = QueueClient()
# Loop over instruments
output = []
with open(csv_name, 'r') as csv_file:
csv_reader = csv.reader(csv_file)
for row in csv_reader:
inst_mon = InstrumentMonitor(connection, row[0])
inst_mon.last_run_file = row[2]
inst_mon.summary_file = row[3]
inst_mon.data_dir = row[4]
inst_mon.file_ext = row[5]
try:
last_run = inst_mon.submit_run_difference(row[1])
row[1] = last_run
except InstrumentMonitorError as ex:
EORM_LOG.error(ex)
output.append(row)
# Write any changes to the CSV
with open(csv_name, 'w', newline='') as csv_file:
csv_writer = csv.writer(csv_file)
for row in output:
csv_writer.writerow(row)
def test_subscribe_to_all_queues(self, mock_subscribe):
"""
Test: subscribe_autoreduce calls subscribe_queues with given arguments,
including a list of multiple queues (all)
When: subscribe_autoreduce is called once with the arguments given
"""
client = QueueClient()
client.subscribe_autoreduce('consumer', None, 'auto')
expected_args = {'queue_list': ['/queue/DataReady',
'/queue/ReductionStarted',
'/queue/ReductionComplete',
'/queue/ReductionError',
'/queue/ReductionSkipped'],
'ack': 'auto',
'listener': None,
'consumer_name': 'consumer'}
mock_subscribe.assert_called_once_with(**expected_args)
def test_default_init(self):
"""
Test: Class variables are created and set
When: QueueClient is initialised with default credentials
"""
client = QueueClient()
self.assertIsNotNone(client.credentials)
self.assertIsNone(client._connection)
self.assertEqual('queue_client', client._consumer_name)
def setUp(self):
""" Start all external services """
# Get all clients
self.database_client = DatabaseClient()
self.database_client.connect()
self.queue_client = QueueClient(ACTIVEMQ_SETTINGS)
self.queue_client.connect()
# Create test archive and add data
self.data_archive_creator = DataArchiveCreator(os.path.join(
get_project_root()),
overwrite=True)
self.archive_explorer = ArchiveExplorer(
os.path.join(get_project_root(), 'data-archive'))
# Add placeholder variables:
# these are used to ensure runs are deleted even if test fails before completion
self.instrument = None
self.rb_number = None
self.run_number = None
def _send_pending_msg(message, delay=None):
""" Sends message to ReductionPending (with the specified delay) """
# To prevent circular dependencies
MessagingUtils._add_project_root_to_path()
from utils.clients.queue_client import QueueClient
message_client = QueueClient()
message_client.connect()
message_client.send('/queue/ReductionPending',
message,
priority='0',
delay=delay)
message_client.disconnect()
def _send_pending_msg(message, delay=None):
"""
Sends data_dict to ReductionPending (with the specified delay)
:param message: (Message) The message to send to the pending queue
:param delay: (int) how long to delay the message sending - if at all (None)
"""
# To prevent circular dependencies
message_client = QueueClient()
message_client.connect()
message_client.send('/queue/ReductionPending',
message,
priority='0',
delay=delay)
message_client.disconnect()
def run(self):
"""
Connect to ActiveMQ via the QueueClient and listen to the
/ReductionPending queue for messages.
"""
activemq_client = QueueClient()
activemq_client.connect()
activemq_client.subscribe_amq(consumer_name=self.consumer_name,
listener=Listener())
def test_ack(self, mock_stomp_ack):
"""
Test: ack sends an ack frame using stomp.ack
When: ack is called while a valid connection is held
"""
client = QueueClient()
client.connect()
client.ack("test", "subscription")
mock_stomp_ack.assert_called_once_with('test', "subscription")
def test_subscribe_to_single_queue(self, mock_stomp_subscribe,
mock_stomp_set_listener):
"""
Test: subscribe_queues handles a single queue (non-list)
and calls stomp.subscribe_queues for it
When: subscribe_queues is called a single queue passed as queue_list
"""
client = QueueClient()
client.connect()
client.subscribe_queues('single-queue', 'consumer', None)
mock_stomp_set_listener.assert_called_once_with('consumer', None)
mock_stomp_subscribe.assert_called_once()
def test_subscribe_to_single_queue(self, mock_stomp_subscribe, mock_stomp_set_listener):
"""
Test: subscribe_queues handles a single queue (non-list)
and calls stomp.subscribe_queues for it
When: subscribe_queues is called a single queue passed as queue_list
"""
client = QueueClient()
client.connect()
client.subscribe_queues('single-queue', 'consumer', None, 'auto')
mock_stomp_set_listener.assert_called_once_with('consumer', None)
test_expected_args = {'destination': 'single-queue',
'id': '1',
'ack': 'auto',
'header': {'activemq.prefetchSize': '1'}}
mock_stomp_subscribe.assert_called_once_with(**test_expected_args)
def main():
""" Main method. """
queue_client = QueueClient()
try:
logger.info("PostProcessAdmin Connecting to ActiveMQ")
queue_client.connect()
logger.info("PostProcessAdmin Successfully Connected to ActiveMQ")
destination, data = sys.argv[1:3] # pylint: disable=unbalanced-tuple-unpacking
message = Message()
message.populate(data)
logger.info("destination: %s", destination)
logger.info("message: %s",
message.serialize(limit_reduction_script=True))
try:
post_proc = PostProcessAdmin(message, queue_client)
log_stream_handler = logging.StreamHandler(
post_proc.admin_log_stream)
logger.addHandler(log_stream_handler)
if destination == '/queue/ReductionPending':
post_proc.reduce()
except ValueError as exp:
message.message = str(
exp) # Note: I believe this should be .message
logger.info("Message data error: %s",
message.serialize(limit_reduction_script=True))
raise
except Exception as exp:
logger.info("PostProcessAdmin error: %s", str(exp))
raise
finally:
try:
logger.removeHandler(log_stream_handler)
except:
pass
except Exception as exp:
logger.info("Something went wrong: %s", str(exp))
try:
queue_client.send(ACTIVEMQ_SETTINGS.reduction_error, message)
logger.info("Called %s ---- %s", ACTIVEMQ_SETTINGS.reduction_error,
message.serialize(limit_reduction_script=True))
finally:
sys.exit()
def setup_connection(consumer_name):
"""
Starts the ActiveMQ connection and registers the event listener
:return: (Listener) A listener instance which has subscribed to an
ActiveMQ queue
"""
# Connect to ActiveMQ
activemq_client = QueueClient()
activemq_client.connect()
# Register the event listener
listener = QueueListener(activemq_client)
# Subscribe to queues
activemq_client.subscribe_autoreduce(consumer_name, listener)
return activemq_client
def setup_connection(consumer_name) -> Tuple[QueueClient, QueueListener]:
"""
Starts the ActiveMQ connection and registers the event listener
:return: A client connected and subscribed to the queue specified in credentials, and
a listener instance which will handle incoming messages
"""
# Connect to ActiveMQ
activemq_client = QueueClient()
activemq_client.connect()
# Register the event listener
listener = QueueListener(activemq_client)
# Subscribe to queues
activemq_client.subscribe_autoreduce(consumer_name, listener)
return activemq_client, listener
def test_subscribe_to_queue_list(self, mock_stomp_subscribe, mock_stomp_set_listener):
"""
Test: subscribe_queues calls stomp.subscribe_queues twice, once for each queue given
When: subscribe_queues is called with a queue_list length of 2
"""
client = QueueClient()
client.connect()
client.subscribe_queues(['test', 'queues'], 'consumer', None, 'auto')
mock_stomp_set_listener.assert_called_once_with('consumer', None)
test_expected_args = {'destination': 'test',
'id': '1',
'ack': 'auto',
'header': {'activemq.prefetchSize': '1'}}
queue_expected_args = {'destination': 'queues',
'id': '1',
'ack': 'auto',
'header': {'activemq.prefetchSize': '1'}}
mock_stomp_subscribe.assert_has_calls([call(**test_expected_args),
call(**queue_expected_args)])
class TestEndToEnd(unittest.TestCase):
""" Class to test pipelines in autoreduction"""
@classmethod
def setUpClass(cls):
# Start all services
external.start_queue_processors()
def setUp(self):
""" Start all external services """
# Get all clients
self.database_client = DatabaseClient()
self.database_client.connect()
self.queue_client = QueueClient(ACTIVEMQ_SETTINGS)
self.queue_client.connect()
# Create test archive and add data
self.data_archive_creator = DataArchiveCreator(os.path.join(
get_project_root()),
overwrite=True)
self.archive_explorer = ArchiveExplorer(
os.path.join(get_project_root(), 'data-archive'))
# Add placeholder variables:
# these are used to ensure runs are deleted even if test fails before completion
self.instrument = None
self.rb_number = None
self.run_number = None
def test_end_to_end_wish(self):
"""
Test that WISH data goes through the system without issue
"""
# Set meta data for test
self.instrument = 'WISH'
self.rb_number = 222
self.run_number = 101
reduce_script = \
'def main(input_file, output_dir):\n' \
'\tprint("WISH system test")\n' \
'\n' \
'if __name__ == "__main__":\n' \
'\tmain()\n'
# Create supporting data structures e.g. Data Archive, Reduce directory
file_location = self._setup_data_structures(
reduce_script=reduce_script, vars_script='')
# Create and send json message to ActiveMQ
data_ready_message = Message(rb_number=self.rb_number,
instrument=self.instrument,
data=file_location,
run_number=self.run_number,
facility="ISIS",
started_by=0)
self.queue_client.send('/queue/DataReady', data_ready_message)
# Get Result from database
results = self._find_run_in_database()
# Validate
self.assertEqual(self.instrument, results[0].instrument.name)
self.assertEqual(self.rb_number,
results[0].experiment.reference_number)
self.assertEqual(self.run_number, results[0].run_number)
self.assertEqual('Completed', results[0].status.value_verbose())
def test_wish_user_script_failure(self):
"""
Test that WISH data goes through the system without issue
"""
# Set meta data for test
self.instrument = 'WISH'
self.rb_number = 222
self.run_number = 101
# Create supporting data structures e.g. Data Archive, Reduce directory
file_location = self._setup_data_structures(reduce_script='fail',
vars_script='')
# Create and send json message to ActiveMQ
data_ready_message = Message(rb_number=self.rb_number,
instrument=self.instrument,
data=file_location,
run_number=self.run_number,
facility="ISIS",
started_by=0)
self.queue_client.send('/queue/DataReady', data_ready_message)
# Get Result from database
results = self._find_run_in_database()
# Validate
self.assertEqual(self.instrument, results[0].instrument.name)
self.assertEqual(self.rb_number,
results[0].experiment.reference_number)
self.assertEqual(self.run_number, results[0].run_number)
self.assertEqual(
'e', results[0].status.value) # verbose value = "Error"
def _setup_data_structures(self, reduce_script, vars_script):
"""
Sets up a fake archive and reduced data save location on the system
:param reduce_script: The content to use in the reduce.py file
:param vars_script: The content to use in the reduce_vars.py file
:return: file_path to the reduced data
"""
raw_file = '{}{}.nxs'.format(self.instrument, self.run_number)
# Create and add data to archive
self.data_archive_creator.make_data_archive([self.instrument], 19,
19, 1)
self.data_archive_creator.add_reduce_script(
instrument=self.instrument, file_content=reduce_script)
self.data_archive_creator.add_reduce_vars_script(
self.instrument, vars_script)
self.data_archive_creator.add_data_to_most_recent_cycle(
self.instrument, raw_file)
# Make temporary location to add reduced files to
self._make_reduction_directory(self.instrument, self.rb_number,
self.run_number)
# Submit message to activemq
cycle_path = self.archive_explorer.get_cycle_directory(
self.instrument, 19, 1)
return os.path.join(cycle_path, raw_file)
@staticmethod
def _make_reduction_directory(instrument, rb_number, run_number):
"""
Make a directory in the expected location for reduced runs to be written to
"""
reduced_dir = os.path.join(get_project_root(), 'reduced-data')
reduced_inst = os.path.join(reduced_dir, str(instrument))
reduced_rb = os.path.join(reduced_inst,
'RB{}'.format(str(rb_number)))
reduced_auto = os.path.join(reduced_rb, 'autoreduced')
reduced_run = os.path.join(reduced_auto, str(run_number))
os.mkdir(reduced_dir)
os.mkdir(reduced_inst)
os.mkdir(reduced_rb)
os.mkdir(reduced_auto)
os.mkdir(reduced_run)
def _find_run_in_database(self):
"""
Find a ReductionRun record in the database
This includes a timeout to wait for several seconds to ensure the database has received
the record in question
:return: The resulting record
"""
wait_times = [0, 1, 2, 3, 5]
results = []
for timeout in wait_times:
# Wait before attempting database access
print(f"Waiting for: {timeout}")
time.sleep(timeout)
# Check database has expected values
instrument_record = db.get_instrument(self.instrument)
results = self.database_client.data_model.ReductionRun.objects \
.filter(instrument=instrument_record.id) \
.filter(run_number=self.run_number) \
.select_related() \
.all()
try:
actual = results[0]
except IndexError:
# If no results found yet then continue
continue
# verbose values = "Completed" or "Error"
if actual.status.value == 'c' or actual.status.value == 'e':
print(
f"Job reached {actual.status.value} status after {timeout} seconds"
)
break
return results
@staticmethod
def _remove_run_from_database(instrument, run_number):
"""
Uses the scripts.manual_operations.manual_remove script
to remove records added to the database
"""
remove.remove(instrument, run_number)
@staticmethod
def _delete_reduction_directory():
""" Delete the temporary reduction directory"""
shutil.rmtree(os.path.join(get_project_root(), 'reduced-data'))
def tearDown(self):
""" Disconnect from services, stop external services and delete data archive """
self.queue_client.disconnect()
self.database_client.disconnect()
self._delete_reduction_directory()
del self.data_archive_creator
# Done in tearDown to ensure run is removed even if test fails early
self._remove_run_from_database(self.instrument, self.run_number)
@classmethod
def tearDownClass(cls):
# Stop external services
external.stop_queue_processors()
