def test_terminate_manager(self, mocked_init, mocked_Logger,
mocked_Profiler):
rmq_params = mock.MagicMock(spec=ConnectionParameters)
rmgr = mock.MagicMock(spec=Base_ResourceManager)
tmgr = Tmgr('test_tmgr', ['pending_queues'], ['completed_queues'],
rmgr, rmq_params, 'test_rts')
tmgr._log = mocked_Logger
tmgr._prof = mocked_Profiler
tmgr._uid = 'tmgr.0000'
tmgr._tmgr_terminate = mp.Event()
tmgr._tmgr_process = mp.Process(target=_tmgr_side_effect,
name='test_tmgr',
args=(tmgr._tmgr_terminate, ))
tmgr._tmgr_process.start()
pid = tmgr._tmgr_process.pid
tmgr.check_manager = mock.MagicMock(return_value=True)
tmgr.terminate_manager()
self.assertIsNone(tmgr._tmgr_process)
self.assertFalse(psutil.pid_exists(pid))
def test_start_heartbeat(self, mocked_init, mocked_Logger,
mocked_Profiler):
rmq_params = mock.MagicMock(spec=ConnectionParameters)
rmgr = mock.MagicMock(spec=Base_ResourceManager)
tmgr = Tmgr('test_tmgr', ['pending_queues'], ['completed_queues'],
rmgr, rmq_params, 'test_rts')
global_boolean = False
def _heartbeat_side_effect():
nonlocal global_boolean
global_boolean = True
tmgr._log = mocked_Logger
tmgr._prof = mocked_Profiler
tmgr._heartbeat = mock.MagicMock(side_effect=_heartbeat_side_effect)
tmgr._uid = 'tmgr.0000'
tmgr._hb_terminate = None
tmgr._hb_thread = None
tmgr.start_heartbeat()
try:
self.assertTrue(global_boolean)
self.assertIsInstance(tmgr._hb_terminate, mt.Event)
self.assertIsInstance(tmgr._hb_thread, mt.Thread)
finally:
if tmgr._hb_thread.is_alive():
tmgr._hb_thread.join()
def test_advance(self, mocked_init, mocked_Logger, mocked_Profiler,
mocked_BlockingConnection):
mocked_BlockingConnection.channel = mock.MagicMock(
spec=pika.BlockingConnection.channel)
mocked_BlockingConnection.close = mock.MagicMock(return_value=None)
mocked_BlockingConnection.channel.queue_delete = mock.MagicMock(
return_value=None)
mocked_BlockingConnection.channel.queue_declare = mock.MagicMock(
return_value=None)
rmq_params = mock.MagicMock(spec=ConnectionParameters)
rmgr = mock.MagicMock(spec=Base_ResourceManager)
tmgr = Tmgr('test_tmgr', ['pending_queues'], ['completed_queues'],
rmgr, rmq_params, 'test_rts')
global_syncs = []
def _sync_side_effect(obj, obj_type, channel, conn_params, queue):
nonlocal global_syncs
global_syncs.append([obj, obj_type, channel, conn_params, queue])
tmgr._log = mocked_Logger
tmgr._prof = mocked_Profiler
tmgr._sync_with_master = mock.MagicMock(side_effect=_sync_side_effect)
tmgr._uid = 'tmgr.0000'
obj = mock.Mock()
obj.parent_stage = {'uid': 'test_stage'}
obj.parent_pipeline = {'uid': 'test_pipe'}
obj.uid = 'test_object'
obj.state = 'test_state'
tmgr._advance(obj, 'Task', None, 'channel', 'params', 'queue')
self.assertEqual(global_syncs[0],
[obj, 'Task', 'channel', 'params', 'queue'])
self.assertIsNone(obj.state)
global_syncs = []
tmgr._advance(obj, 'Stage', 'new_state', 'channel', 'params', 'queue')
self.assertEqual(global_syncs[0],
[obj, 'Stage', 'channel', 'params', 'queue'])
self.assertEqual(obj.state, 'new_state')
def test_terminate_heartbeat(self, mocked_init, mocked_Logger,
mocked_Profiler, mocked_BlockingConnection):
mocked_BlockingConnection.channel = mock.MagicMock(
spec=pika.BlockingConnection.channel)
mocked_BlockingConnection.close = mock.MagicMock(return_value=None)
mocked_BlockingConnection.channel.queue_delete = mock.MagicMock(
return_value=None)
mocked_BlockingConnection.channel.queue_declare = mock.MagicMock(
return_value=None)
mocked_BlockingConnection.close = mock.MagicMock(return_value=None)
rmq_params = mock.MagicMock(spec=ConnectionParameters)
rmgr = mock.MagicMock(spec=Base_ResourceManager)
tmgr = Tmgr('test_tmgr', ['pending_queues'], ['completed_queues'],
rmgr, rmq_params, 'test_rts')
tmgr._rmq_conn_params = rmq_params
tmgr._hb_request_q = 'test_tmgr-hb-request'
tmgr._hb_response_q = 'test_tmgr-hb-response'
tmgr._log = mocked_Logger
tmgr._prof = mocked_Profiler
tmgr._uid = 'tmgr.0000'
tmgr.check_heartbeat = mock.MagicMock(return_value=True)
tmgr.check_manager = mock.MagicMock(return_value=True)
tmgr._hb_terminate = mt.Event()
tmgr._hb_thread = mt.Thread(target=_tmgr_side_effect,
name='test_tmgr',
args=(tmgr._hb_terminate, ))
tmgr._hb_thread.start()
tmgr.terminate_heartbeat()
self.assertIsNone(tmgr._hb_thread)
def test_heartbeat(self, mocked_init, mocked_Profiler):
hostname = os.environ.get('RMQ_HOSTNAME', 'localhost')
port = int(os.environ.get('RMQ_PORT', '5672'))
username = os.environ.get('RMQ_USERNAME', 'guest')
password = os.environ.get('RMQ_PASSWORD', 'guest')
credentials = pika.PlainCredentials(username, password)
rmq_conn_params = pika.ConnectionParameters(host=hostname,
port=port,
credentials=credentials)
tmgr = BaseTmgr(None, None, None, None, None, None)
tmgr._uid = 'tmgr.0000'
tmgr._log = ru.Logger('radical.entk.manager.base', level='DEBUG')
tmgr._prof = mocked_Profiler
tmgr._hb_interval = 0.1
tmgr._hb_terminate = mt.Event()
tmgr._hb_request_q = 'tmgr-hb-request'
tmgr._hb_response_q = 'tmgr-hb-response'
tmgr._rmq_conn_params = rmq_conn_params
mq_connection = pika.BlockingConnection(rmq_conn_params)
mq_channel = mq_connection.channel()
mq_channel.queue_declare(queue='tmgr-hb-request')
mq_channel.queue_declare(queue='tmgr-hb-response')
tmgr._log.info('Starting test')
master_thread = mt.Thread(target=tmgr._heartbeat,
name='tmgr_heartbeat')
master_thread.start()
time.sleep(0.1)
body = None
try:
for _ in range(5):
while body is None:
_, props, body = mq_channel.basic_get(
queue='tmgr-hb-request')
self.assertEqual(body, b'request')
nprops = pika.BasicProperties(
correlation_id=props.correlation_id)
mq_channel.basic_publish(exchange='',
routing_key='tmgr-hb-response',
properties=nprops,
body='response')
self.assertTrue(master_thread.is_alive())
body = None
time.sleep(0.5)
self.assertFalse(master_thread.is_alive())
master_thread.join()
mq_channel.queue_delete(queue='tmgr-hb-request')
mq_channel.queue_delete(queue='tmgr-hb-response')
mq_channel.queue_declare(queue='tmgr-hb-request')
mq_channel.queue_declare(queue='tmgr-hb-response')
master_thread = mt.Thread(target=tmgr._heartbeat,
name='tmgr_heartbeat')
master_thread.start()
body = None
while body is None:
_, props, body = mq_channel.basic_get(queue='tmgr-hb-request')
mq_channel.basic_publish(exchange='',
routing_key='tmgr-hb-response',
body='response')
time.sleep(0.2)
self.assertFalse(master_thread.is_alive())
except Exception as ex:
tmgr._hb_terminate.set()
master_thread.join()
mq_channel.queue_delete(queue='tmgr-hb-request')
mq_channel.queue_delete(queue='tmgr-hb-response')
mq_channel.close()
mq_connection.close()
raise ex
else:
tmgr._hb_terminate.set()
master_thread.join()
mq_channel.queue_delete(queue='tmgr-hb-request')
mq_channel.queue_delete(queue='tmgr-hb-response')
mq_channel.close()
mq_connection.close()