def test_update_task_insights(self):
"""
Test whether task insights are being updated correctly
"""
MockDatetimeNow.RETURN_VALUES = [datetime(1970, 1, 1, 0, 0, 1, 0),
datetime(1970, 1, 1, 0, 0, 2, 0)]
MockDatetimeNow.CURRENT_IDX = 0
insights = WorkerInsights(mp.get_context('fork'), n_jobs=2)
insights.max_task_duration_last_updated = datetime(1970, 1, 1, 0, 0, 0, 0)
# Shouldn't do anything when insights haven't been enabled
with self.subTest(insights_enabled=False), patch('mpire.insights.datetime', new=MockDatetimeNow):
for worker_id in range(2):
insights.init_worker(worker_id)
insights.update_task_insights()
self.assertIsNone(insights.max_task_duration)
self.assertIsNone(insights.max_task_args)
self.assertEqual(insights.max_task_duration_last_updated, datetime(1970, 1, 1, 0, 0, 0, 0))
insights.reset_insights(enable_insights=True)
insights.max_task_duration_last_updated = datetime(1970, 1, 1, 0, 0, 0, 0)
MockDatetimeNow.CURRENT_IDX = 0
with self.subTest(insights_enabled=True), patch('mpire.insights.datetime', new=MockDatetimeNow):
for worker_id, max_task_duration_list in [(0, [(5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9')]),
(1, [(0, '0'), (1, '1'), (2, '2'), (3, '3'), (4, '4')])]:
insights.init_worker(worker_id)
insights.max_task_duration_list = max_task_duration_list
insights.update_task_insights()
self.assertEqual(insights.max_task_duration_last_updated, datetime(1970, 1, 1, 0, 0, worker_id + 1, 0))
self.assertListEqual(list(insights.max_task_duration), [5, 6, 7, 8, 9, 0, 1, 2, 3, 4])
self.assertListEqual(list(insights.max_task_args), ['5', '6', '7', '8', '9', '0', '1', '2', '3', '4'])
def test_update_task_insights_once_in_a_while(self):
"""
Test whether the update_task_insights is triggered correctly. It should be called two times based on the
mocked datetime.now() values: for the last two entries the time difference compared to
max_task_duration_last_updated is bigger than two seconds.
"""
MockDatetimeNow.RETURN_VALUES = [datetime(1970, 1, 1, 0, 0, 0, 0),
datetime(1970, 1, 1, 0, 0, 2, 0),
datetime(1970, 1, 1, 0, 0, 3, 0),
datetime(1970, 1, 1, 0, 0, 7, 0),
datetime(1970, 1, 1, 0, 0, 8, 0)]
MockDatetimeNow.CURRENT_IDX = 0
insights = WorkerInsights(mp.get_context('fork'), n_jobs=5)
insights.max_task_duration_last_updated = datetime(1970, 1, 1, 0, 0, 1, 0)
# Shouldn't do anything when insights haven't been enabled
with self.subTest(insights_enabled=False), patch('mpire.insights.datetime', new=MockDatetimeNow), \
patch.object(insights, 'update_task_insights') as p:
for worker_id in range(5):
insights.init_worker(worker_id)
insights.update_task_insights_once_in_a_while()
self.assertEqual(p.call_count, 0)
insights.reset_insights(enable_insights=True)
insights.max_task_duration_last_updated = datetime(1970, 1, 1, 0, 0, 1, 0)
MockDatetimeNow.CURRENT_IDX = 0
with self.subTest(insights_enabled=True), patch('mpire.insights.datetime', new=MockDatetimeNow), \
patch.object(insights, 'update_task_insights') as p:
for worker_id in range(5):
insights.init_worker(worker_id)
insights.update_task_insights_once_in_a_while()
self.assertEqual(p.call_count, 2)
def test_update_start_up_time(self):
"""
Test that the start up time is correctly added to worker_start_up_time for the right index
"""
MockDatetimeNow.RETURN_VALUES = [datetime(1970, 1, 1, 0, 0, 0, 0),
datetime(1970, 1, 1, 0, 0, 2, 0),
datetime(1970, 1, 1, 0, 0, 3, 0),
datetime(1970, 1, 1, 0, 0, 7, 0),
datetime(1970, 1, 1, 0, 0, 8, 0)]
MockDatetimeNow.CURRENT_IDX = 0
insights = WorkerInsights(mp.get_context('fork'), n_jobs=5)
# Shouldn't do anything when insights haven't been enabled
with self.subTest(insights_enabled=False), patch('mpire.insights.datetime', new=MockDatetimeNow):
for worker_id in range(5):
insights.init_worker(worker_id)
insights.update_start_up_time(datetime(1970, 1, 1, 0, 0, 0, 0))
self.assertIsNone(insights.worker_start_up_time)
insights.reset_insights(enable_insights=True)
insights.max_task_duration_last_updated = datetime(1970, 1, 1, 0, 0, 0, 0)
MockDatetimeNow.CURRENT_IDX = 0
with self.subTest(insights_enabled=True), patch('mpire.insights.datetime', new=MockDatetimeNow):
for worker_id in range(5):
insights.init_worker(worker_id)
insights.update_start_up_time(datetime(1970, 1, 1, 0, 0, 0, 0))
self.assertListEqual(list(insights.worker_start_up_time), [0, 2, 3, 7, 8])
def test_init_worker(self):
"""
Test that the right containers are selected given a worker ID
"""
insights = WorkerInsights(mp.get_context('fork'), n_jobs=5)
# Should've only set worker ID when insights haven't been enabled
with self.subTest(insights_enabled=False):
insights.init_worker(3)
self.assertEqual(insights.worker_id, 3)
self.assertIsNone(insights.max_task_duration_list)
with self.subTest(insights_enabled=True):
insights.reset_insights(enable_insights=True)
insights.max_task_duration[:] = range(25)
insights.max_task_args[:] = map(str, range(25))
for worker_id, expected_task_duration_list in [
(0, [(0.0, '0'), (1.0, '1'), (2.0, '2'), (3.0, '3'), (4.0, '4')]),
(1, [(5.0, '5'), (6.0, '6'), (7.0, '7'), (8.0, '8'), (9.0, '9')]),
(4, [(20.0, '20'), (21.0, '21'), (22.0, '22'), (23.0, '23'), (24.0, '24')])
]:
with self.subTest(worker_id=worker_id):
insights.init_worker(worker_id)
self.assertEqual(insights.worker_id, worker_id)
self.assertListEqual(insights.max_task_duration_list, expected_task_duration_list)
def test_update_n_completed_tasks(self):
"""
Test that the number of completed tasks is correctly added to worker_n_completed_tasks for the right index
"""
insights = WorkerInsights(mp.get_context('fork'), n_jobs=5)
# Shouldn't do anything when insights haven't been enabled
with self.subTest(insights_enabled=False):
for worker_id, call_n_times in [(0, 1), (1, 0), (2, 5), (3, 8), (4, 11)]:
insights.init_worker(worker_id)
for _ in range(call_n_times):
insights.update_n_completed_tasks()
self.assertIsNone(insights.worker_n_completed_tasks)
with self.subTest(insights_enabled=True):
insights.reset_insights(enable_insights=True)
for worker_id, call_n_times in [(0, 1), (1, 0), (2, 5), (3, 8), (4, 11)]:
insights.init_worker(worker_id)
for _ in range(call_n_times):
insights.update_n_completed_tasks()
self.assertListEqual(list(insights.worker_n_completed_tasks), [1, 0, 5, 8, 11])