def test_update_task_insights(self): """ Test whether task insights are being updated correctly """ MockDatetimeNow.RETURN_VALUES = [datetime(1970, 1, 1, 0, 0, 1, 0), datetime(1970, 1, 1, 0, 0, 2, 0)] MockDatetimeNow.CURRENT_IDX = 0 insights = WorkerInsights(mp.get_context('fork'), n_jobs=2) insights.max_task_duration_last_updated = datetime(1970, 1, 1, 0, 0, 0, 0) # Shouldn't do anything when insights haven't been enabled with self.subTest(insights_enabled=False), patch('mpire.insights.datetime', new=MockDatetimeNow): for worker_id in range(2): insights.init_worker(worker_id) insights.update_task_insights() self.assertIsNone(insights.max_task_duration) self.assertIsNone(insights.max_task_args) self.assertEqual(insights.max_task_duration_last_updated, datetime(1970, 1, 1, 0, 0, 0, 0)) insights.reset_insights(enable_insights=True) insights.max_task_duration_last_updated = datetime(1970, 1, 1, 0, 0, 0, 0) MockDatetimeNow.CURRENT_IDX = 0 with self.subTest(insights_enabled=True), patch('mpire.insights.datetime', new=MockDatetimeNow): for worker_id, max_task_duration_list in [(0, [(5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9')]), (1, [(0, '0'), (1, '1'), (2, '2'), (3, '3'), (4, '4')])]: insights.init_worker(worker_id) insights.max_task_duration_list = max_task_duration_list insights.update_task_insights() self.assertEqual(insights.max_task_duration_last_updated, datetime(1970, 1, 1, 0, 0, worker_id + 1, 0)) self.assertListEqual(list(insights.max_task_duration), [5, 6, 7, 8, 9, 0, 1, 2, 3, 4]) self.assertListEqual(list(insights.max_task_args), ['5', '6', '7', '8', '9', '0', '1', '2', '3', '4'])
def test_update_task_insights_once_in_a_while(self): """ Test whether the update_task_insights is triggered correctly. It should be called two times based on the mocked datetime.now() values: for the last two entries the time difference compared to max_task_duration_last_updated is bigger than two seconds. """ MockDatetimeNow.RETURN_VALUES = [datetime(1970, 1, 1, 0, 0, 0, 0), datetime(1970, 1, 1, 0, 0, 2, 0), datetime(1970, 1, 1, 0, 0, 3, 0), datetime(1970, 1, 1, 0, 0, 7, 0), datetime(1970, 1, 1, 0, 0, 8, 0)] MockDatetimeNow.CURRENT_IDX = 0 insights = WorkerInsights(mp.get_context('fork'), n_jobs=5) insights.max_task_duration_last_updated = datetime(1970, 1, 1, 0, 0, 1, 0) # Shouldn't do anything when insights haven't been enabled with self.subTest(insights_enabled=False), patch('mpire.insights.datetime', new=MockDatetimeNow), \ patch.object(insights, 'update_task_insights') as p: for worker_id in range(5): insights.init_worker(worker_id) insights.update_task_insights_once_in_a_while() self.assertEqual(p.call_count, 0) insights.reset_insights(enable_insights=True) insights.max_task_duration_last_updated = datetime(1970, 1, 1, 0, 0, 1, 0) MockDatetimeNow.CURRENT_IDX = 0 with self.subTest(insights_enabled=True), patch('mpire.insights.datetime', new=MockDatetimeNow), \ patch.object(insights, 'update_task_insights') as p: for worker_id in range(5): insights.init_worker(worker_id) insights.update_task_insights_once_in_a_while() self.assertEqual(p.call_count, 2)
def test_update_start_up_time(self): """ Test that the start up time is correctly added to worker_start_up_time for the right index """ MockDatetimeNow.RETURN_VALUES = [datetime(1970, 1, 1, 0, 0, 0, 0), datetime(1970, 1, 1, 0, 0, 2, 0), datetime(1970, 1, 1, 0, 0, 3, 0), datetime(1970, 1, 1, 0, 0, 7, 0), datetime(1970, 1, 1, 0, 0, 8, 0)] MockDatetimeNow.CURRENT_IDX = 0 insights = WorkerInsights(mp.get_context('fork'), n_jobs=5) # Shouldn't do anything when insights haven't been enabled with self.subTest(insights_enabled=False), patch('mpire.insights.datetime', new=MockDatetimeNow): for worker_id in range(5): insights.init_worker(worker_id) insights.update_start_up_time(datetime(1970, 1, 1, 0, 0, 0, 0)) self.assertIsNone(insights.worker_start_up_time) insights.reset_insights(enable_insights=True) insights.max_task_duration_last_updated = datetime(1970, 1, 1, 0, 0, 0, 0) MockDatetimeNow.CURRENT_IDX = 0 with self.subTest(insights_enabled=True), patch('mpire.insights.datetime', new=MockDatetimeNow): for worker_id in range(5): insights.init_worker(worker_id) insights.update_start_up_time(datetime(1970, 1, 1, 0, 0, 0, 0)) self.assertListEqual(list(insights.worker_start_up_time), [0, 2, 3, 7, 8])
def test_init_worker(self): """ Test that the right containers are selected given a worker ID """ insights = WorkerInsights(mp.get_context('fork'), n_jobs=5) # Should've only set worker ID when insights haven't been enabled with self.subTest(insights_enabled=False): insights.init_worker(3) self.assertEqual(insights.worker_id, 3) self.assertIsNone(insights.max_task_duration_list) with self.subTest(insights_enabled=True): insights.reset_insights(enable_insights=True) insights.max_task_duration[:] = range(25) insights.max_task_args[:] = map(str, range(25)) for worker_id, expected_task_duration_list in [ (0, [(0.0, '0'), (1.0, '1'), (2.0, '2'), (3.0, '3'), (4.0, '4')]), (1, [(5.0, '5'), (6.0, '6'), (7.0, '7'), (8.0, '8'), (9.0, '9')]), (4, [(20.0, '20'), (21.0, '21'), (22.0, '22'), (23.0, '23'), (24.0, '24')]) ]: with self.subTest(worker_id=worker_id): insights.init_worker(worker_id) self.assertEqual(insights.worker_id, worker_id) self.assertListEqual(insights.max_task_duration_list, expected_task_duration_list)
def test_update_n_completed_tasks(self): """ Test that the number of completed tasks is correctly added to worker_n_completed_tasks for the right index """ insights = WorkerInsights(mp.get_context('fork'), n_jobs=5) # Shouldn't do anything when insights haven't been enabled with self.subTest(insights_enabled=False): for worker_id, call_n_times in [(0, 1), (1, 0), (2, 5), (3, 8), (4, 11)]: insights.init_worker(worker_id) for _ in range(call_n_times): insights.update_n_completed_tasks() self.assertIsNone(insights.worker_n_completed_tasks) with self.subTest(insights_enabled=True): insights.reset_insights(enable_insights=True) for worker_id, call_n_times in [(0, 1), (1, 0), (2, 5), (3, 8), (4, 11)]: insights.init_worker(worker_id) for _ in range(call_n_times): insights.update_n_completed_tasks() self.assertListEqual(list(insights.worker_n_completed_tasks), [1, 0, 5, 8, 11])