def test_consume_results( self, mock_sla_checker, mock_result_consumer_wait_and_abort, mock_task_get_status): mock_sla_instance = mock.MagicMock() mock_sla_checker.return_value = mock_sla_instance mock_task_get_status.return_value = consts.TaskStatus.RUNNING key = {"kw": {"fake": 2}, "name": "fake", "pos": 0} task = mock.MagicMock() subtask = mock.Mock(spec=objects.Subtask) workload = mock.Mock(spec=objects.Workload) runner = mock.MagicMock() results = [ [{"duration": 1, "timestamp": 3}], [{"duration": 2, "timestamp": 2}] ] runner.result_queue = collections.deque(results) runner.event_queue = collections.deque() with engine.ResultConsumer( key, task, subtask, workload, runner, False) as consumer_obj: pass mock_sla_instance.add_iteration.assert_has_calls([ mock.call({"duration": 1, "timestamp": 3}), mock.call({"duration": 2, "timestamp": 2})]) self.assertEqual([{"duration": 2, "timestamp": 2}, {"duration": 1, "timestamp": 3}], consumer_obj.results)
def test_wait_and_abort_on_no_abort( self, mock_sleep, mock_task_engine__prepare_context, mock_task_get_status, mock_event, mock_thread): runner = mock.MagicMock() workload_cfg = mock.MagicMock() task = mock.MagicMock() subtask = mock.Mock(spec=objects.Subtask) workload = mock.Mock(spec=objects.Workload) mock_task_get_status.return_value = consts.TaskStatus.RUNNING mock_is_done = mock.MagicMock() mock_event.return_value = mock_is_done ctx_manager = mock.MagicMock() mock_is_done.isSet.side_effect = [False, False, False, False, True] res = engine.ResultConsumer(workload_cfg, task=task, subtask=subtask, workload=workload, runner=runner, abort_on_sla_failure=True, ctx_manager=ctx_manager) res.wait_and_abort() # check method don't abort runner if task is not aborted self.assertFalse(runner.abort.called) # test task.get_status is checked until is_done is not set self.assertEqual(4, mock_task_get_status.call_count)
def test_consume_results_no_iteration( self, mock_sla_checker, mock_result_consumer_wait_and_abort, mock_task_get_status, mock_time, mock_log, mock_hook_executor): mock_time.side_effect = [0, 1] mock_sla_instance = mock.MagicMock() mock_sla_results = mock.MagicMock() mock_sla_checker.return_value = mock_sla_instance mock_sla_instance.results.return_value = mock_sla_results mock_task_get_status.return_value = consts.TaskStatus.RUNNING key = {"kw": {"fake": 2}, "name": "fake", "pos": 0} task = mock.MagicMock() subtask = mock.Mock(spec=objects.Subtask) workload = mock.Mock(spec=objects.Workload) runner = mock.MagicMock() results = [] runner.result_queue = collections.deque(results) runner.event_queue = collections.deque() with engine.ResultConsumer( key, task, subtask, workload, runner, False): pass self.assertFalse(workload.add_workload_data.called) workload.set_results.assert_called_once_with({ "full_duration": 1, "sla": mock_sla_results, "load_duration": 0 })
def test_consume_results(self, mock_sla_checker, mock_result_consumer_wait_and_abort, mock_task_get_status): mock_sla_instance = mock.MagicMock() mock_sla_checker.return_value = mock_sla_instance mock_task_get_status.return_value = consts.TaskStatus.RUNNING key = {"kw": {"fake": 2}, "name": "fake", "pos": 0} task = mock.MagicMock() runner = mock.MagicMock() results = [{ "duration": 1, "timestamp": 3 }, { "duration": 2, "timestamp": 2 }] runner.result_queue = collections.deque(results) with engine.ResultConsumer(key, task, runner, False) as consumer_obj: pass self.assertEqual(list(map(mock.call, results)), mock_sla_instance.add_iteration.mock_calls) self.assertEqual(sorted(results, key=lambda x: x["timestamp"]), consumer_obj.results)
def test_consume_results( self, mock_sla_checker, mock_result_consumer_wait_and_abort, mock_task_get_status): mock_sla_instance = mock.MagicMock() mock_sla_checker.return_value = mock_sla_instance mock_task_get_status.return_value = consts.TaskStatus.RUNNING workload_cfg = {"fake": 2, "hooks": []} task = mock.MagicMock() subtask = mock.Mock(spec=objects.Subtask) workload = mock.Mock(spec=objects.Workload) runner = mock.MagicMock() results = [ [{"duration": 1, "timestamp": 3}], [{"duration": 2, "timestamp": 2}] ] runner.result_queue = collections.deque(results) runner.event_queue = collections.deque() ctx_manager = mock.MagicMock() with engine.ResultConsumer(workload_cfg, task=task, subtask=subtask, workload=workload, runner=runner, abort_on_sla_failure=False, ctx_manager=ctx_manager) as consumer_obj: pass mock_sla_instance.add_iteration.assert_has_calls([ mock.call({"duration": 1, "timestamp": 3}), mock.call({"duration": 2, "timestamp": 2})]) self.assertEqual([{"duration": 2, "timestamp": 2}, {"duration": 1, "timestamp": 3}], consumer_obj.results)
def test_consume_results_sla_failure_abort( self, mock_sla_checker, mock_result_consumer_wait_and_abort, mock_task_get_status): mock_sla_instance = mock.MagicMock() mock_sla_checker.return_value = mock_sla_instance mock_sla_instance.add_iteration.side_effect = [True, True, False, False] workload_cfg = {"fake": 2, "hooks": []} task = mock.MagicMock() subtask = mock.Mock(spec=objects.Subtask) workload = mock.Mock(spec=objects.Workload) runner = mock.MagicMock() runner.result_queue = collections.deque( [[{"duration": 1, "timestamp": 1}, {"duration": 2, "timestamp": 2}]] * 4) ctx_manager = mock.MagicMock() with engine.ResultConsumer(workload_cfg, task=task, subtask=subtask, workload=workload, runner=runner, abort_on_sla_failure=True, ctx_manager=ctx_manager): pass self.assertTrue(runner.abort.called) task.update_status.assert_called_once_with( consts.TaskStatus.SOFT_ABORTING)
def test_consume_results_no_iteration( self, mock_sla_checker, mock_result_consumer_wait_and_abort, mock_task_get_status, mock_time, mock_log, mock_hook_executor): mock_time.side_effect = [0, 1] mock_sla_instance = mock.MagicMock() mock_sla_results = mock.MagicMock() mock_sla_checker.return_value = mock_sla_instance mock_sla_instance.results.return_value = mock_sla_results mock_task_get_status.return_value = consts.TaskStatus.RUNNING workload_cfg = {"fake": 2, "hooks": []} task = mock.MagicMock() subtask = mock.Mock(spec=objects.Subtask) workload = mock.Mock(spec=objects.Workload) runner = mock.MagicMock() results = [] runner.result_queue = collections.deque(results) runner.event_queue = collections.deque() ctx_manager = mock.MagicMock() with engine.ResultConsumer(workload_cfg, task=task, subtask=subtask, workload=workload, runner=runner, abort_on_sla_failure=False, ctx_manager=ctx_manager): pass self.assertFalse(workload.add_workload_data.called) workload.set_results.assert_called_once_with( full_duration=1, sla_results=mock_sla_results, load_duration=0, start_time=None, contexts_results=ctx_manager.contexts_results())
def test_consume_results_abort_manually(self, mock_sla_checker, mock_event, mock_thread, mock_task_get_status, mock_hook_executor): runner = mock.MagicMock(result_queue=False) is_done = mock.MagicMock() is_done.isSet.side_effect = (False, True) task = mock.MagicMock() mock_task_get_status.return_value = consts.TaskStatus.ABORTED subtask = mock.Mock(spec=objects.Subtask) workload = mock.Mock(spec=objects.Workload) workload_cfg = {"fake": 2, "hooks": []} mock_hook_executor_instance = mock_hook_executor.return_value ctx_manager = mock.MagicMock() with engine.ResultConsumer(workload_cfg, task=task, subtask=subtask, workload=workload, runner=runner, abort_on_sla_failure=True, ctx_manager=ctx_manager): pass mock_sla_checker.assert_called_once_with(workload_cfg) mock_hook_executor.assert_called_once_with(workload_cfg, task) self.assertFalse(mock_hook_executor_instance.on_iteration.called) mocked_set_aborted = mock_sla_checker.return_value.set_aborted_manually mocked_set_aborted.assert_called_once_with()
def test_consume_results_no_iteration( self, mock_sla_checker, mock_result_consumer_wait_and_abort, mock_task_get_status, mock_time, mock_log, mock_hook_executor): mock_time.side_effect = [0, 1] mock_sla_instance = mock.MagicMock() mock_sla_results = mock.MagicMock() mock_sla_checker.return_value = mock_sla_instance mock_sla_instance.results.return_value = mock_sla_results mock_task_get_status.return_value = consts.TaskStatus.RUNNING key = {"kw": {"fake": 2}, "name": "fake", "pos": 0} task = mock.MagicMock() runner = mock.MagicMock() results = [] runner.result_queue = collections.deque(results) runner.event_queue = collections.deque() with engine.ResultConsumer( key, task, runner, False): pass task.append_results.assert_has_calls([mock.call( key, { "raw": [], "full_duration": 1, "sla": mock_sla_results, "load_duration": 0 } )], any_order=True)
def test_consume_results_sla_failure_continue(self, mock_sla_checker, mock_task_get_status): mock_sla_instance = mock.MagicMock() mock_sla_checker.return_value = mock_sla_instance mock_task_get_status.return_value = consts.TaskStatus.CRASHED mock_sla_instance.add_iteration.side_effect = [ True, True, False, False ] workload_cfg = {"fake": 2, "hooks": []} task = mock.MagicMock() subtask = mock.Mock(spec=objects.Subtask) workload = mock.Mock(spec=objects.Workload) runner = mock.MagicMock() runner.result_queue = collections.deque([[{ "duration": 1, "timestamp": 4 }]] * 4) runner.event_queue = collections.deque() ctx_manager = mock.MagicMock() with engine.ResultConsumer(workload_cfg, task=task, subtask=subtask, workload=workload, runner=runner, ctx_manager=ctx_manager, abort_on_sla_failure=False): pass self.assertEqual(0, runner.abort.call_count)
def test_consume_results_with_unexpected_failure(self, mock_sla_checker, mock_event, mock_thread, mock_task_get_status): mock_sla_instance = mock.MagicMock() mock_sla_checker.return_value = mock_sla_instance workload_cfg = {"fake": 2, "hooks": []} task = mock.MagicMock() subtask = mock.Mock(spec=objects.Subtask) workload = mock.Mock(spec=objects.Workload) runner = mock.MagicMock() runner.result_queue = collections.deque([1]) runner.event_queue = collections.deque() ctx_manager = mock.MagicMock() exc = MyException() try: with engine.ResultConsumer(workload_cfg, task=task, subtask=subtask, workload=workload, runner=runner, ctx_manager=ctx_manager, abort_on_sla_failure=False): raise exc except MyException: pass else: self.fail("ResultConsumer should re-raise the exception.") mock_sla_instance.set_unexpected_failure.assert_has_calls( [mock.call(exc)])
def test_wait_and_abort_on_abort( self, mock_sleep, mock_task_engine__prepare_context, mock_task_get_status, mock_event, mock_thread): runner = mock.MagicMock() workload_cfg = mock.MagicMock() task = mock.MagicMock() subtask = mock.Mock(spec=objects.Subtask) workload = mock.Mock(spec=objects.Workload) mock_task_get_status.side_effect = (consts.TaskStatus.RUNNING, consts.TaskStatus.RUNNING, consts.TaskStatus.ABORTING) mock_is_done = mock.MagicMock() mock_event.return_value = mock_is_done mock_is_done.isSet.return_value = False ctx_manager = mock.MagicMock() res = engine.ResultConsumer(workload_cfg, task=task, subtask=subtask, workload=workload, runner=runner, abort_on_sla_failure=True, ctx_manager=ctx_manager) res.wait_and_abort() runner.abort.assert_called_with() # test task.get_status is checked until is_done is not set self.assertEqual(3, mock_task_get_status.call_count)
def test_consume_results_abort_manually(self, mock_sla_checker, mock_event, mock_thread, mock_task_get_status, mock_hook_executor): runner = mock.MagicMock(result_queue=False) is_done = mock.MagicMock() is_done.isSet.side_effect = (False, True) task = mock.MagicMock() mock_task_get_status.return_value = consts.TaskStatus.ABORTED subtask = mock.Mock(spec=objects.Subtask) workload = mock.Mock(spec=objects.Workload) key = {"kw": {"fake": 2}, "name": "fake", "pos": 0} mock_hook_executor_instance = mock_hook_executor.return_value with engine.ResultConsumer(key, task, subtask, workload, runner, True): pass mock_sla_checker.assert_called_once_with(key["kw"]) mock_hook_executor.assert_called_once_with(key["kw"], task) self.assertFalse(mock_hook_executor_instance.on_iteration.called) mocked_set_aborted = mock_sla_checker.return_value.set_aborted_manually mocked_set_aborted.assert_called_once_with()
def test_consume_events( self, mock_sla_checker, mock_result_consumer_wait_and_abort, mock_task_get_status, mock_time, mock_hook_executor, mock_log): mock_time.side_effect = [0, 1] mock_sla_instance = mock_sla_checker.return_value mock_sla_results = mock_sla_instance.results.return_value mock_hook_executor_instance = mock_hook_executor.return_value mock_hook_results = mock_hook_executor_instance.results.return_value mock_task_get_status.return_value = consts.TaskStatus.RUNNING workload_cfg = {"fake": 2, "hooks": [{"config": True}]} task = mock.MagicMock() subtask = mock.Mock(spec=objects.Subtask) workload = mock.Mock(spec=objects.Workload) runner = mock.MagicMock() events = [ {"type": "iteration", "value": 1}, {"type": "iteration", "value": 2}, {"type": "iteration", "value": 3} ] runner.result_queue = collections.deque() runner.event_queue = collections.deque(events) ctx_manager = mock.MagicMock() consumer_obj = engine.ResultConsumer( workload_cfg, task=task, subtask=subtask, workload=workload, runner=runner, abort_on_sla_failure=False, ctx_manager=ctx_manager) stop_event = threading.Event() def set_stop_event(event_type, value): if not runner.event_queue: stop_event.set() mock_hook_executor_instance.on_event.side_effect = set_stop_event with consumer_obj: stop_event.wait(1) mock_hook_executor_instance.on_event.assert_has_calls([ mock.call(event_type="iteration", value=1), mock.call(event_type="iteration", value=2), mock.call(event_type="iteration", value=3) ]) self.assertFalse(workload.add_workload_data.called) workload.set_results.assert_called_once_with( full_duration=1, load_duration=0, sla_results=mock_sla_results, hooks_results=mock_hook_results, start_time=None, contexts_results=ctx_manager.contexts_results())
def test_consume_results_sla_failure_continue(self, mock_sla_checker): mock_sla_instance = mock.MagicMock() mock_sla_checker.return_value = mock_sla_instance mock_sla_instance.add_iteration.side_effect = [True, True, False, False] key = {"kw": {"fake": 2}, "name": "fake", "pos": 0} task = mock.MagicMock() runner = mock.MagicMock() runner.result_queue = collections.deque([1, 2, 3, 4]) with engine.ResultConsumer(key, task, runner, False): pass self.assertEqual(0, runner.abort.call_count)
def test_consume_events( self, mock_sla_checker, mock_result_consumer_wait_and_abort, mock_task_get_status, mock_time, mock_hook_executor, mock_log): mock_time.side_effect = [0, 1] mock_sla_instance = mock_sla_checker.return_value mock_sla_results = mock_sla_instance.results.return_value mock_hook_executor_instance = mock_hook_executor.return_value mock_hook_results = mock_hook_executor_instance.results.return_value mock_task_get_status.return_value = consts.TaskStatus.RUNNING key = {"kw": {"fake": 2, "hooks": []}, "name": "fake", "pos": 0} task = mock.MagicMock() subtask = mock.Mock(spec=objects.Subtask) workload = mock.Mock(spec=objects.Workload) runner = mock.MagicMock() events = [ {"type": "iteration", "value": 1}, {"type": "iteration", "value": 2}, {"type": "iteration", "value": 3} ] runner.result_queue = collections.deque() runner.event_queue = collections.deque(events) consumer_obj = engine.ResultConsumer(key, task, subtask, workload, runner, False) stop_event = threading.Event() def set_stop_event(event_type, value): if not runner.event_queue: stop_event.set() mock_hook_executor_instance.on_event.side_effect = set_stop_event with consumer_obj: stop_event.wait(1) mock_hook_executor_instance.on_event.assert_has_calls([ mock.call(event_type="iteration", value=1), mock.call(event_type="iteration", value=2), mock.call(event_type="iteration", value=3) ]) self.assertFalse(workload.add_workload_data.called) workload.set_results.assert_called_once_with({ "full_duration": 1, "sla": mock_sla_results, "hooks": mock_hook_results, "load_duration": 0 })
def test_consume_results_chunked( self, mock_sla_checker, mock_result_consumer_wait_and_abort, mock_task_get_status, mock_conf): mock_conf.raw_result_chunk_size = 2 mock_sla_instance = mock.MagicMock() mock_sla_checker.return_value = mock_sla_instance mock_task_get_status.return_value = consts.TaskStatus.RUNNING workload_cfg = {"fake": 2, "hooks": []} task = mock.MagicMock(spec=objects.Task) subtask = mock.Mock(spec=objects.Subtask) workload = mock.Mock(spec=objects.Workload) runner = mock.MagicMock() results = [ [{"duration": 1, "timestamp": 3}, {"duration": 2, "timestamp": 2}, {"duration": 3, "timestamp": 3}], [{"duration": 4, "timestamp": 2}, {"duration": 5, "timestamp": 3}], [{"duration": 6, "timestamp": 2}], [{"duration": 7, "timestamp": 1}], ] runner.result_queue = collections.deque(results) runner.event_queue = collections.deque() with engine.ResultConsumer(workload_cfg, task, subtask, workload, runner, False) as consumer_obj: pass mock_sla_instance.add_iteration.assert_has_calls([ mock.call({"duration": 1, "timestamp": 3}), mock.call({"duration": 2, "timestamp": 2}), mock.call({"duration": 3, "timestamp": 3}), mock.call({"duration": 4, "timestamp": 2}), mock.call({"duration": 5, "timestamp": 3}), mock.call({"duration": 6, "timestamp": 2}), mock.call({"duration": 7, "timestamp": 1})]) self.assertEqual([{"duration": 7, "timestamp": 1}], consumer_obj.results) workload.add_workload_data.assert_has_calls([ mock.call(0, {"raw": [{"duration": 2, "timestamp": 2}, {"duration": 1, "timestamp": 3}]}), mock.call(1, {"raw": [{"duration": 4, "timestamp": 2}, {"duration": 3, "timestamp": 3}]}), mock.call(2, {"raw": [{"duration": 6, "timestamp": 2}, {"duration": 5, "timestamp": 3}]}), mock.call(3, {"raw": [{"duration": 7, "timestamp": 1}]})])
def test_consume_results(self, mock_sla_checker): mock_sla_instance = mock.MagicMock() mock_sla_checker.return_value = mock_sla_instance key = {"kw": {"fake": 2}, "name": "fake", "pos": 0} task = mock.MagicMock() runner = mock.MagicMock() runner.result_queue = collections.deque([1, 2]) with engine.ResultConsumer( key, task, runner, False) as consumer_obj: pass expected_iteration_calls = [mock.call(1), mock.call(2)] self.assertEqual(expected_iteration_calls, mock_sla_instance.add_iteration.mock_calls) self.assertEqual([1, 2], consumer_obj.results)
def test_comsume_results_with_unexpected_failure(self, mock_sla_checker): mock_sla_instance = mock.MagicMock() mock_sla_checker.return_value = mock_sla_instance key = {"kw": {"fake": 2}, "name": "fake", "pos": 0} task = mock.MagicMock() runner = mock.MagicMock() runner.result_queue = collections.deque([1]) exc = TestException() try: with engine.ResultConsumer(key, task, runner, False): raise exc except TestException: pass mock_sla_instance.set_unexpected_failure.assert_has_calls( [mock.call(exc)])
def test_consume_results_sla_failure_abort( self, mock_sla_checker, mock_result_consumer_wait_and_abort, mock_task_get_status): mock_sla_instance = mock.MagicMock() mock_sla_checker.return_value = mock_sla_instance mock_sla_instance.add_iteration.side_effect = [ True, True, False, False ] key = {"kw": {"fake": 2}, "name": "fake", "pos": 0} task = mock.MagicMock() runner = mock.MagicMock() runner.result_queue = collections.deque([1, 2, 3, 4]) with engine.ResultConsumer(key, task, runner, True): pass self.assertTrue(runner.abort.called)
def test_consume_results_sla_failure_continue(self, mock_sla_checker, mock_task_get_status): mock_sla_instance = mock.MagicMock() mock_sla_checker.return_value = mock_sla_instance mock_task_get_status.return_value = consts.TaskStatus.FAILED mock_sla_instance.add_iteration.side_effect = [True, True, False, False] key = {"kw": {"fake": 2}, "name": "fake", "pos": 0} task = mock.MagicMock() runner = mock.MagicMock() runner.result_queue = collections.deque( [[{"duration": 1, "timestamp": 4}]] * 4) with engine.ResultConsumer(key, task, runner, False): pass self.assertEqual(0, runner.abort.call_count)
def test_wait_and_abort_on_abort( self, mock_task_engine__get_runner, mock_sleep, mock_task_engine__prepare_context, mock_task_get_status, mock_event, mock_thread): runner = mock.MagicMock() key = mock.MagicMock() task = mock.MagicMock() mock_task_get_status.side_effect = (consts.TaskStatus.RUNNING, consts.TaskStatus.RUNNING, consts.TaskStatus.ABORTING) mock_is_done = mock.MagicMock() mock_event.return_value = mock_is_done mock_is_done.isSet.return_value = False res = engine.ResultConsumer(key, task, runner, True) res.wait_and_abort() runner.abort.assert_called_with() # test task.get_status is checked until is_done is not set self.assertEqual(3, mock_task_get_status.call_count)
def test_wait_and_abort_on_no_abort( self, mock_task_engine__get_runner, mock_sleep, mock_task_engine__prepare_context, mock_task_get_status, mock_event, mock_thread): runner = mock.MagicMock() key = mock.MagicMock() task = mock.MagicMock() mock_task_get_status.return_value = consts.TaskStatus.RUNNING mock_is_done = mock.MagicMock() mock_event.return_value = mock_is_done mock_is_done.isSet.side_effect = [False, False, False, False, True] res = engine.ResultConsumer(key, task, runner, True) res.wait_and_abort() # check method don't abort runner if task is not aborted self.assertFalse(runner.abort.called) # test task.get_status is checked until is_done is not set self.assertEqual(4, mock_task_get_status.call_count)
def test_consume_results_abort_manually(self, mock_sla_checker, mock_event, mock_thread, mock_task_get_status): runner = mock.MagicMock(result_queue=False) is_done = mock.MagicMock() is_done.isSet.side_effect = (False, True) task = mock.MagicMock() mock_task_get_status.return_value = consts.TaskStatus.ABORTED key = {"kw": {"fake": 2}, "name": "fake", "pos": 0} eng = engine.BenchmarkEngine({}, task) eng.duration = 123 eng.full_duration = 456 with engine.ResultConsumer(key, task, runner, True): pass mock_sla_checker.assert_called_once_with(key["kw"]) mocked_set_aborted = mock_sla_checker.return_value.set_aborted_manually mocked_set_aborted.assert_called_once_with()
def test_consume_results_with_unexpected_failure(self, mock_sla_checker, mock_event, mock_thread, mock_task_get_status): mock_sla_instance = mock.MagicMock() mock_sla_checker.return_value = mock_sla_instance key = {"kw": {"fake": 2}, "name": "fake", "pos": 0} task = mock.MagicMock() subtask = mock.Mock(spec=objects.Subtask) workload = mock.Mock(spec=objects.Workload) runner = mock.MagicMock() runner.result_queue = collections.deque([1]) runner.event_queue = collections.deque() exc = MyException() try: with engine.ResultConsumer(key, task, subtask, workload, runner, False): raise exc except MyException: pass mock_sla_instance.set_unexpected_failure.assert_has_calls( [mock.call(exc)])