def testProcessCommandEvent_ignoreOutOfDateEvent( self, mock_notify, mock_command_event_type_count): """Should complete command state for InvocationCompleted events.""" _, request_id, _, command_id = self.command.key.flat() command_manager.ScheduleTasks([self.command]) tasks = command_manager.GetActiveTasks(self.command) command_task_store.LeaseTask(tasks[0].task_id) command_event_test_util.CreateCommandAttempt( self.command, "0", common.CommandState.UNKNOWN, task=tasks[0]) invocation_completed_event = command_event_test_util.CreateTestCommandEvent( request_id, command_id, "0", "InvocationCompleted", task=tasks[0], time=TIMESTAMP_INT + 60) invocation_started_event = command_event_test_util.CreateTestCommandEvent( request_id, command_id, "0", "InvocationStarted", task=tasks[0], time=TIMESTAMP_INT + 30) queried_command = command_manager.GetCommand(request_id, command_id) self.assertEqual(common.CommandState.QUEUED, queried_command.state) command_event_handler.ProcessCommandEvent(invocation_completed_event) queried_command = command_manager.GetCommand(request_id, command_id) command_attempt = command_manager.GetCommandAttempts( request_id, command_id)[0] self.assertEqual( TIMESTAMP + datetime.timedelta(seconds=60), command_attempt.last_event_time) self.assertEqual(common.CommandState.COMPLETED, queried_command.state) # The second event is ignored. command_event_handler.ProcessCommandEvent(invocation_started_event) queried_command = command_manager.GetCommand(request_id, command_id) command_attempt = command_manager.GetCommandAttempts( request_id, command_id)[0] self.assertEqual( TIMESTAMP + datetime.timedelta(seconds=60), command_attempt.last_event_time) self.assertEqual(common.CommandState.COMPLETED, queried_command.state) tasks = command_manager.GetActiveTasks(self.command) self.assertEqual(len(tasks), 0) attempts = command_manager.GetCommandAttempts( request_id, command_id) self.assertEqual(1, len(attempts)) self.assertEqual("summary", attempts[0].summary) self.assertEqual(1000, attempts[0].total_test_count) self.assertEqual(100, attempts[0].failed_test_count) mock_notify.assert_called_with(request_id) # Metrics should still be logged for out of order events started_metric_fields = { metric.METRIC_FIELD_HOSTNAME: "hostname", metric.METRIC_FIELD_TYPE: "InvocationStarted" } completed_metric_fields = { metric.METRIC_FIELD_HOSTNAME: "hostname", metric.METRIC_FIELD_TYPE: "InvocationCompleted" } mock_command_event_type_count.Increment.assert_has_calls( [mock.call(completed_metric_fields), mock.call(started_metric_fields)])
def testProcessCommandEvent_nonFinal_delete(self, attempt_metric): # Test ProcessCommandEvent for a non-final state with deletion command = self._CreateCommand(run_count=2) command_manager.ScheduleTasks([command]) _, request_id, _, command_id = command.key.flat() tasks = command_manager.GetActiveTasks(command) self.assertEqual(len(tasks), 2) command_task_store.LeaseTask(tasks[0].task_id) command_event_test_util.CreateCommandAttempt( command, "attempt0", common.CommandState.UNKNOWN, task=tasks[0]) event = command_event_test_util.CreateTestCommandEvent( request_id, command_id, "attempt0", common.InvocationEventType.INVOCATION_COMPLETED, task=tasks[0], time=TIMESTAMP) commander.ProcessCommandEvent(event) tasks = command_manager.GetActiveTasks(command) self.assertEqual(len(tasks), 1) self.assertEqual(tasks[0].task_id, "%s-%s-1" % (request_id, command_id)) self.assertEqual(tasks[0].leasable, True) self.assertEqual(tasks[0].run_index, 1) self.assertEqual(tasks[0].attempt_index, 0) command = command.key.get(use_cache=False) self.assertEqual(common.CommandState.QUEUED, command.state) request = command.key.parent().get(use_cache=False) self.assertEqual(common.RequestState.QUEUED, request.state) attempt_metric.assert_called_once_with(cluster_id=command.cluster, run_target=command.run_target, hostname="hostname", state="COMPLETED")
def testProcessCommandEvent_final(self, plugin, attempt_metric): # Test ProcessCommandEvent for a final state command = self._CreateCommand() command_manager.ScheduleTasks([command]) _, request_id, _, command_id = command.key.flat() tasks = command_manager.GetActiveTasks(command) self.assertEqual(len(tasks), 1) command_task_store.LeaseTask(tasks[0].task_id) attempt = command_event_test_util.CreateCommandAttempt( command, "attempt0", common.CommandState.UNKNOWN, task=tasks[0]) event = command_event_test_util.CreateTestCommandEvent( request_id, command_id, "attempt0", common.InvocationEventType.INVOCATION_COMPLETED, task=tasks[0], time=TIMESTAMP) commander.ProcessCommandEvent(event) tasks = command_manager.GetActiveTasks(command) self.assertEqual(len(tasks), 0) command = command.key.get(use_cache=False) self.assertEqual(common.CommandState.COMPLETED, command.state) request = command.key.parent().get(use_cache=False) self.assertEqual(common.RequestState.COMPLETED, request.state) attempt_metric.assert_called_once_with(cluster_id=command.cluster, run_target=command.run_target, hostname="hostname", state="COMPLETED") plugin.assert_has_calls([ mock.call.OnCreateCommands([ plugin_base.CommandInfo(command_id=COMMAND_ID, command_line="command_line1", run_count=1, shard_count=1, shard_index=0) ], { "ants_invocation_id": "i123", "command_ants_work_unit_id": "w123" }, {}), mock.call.OnProcessCommandEvent( command, hamcrest.match_equality( hamcrest.all_of( hamcrest.has_property("command_id", attempt.command_id), hamcrest.has_property("attempt_id", attempt.attempt_id), hamcrest.has_property("task_id", attempt.task_id), )), event_data={ "total_test_count": 1000, "device_lost_detected": 0, "failed_test_run_count": 10, "passed_test_count": 900, "failed_test_count": 100, "summary": "summary" }), ])
def testProcessCommandEvent_multiDevice( self, mock_notify, mock_command_event_type_count): """Should populate multiple devices.""" _, request_id, _, command_id = self.command.key.flat() command_manager.ScheduleTasks([self.command]) tasks = command_manager.GetActiveTasks(self.command) self.assertEqual(len(tasks), 1) command_task_store.LeaseTask(tasks[0].task_id) command_event_test_util.CreateCommandAttempt( self.command, "0", common.CommandState.UNKNOWN, task=tasks[0]) queried_command = command_manager.GetCommand(request_id, command_id) self.assertNotEqual(common.CommandState.RUNNING, queried_command.state) event = command_event_test_util.CreateTestCommandEvent( request_id, command_id, "0", "InvocationStarted", task=tasks[0], device_serials=["d1", "d2"]) command_event_handler.ProcessCommandEvent(event) queried_command = command_manager.GetCommand(request_id, command_id) self.assertEqual(common.CommandState.RUNNING, queried_command.state) tasks = command_manager.GetActiveTasks(self.command) self.assertEqual(len(tasks), 1) self.assertEqual(tasks[0].attempt_index, 0) self.assertEqual(tasks[0].leasable, False) attempts = command_manager.GetCommandAttempts(request_id, command_id) self.assertEqual(1, len(attempts)) self.assertEqual(["d1", "d2"], attempts[0].device_serials) mock_notify.assert_called_with(request_id) expected_metric_fields = { metric.METRIC_FIELD_HOSTNAME: "hostname", metric.METRIC_FIELD_TYPE: "InvocationStarted" } mock_command_event_type_count.Increment.assert_called_once_with( expected_metric_fields)
def testProcessCommandEvent_TFShutdown( self, mock_notify, mock_command_event_type_count): """Mark command events that were terminated by TF shutdown as CANCELED.""" _, request_id, _, command_id = self.command.key.flat() command_manager.ScheduleTasks([self.command]) error = "RunInterruptedException: Tradefed is shutting down." for i in range(3): tasks = command_manager.GetActiveTasks(self.command) self.assertEqual(len(tasks), 1) command_task_store.LeaseTask(tasks[0].task_id) command_event_test_util.CreateCommandAttempt( self.command, str(i), common.CommandState.UNKNOWN, task=tasks[0]) event = command_event_test_util.CreateTestCommandEvent( request_id, command_id, str(i), "InvocationCompleted", task=tasks[0], data={"error": error}) command_event_handler.ProcessCommandEvent(event) # After three cancelled attempts, the command should still be queued. queried_command = self.command.key.get() self.assertEqual(common.CommandState.QUEUED, queried_command.state) tasks = command_manager.GetActiveTasks(self.command) self.assertEqual(len(tasks), 1) attempts = command_manager.GetCommandAttempts(request_id, command_id) self.assertEqual(3, len(attempts)) mock_notify.assert_called_with(request_id) expected_metric_fields = { metric.METRIC_FIELD_HOSTNAME: "hostname", metric.METRIC_FIELD_TYPE: "InvocationCompleted" } mock_command_event_type_count.Increment.assert_has_calls( [mock.call(expected_metric_fields)] * 3)
def testProcessCommandEvent_invocationCompleted_multipleRunsWithErrors( self, mock_notify): """Should complete command state for InvocationCompleted events. This tests a case when a run count is greater than command_manager.MAX_TASK_COUNT. Args: mock_notify: mock function to notify request state changes. """ run_count = 100 _, request_id, _, command_id = self.command.key.flat() self.command.run_count = run_count self.command.put() command_manager.ScheduleTasks([self.command]) max_error_count = ( command_manager.MAX_ERROR_COUNT_BASE + int(run_count * command_manager.MAX_ERROR_COUNT_RATIO)) for i in range(max_error_count): tasks = command_manager.GetActiveTasks(self.command) self.assertEqual( len(tasks), min(run_count - i, command_manager.MAX_TASK_COUNT)) next_leasable_task = next((t for t in tasks if t.leasable), None) command_task_store.LeaseTask(next_leasable_task.task_id) command_event_test_util.CreateCommandAttempt( self.command, str(i), common.CommandState.UNKNOWN, task=next_leasable_task) # It should be marked as error at the max error count attempts. queried_command = command_manager.GetCommand(request_id, command_id) self.assertNotEqual(common.CommandState.ERROR, queried_command.state) event = command_event_test_util.CreateTestCommandEvent( request_id, command_id, str(i), "InvocationCompleted", task=next_leasable_task, error="error") command_event_handler.ProcessCommandEvent(event) queried_command = command_manager.GetCommand(request_id, command_id) self.assertEqual(common.CommandState.ERROR, queried_command.state) tasks = command_manager.GetActiveTasks(self.command) self.assertEqual(len(tasks), 0) command_attempts = command_manager.GetCommandAttempts( request_id, command_id) self.assertEqual(len(command_attempts), max_error_count) run_attempt_pairs = [ (attempt.run_index, attempt.attempt_index) for attempt in command_attempts] self.assertEqual(len(set(run_attempt_pairs)), len(command_attempts)) mock_notify.assert_called_with(request_id)
def testProcessCommandEvent_invocationCompleted_multipleRuns_20( self, mock_notify): """Should complete command state for InvocationCompleted events. This tests a case when a run count is greater than command_manager.MAX_TASK_COUNT. Args: mock_notify: mock function to notify request state changes. """ run_count = 100 _, request_id, _, command_id = self.command.key.flat() self.command.run_count = run_count self.command.put() command_manager.ScheduleTasks([self.command]) for i in range(run_count): tasks = command_manager.GetActiveTasks(self.command) self.assertEqual( len(tasks), min(run_count - i, command_manager.MAX_TASK_COUNT)) next_leasable_task = next((t for t in tasks if t.leasable), None) command_task_store.LeaseTask(next_leasable_task.task_id) queried_command = command_manager.GetCommand(request_id, command_id) self.assertNotEqual(common.CommandState.COMPLETED, queried_command.state) command_event_test_util.CreateCommandAttempt( self.command, str(i), common.CommandState.UNKNOWN, task=next_leasable_task) event = command_event_test_util.CreateTestCommandEvent( request_id, command_id, str(i), "InvocationCompleted", task=next_leasable_task) command_event_handler.ProcessCommandEvent(event) queried_command = command_manager.GetCommand(request_id, command_id) self.assertEqual(common.CommandState.COMPLETED, queried_command.state) tasks = command_manager.GetActiveTasks(self.command) self.assertEqual(len(tasks), 0) command_attempts = command_manager.GetCommandAttempts( request_id, command_id) self.assertEqual(len(command_attempts), run_count) self.assertSetEqual( set(attempt.run_index for attempt in command_attempts), set(range(run_count))) mock_notify.assert_called_with(request_id)
def testProcessCommandEvent_notUpdated(self, plugin, attempt_metric): """Test ProcessCommandEvent with no update.""" command = self._CreateCommand() command_manager.ScheduleTasks([command]) tasks = command_manager.GetActiveTasks(command) self.assertEqual(len(tasks), 1) command_task_store.LeaseTask(tasks[0].task_id) command = command.key.get(use_cache=False) self.assertEqual(common.CommandState.QUEUED, command.state) request = command.key.parent().get(use_cache=False) self.assertEqual(common.RequestState.QUEUED, request.state) _, request_id, _, command_id = command.key.flat() event = command_event_test_util.CreateTestCommandEvent( request_id, command_id, "attempt0", common.InvocationEventType.INVOCATION_STARTED, time=TIMESTAMP) commander.ProcessCommandEvent(event) tasks = command_manager.GetActiveTasks(command) self.assertEqual(len(tasks), 1) self.assertEqual(tasks[0].leasable, False) self.assertEqual(tasks[0].attempt_index, 0) command = command.key.get(use_cache=False) self.assertEqual(common.CommandState.QUEUED, command.state) request = command.key.parent().get(use_cache=False) self.assertEqual(common.RequestState.QUEUED, request.state) attempt_metric.assert_not_called() plugin.assert_has_calls([ mock.call.OnCreateCommands([ plugin_base.CommandInfo(command_id=COMMAND_ID, command_line="command_line1", run_count=1, shard_count=1, shard_index=0) ], { "ants_invocation_id": "i123", "command_ants_work_unit_id": "w123" }, {}), ])
def testProcessCommandEvent_allocationFailed(self, mock_command_event_type_count, mock_event_legacy_count): """Should reschedule tasks that send AllocationFailed events.""" _, request_id, _, command_id = self.command.key.flat() command_manager.ScheduleTasks([self.command]) # Command should stay queued even after MAX_CANCELED_COUNT_BASE different # attempts result in an AllocationFailed error. num_attempts = command_manager.MAX_CANCELED_COUNT_BASE for i in range(num_attempts): tasks = command_manager.GetActiveTasks(self.command) self.assertEqual(len(tasks), 1) command_task_store.LeaseTask(tasks[0].task_id) command_event_test_util.CreateCommandAttempt( self.command, str(i), common.CommandState.UNKNOWN, task=tasks[0]) queried_command = command_manager.GetCommand(request_id, command_id) self.assertNotEqual(common.CommandState.ERROR, queried_command.state) event = command_event_test_util.CreateTestCommandEvent( request_id, command_id, str(i), "AllocationFailed", task=tasks[0]) command_event_handler.ProcessCommandEvent(event) queried_command = command_manager.GetCommand(request_id, command_id) self.assertEqual(common.CommandState.CANCELED, queried_command.state) tasks = command_manager.GetActiveTasks(self.command) self.assertEqual(len(tasks), 0) command_attempts = command_manager.GetCommandAttempts( request_id, command_id) self.assertEqual( len(command_attempts), command_manager.MAX_CANCELED_COUNT_BASE) self.assertSetEqual( set(attempt.attempt_index for attempt in command_attempts), set(range(command_manager.MAX_CANCELED_COUNT_BASE))) expected_metric_fields = { metric.METRIC_FIELD_HOSTNAME: "hostname", metric.METRIC_FIELD_TYPE: "AllocationFailed" } mock_command_event_type_count.Increment.assert_has_calls( [mock.call(expected_metric_fields)] * num_attempts) mock_event_legacy_count.Increment.assert_has_calls([mock.call({})] * num_attempts)
def testProcessCommandEvent_unknownEvent(self, mock_notify): """Should update command state for unknown events.""" _, request_id, _, command_id = self.command.key.flat() command_manager.ScheduleTasks([self.command]) tasks = command_manager.GetActiveTasks(self.command) self.assertEqual(len(tasks), 1) command_task_store.LeaseTask(tasks[0].task_id) command_event_test_util.CreateCommandAttempt( self.command, "0", common.CommandState.UNKNOWN, task=tasks[0]) queried_command = command_manager.GetCommand(request_id, command_id) self.assertEqual(common.CommandState.QUEUED, queried_command.state) event = command_event_test_util.CreateTestCommandEvent( request_id, command_id, "0", "SomeRandomInexistentType", task=tasks[0]) command_event_handler.ProcessCommandEvent(event) queried_command = command_manager.GetCommand(request_id, command_id) self.assertEqual(common.CommandState.QUEUED, queried_command.state) tasks = command_manager.GetActiveTasks(self.command) self.assertEqual(len(tasks), 1) mock_notify.not_called()
def testProcessCommandEvent_InvocationStarted( self, mock_notify, mock_command_event_type_count): """Should update command state for InvocationStarted events. State should become RUNNING if it isn't already. Args: mock_notify: mock function to notify request state changes. mock_command_event_type_count: mock command event type count metric """ _, request_id, _, command_id = self.command.key.flat() command_manager.ScheduleTasks([self.command]) tasks = command_manager.GetActiveTasks(self.command) self.assertEqual(len(tasks), 1) command_task_store.LeaseTask(tasks[0].task_id) command_event_test_util.CreateCommandAttempt( self.command, "0", common.CommandState.UNKNOWN, task=tasks[0]) queried_command = command_manager.GetCommand(request_id, command_id) self.assertNotEqual(common.CommandState.RUNNING, queried_command.state) event = command_event_test_util.CreateTestCommandEvent( request_id, command_id, "0", "InvocationStarted", task=tasks[0]) command_event_handler.ProcessCommandEvent(event) queried_command = command_manager.GetCommand(request_id, command_id) self.assertEqual(common.CommandState.RUNNING, queried_command.state) tasks = command_manager.GetActiveTasks(self.command) self.assertEqual(len(tasks), 1) self.assertEqual(tasks[0].attempt_index, 0) self.assertEqual(tasks[0].leasable, False) attempts = command_manager.GetCommandAttempts(request_id, command_id) self.assertEqual(1, len(attempts)) self.assertEqual(["0123456789ABCDEF"], attempts[0].device_serials) mock_notify.assert_called_with(request_id) expected_metric_fields = { metric.METRIC_FIELD_HOSTNAME: "hostname", metric.METRIC_FIELD_TYPE: "InvocationStarted" } mock_command_event_type_count.Increment.assert_called_once_with( expected_metric_fields)
def testProcessCommandEvent_fetchFailed( self, mock_notify, mock_command_event_type_count): """Should error commands from FetchFailed events.""" _, request_id, _, command_id = self.command.key.flat() command_manager.ScheduleTasks([self.command]) # It should be marked as error at the MAX_ERROR_COUNT_BASE attempt. for i in range(command_manager.MAX_ERROR_COUNT_BASE): tasks = command_manager.GetActiveTasks(self.command) self.assertEqual(len(tasks), 1) self.assertEqual(tasks[0].attempt_index, i) command_task_store.LeaseTask(tasks[0].task_id) command_event_test_util.CreateCommandAttempt( self.command, str(i), common.CommandState.UNKNOWN, task=tasks[0]) queried_command = command_manager.GetCommand(request_id, command_id) self.assertNotEqual(common.CommandState.ERROR, queried_command.state) event = command_event_test_util.CreateTestCommandEvent( request_id, command_id, str(i), "FetchFailed", task=tasks[0]) command_event_handler.ProcessCommandEvent(event) queried_command = command_manager.GetCommand(request_id, command_id) self.assertEqual(common.CommandState.ERROR, queried_command.state) tasks = command_manager.GetActiveTasks(self.command) self.assertEqual(len(tasks), 0) command_attempts = command_manager.GetCommandAttempts( request_id, command_id) self.assertEqual( len(command_attempts), command_manager.MAX_ERROR_COUNT_BASE) self.assertSetEqual( set(attempt.attempt_index for attempt in command_attempts), set(range(command_manager.MAX_ERROR_COUNT_BASE))) mock_notify.assert_called_with(request_id) expected_metric_fields = { metric.METRIC_FIELD_HOSTNAME: "hostname", metric.METRIC_FIELD_TYPE: "FetchFailed" } mock_command_event_type_count.Increment.assert_has_calls( [mock.call(expected_metric_fields) ] * command_manager.MAX_ERROR_COUNT_BASE)
def testProcessCommandEvent_invocationCompleted( self, mock_notify, mock_command_event_type_count): """Should complete command state for InvocationCompleted events. Args: mock_notify: mock function to notify request state changes. mock_command_event_type_count: mock command event type count metric """ _, request_id, _, command_id = self.command.key.flat() command_manager.ScheduleTasks([self.command]) tasks = command_manager.GetActiveTasks(self.command) self.assertEqual(len(tasks), 1) command_task_store.LeaseTask(tasks[0].task_id) command_event_test_util.CreateCommandAttempt( self.command, "0", common.CommandState.UNKNOWN, task=tasks[0]) queried_command = command_manager.GetCommand(request_id, command_id) self.assertNotEqual(common.CommandState.COMPLETED, queried_command.state) event = command_event_test_util.CreateTestCommandEvent( request_id, command_id, "0", "InvocationCompleted", task=tasks[0]) command_event_handler.ProcessCommandEvent(event) queried_command = command_manager.GetCommand(request_id, command_id) self.assertEqual(common.CommandState.COMPLETED, queried_command.state) tasks = command_manager.GetActiveTasks(self.command) self.assertEqual(len(tasks), 0) attempts = command_manager.GetCommandAttempts(request_id, command_id) self.assertEqual(1, len(attempts)) self.assertEqual("summary", attempts[0].summary) self.assertEqual(1000, attempts[0].total_test_count) self.assertEqual(100, attempts[0].failed_test_count) self.assertEqual(10, attempts[0].failed_test_run_count) mock_notify.assert_called_with(request_id) expected_metric_fields = { metric.METRIC_FIELD_HOSTNAME: "hostname", metric.METRIC_FIELD_TYPE: "InvocationCompleted" } mock_command_event_type_count.Increment.assert_called_once_with( expected_metric_fields)
def testProcessCommandEvent_fatalEvent( self, mock_notify, mock_command_event_type_count): """Should not reschedule a configuration error, request should error out. Args: mock_notify: mock function to notify request state changes. mock_command_event_type_count: mock command event type count metric """ _, request_id, _, command_id = self.command.key.flat() command_manager.ScheduleTasks([self.command]) tasks = command_manager.GetActiveTasks(self.command) self.assertEqual(len(tasks), 1) command_task_store.LeaseTask(tasks[0].task_id) command_event_test_util.CreateCommandAttempt( self.command, "0", common.CommandState.UNKNOWN, task=tasks[0]) queried_command = command_manager.GetCommand(request_id, command_id) self.assertNotEqual(common.CommandState.FATAL, queried_command.state) event = command_event_test_util.CreateTestCommandEvent( request_id, command_id, "0", "ConfigurationError", data={"error_status": "CUSTOMER_ISSUE"}, task=tasks[0]) command_event_handler.ProcessCommandEvent(event) queried_command = command_manager.GetCommand(request_id, command_id) self.assertEqual(common.CommandState.FATAL, queried_command.state) tasks = command_manager.GetActiveTasks(self.command) self.assertEqual(len(tasks), 0) attempts = command_manager.GetCommandAttempts(request_id, command_id) self.assertEqual(1, len(attempts)) mock_notify.assert_called_with(request_id) expected_metric_fields = { metric.METRIC_FIELD_HOSTNAME: "hostname", metric.METRIC_FIELD_TYPE: "ConfigurationError" } mock_command_event_type_count.Increment.assert_called_once_with( expected_metric_fields)
def testProcessCommandEvent_TooMuchContentionGetsRetried(self, mock_notify): """Should error commands from FetchFailed events.""" _, request_id, _, command_id = self.command.key.flat() command_manager.ScheduleTasks([self.command]) tasks = command_manager.GetActiveTasks(self.command) event = command_event_test_util.CreateTestCommandEvent( request_id, command_id, 123, "FetchFailed", task=tasks[0]) mock_notify.side_effect = Exception( '<_MultiThreadedRendezvous of RPC that terminated with: ' 'status = StatusCode.ABORTED details = "too much content' 'ion on these datastore entities. please try again. enti' 'ty groups: [(app=p~ee-cloud-datastore, Asset, "NOAA/GOE' 'S/17/FDCC/2020195084117700000")]" debug_error_string = ' '"{"created":"@1594629864.343436240","description":"Erro' 'r received from peer ipv6:[2607:f8b0:4001:c01::5f]:443"' ',"file":"third_party/grpc/src/core/lib/surface/call.cc"' ',"file_line":1062,"grpc_message":"too much contention o' 'n these datastore entities. please try again. entity gr' 'oups: [(app=p~ee-cloud-datastore, Asset, "NOAA/GOES/17/' 'FDCC/2020195084117700000")]","grpc_status":10}" >') with self.assertRaises(common.TooMuchContentionError): command_event_handler.ProcessCommandEvent(event) self.assertEqual(4, mock_notify.call_count)
def testProcessCommandEvent_pendingCommands(self, attempt_metric, monitor): # Test ProcessCommandEvent for a non-final state with deletion request_id = "1001" command_infos = [ datastore_entities.CommandInfo(command_line="command_line %04d" % i, cluster="cluster %04d" % i, run_target="run_target %04d" % i, run_count=1, shard_count=1) for i in range(10) ] request = datastore_test_util.CreateRequest( request_id=request_id, user="******", command_infos=command_infos, max_concurrent_tasks=5, plugin_data={ "FOO": "foo", "BAR": "'bar", }) commands = command_manager.CreateCommands(request_id=request_id, command_infos=command_infos, priority=request.priority, shard_indexes=[0] * len(command_infos)) command_manager.ScheduleTasks(commands[:5]) _, request_id, _, command_id = commands[0].key.flat() pending_commands = command_manager.GetCommands( request_id, common.CommandState.UNKNOWN) self.assertEqual(5, len(pending_commands)) queued_commands = command_manager.GetCommands( request_id, common.CommandState.QUEUED) self.assertEqual(5, len(queued_commands)) tasks = command_manager.GetActiveTasks(commands[0]) self.assertEqual(1, len(tasks)) command_task_store.LeaseTask(tasks[0].task_id) command_event_test_util.CreateCommandAttempt( commands[0], "attempt0", common.CommandState.UNKNOWN, task=tasks[0]) event = command_event_test_util.CreateTestCommandEvent( request_id, command_id, "attempt0", common.InvocationEventType.INVOCATION_COMPLETED, task=tasks[0], time=TIMESTAMP) commander.ProcessCommandEvent(event) tasks = command_manager.GetActiveTasks(commands[0]) self.assertEqual(0, len(tasks)) command = commands[0].key.get(use_cache=False) self.assertEqual(common.CommandState.COMPLETED, command.state) attempt_metric.assert_called_once_with(cluster_id=command.cluster, run_target=command.run_target, hostname="hostname", state="COMPLETED") next_command = pending_commands[0] monitor.assert_called_once_with([next_command]) next_command = pending_commands[0].key.get(use_cache=False) self.assertEqual(common.CommandState.QUEUED, next_command.state) pending_commands = command_manager.GetCommands( request_id, common.CommandState.UNKNOWN) self.assertEqual(4, len(pending_commands)) queued_commands = command_manager.GetCommands( request_id, common.CommandState.QUEUED) self.assertEqual(5, len(queued_commands)) completed_commands = command_manager.GetCommands( request_id, common.CommandState.COMPLETED) self.assertEqual(1, len(completed_commands))
def testProcessCommandEvent_notUpdatedButFinal(self, plugin, attempt_metric): command = self._CreateCommand() command_manager.ScheduleTasks([command]) _, request_id, _, command_id = command.key.flat() # Setup to ensure we are properly in the error state for i in range(command_manager.MAX_ERROR_COUNT_BASE): tasks = command_manager.GetActiveTasks(command) self.assertEqual(len(tasks), 1) command_task_store.LeaseTask(tasks[0].task_id) attempt = command_event_test_util.CreateCommandAttempt( command, str(i), common.CommandState.RUNNING, task=tasks[0]) event = command_event_test_util.CreateTestCommandEvent( request_id, command_id, str(i), common.InvocationEventType.INVOCATION_COMPLETED, error="error", task=tasks[0], time=TIMESTAMP) commander.ProcessCommandEvent(event) tasks = command_manager.GetActiveTasks(command) self.assertEqual(len(tasks), 0) command = command.key.get(use_cache=False) self.assertEqual(common.CommandState.ERROR, command.state) request = command.key.parent().get(use_cache=False) self.assertEqual(common.RequestState.ERROR, request.state) attempt_metric.reset_mock() plugin.reset_mock() # Process last event again to ensure that we don't update commander.ProcessCommandEvent(event) tasks = command_manager.GetActiveTasks(command) self.assertEqual(len(tasks), 0) command = command.key.get(use_cache=False) self.assertEqual(common.CommandState.ERROR, command.state) request = command.key.parent().get(use_cache=False) self.assertEqual(common.RequestState.ERROR, request.state) attempt_metric.assert_called_once_with(cluster_id=command.cluster, run_target=command.run_target, hostname="hostname", state="ERROR") plugin.assert_has_calls([ mock.call.OnProcessCommandEvent( command, hamcrest.match_equality( hamcrest.all_of( hamcrest.has_property("command_id", attempt.command_id), hamcrest.has_property("attempt_id", attempt.attempt_id), hamcrest.has_property("task_id", attempt.task_id), )), event_data={ "summary": "summary", "total_test_count": 1000, "failed_test_count": 100, "passed_test_count": 900, "failed_test_run_count": 10, "device_lost_detected": 0, "error": "error" }), ])