def testProcessCommandEvent_ignoreOutOfDateEvent( self, mock_notify, mock_command_event_type_count): """Should complete command state for InvocationCompleted events.""" _, request_id, _, command_id = self.command.key.flat() command_manager.ScheduleTasks([self.command]) tasks = command_manager.GetActiveTasks(self.command) command_task_store.LeaseTask(tasks[0].task_id) command_event_test_util.CreateCommandAttempt( self.command, "0", common.CommandState.UNKNOWN, task=tasks[0]) invocation_completed_event = command_event_test_util.CreateTestCommandEvent( request_id, command_id, "0", "InvocationCompleted", task=tasks[0], time=TIMESTAMP_INT + 60) invocation_started_event = command_event_test_util.CreateTestCommandEvent( request_id, command_id, "0", "InvocationStarted", task=tasks[0], time=TIMESTAMP_INT + 30) queried_command = command_manager.GetCommand(request_id, command_id) self.assertEqual(common.CommandState.QUEUED, queried_command.state) command_event_handler.ProcessCommandEvent(invocation_completed_event) queried_command = command_manager.GetCommand(request_id, command_id) command_attempt = command_manager.GetCommandAttempts( request_id, command_id)[0] self.assertEqual( TIMESTAMP + datetime.timedelta(seconds=60), command_attempt.last_event_time) self.assertEqual(common.CommandState.COMPLETED, queried_command.state) # The second event is ignored. command_event_handler.ProcessCommandEvent(invocation_started_event) queried_command = command_manager.GetCommand(request_id, command_id) command_attempt = command_manager.GetCommandAttempts( request_id, command_id)[0] self.assertEqual( TIMESTAMP + datetime.timedelta(seconds=60), command_attempt.last_event_time) self.assertEqual(common.CommandState.COMPLETED, queried_command.state) tasks = command_manager.GetActiveTasks(self.command) self.assertEqual(len(tasks), 0) attempts = command_manager.GetCommandAttempts( request_id, command_id) self.assertEqual(1, len(attempts)) self.assertEqual("summary", attempts[0].summary) self.assertEqual(1000, attempts[0].total_test_count) self.assertEqual(100, attempts[0].failed_test_count) mock_notify.assert_called_with(request_id) # Metrics should still be logged for out of order events started_metric_fields = { metric.METRIC_FIELD_HOSTNAME: "hostname", metric.METRIC_FIELD_TYPE: "InvocationStarted" } completed_metric_fields = { metric.METRIC_FIELD_HOSTNAME: "hostname", metric.METRIC_FIELD_TYPE: "InvocationCompleted" } mock_command_event_type_count.Increment.assert_has_calls( [mock.call(completed_metric_fields), mock.call(started_metric_fields)])
def testSyncCommandAttempt_nonFinalState(self, mock_update, sync, mock_now): now = datetime.datetime.utcnow() mock_now.return_value = now update_time = ( now - datetime.timedelta( minutes=command_manager.MAX_COMMAND_EVENT_DELAY_MIN - 1)) _, request_id, _, command_id = self.command.key.flat() # disable auto_now or update_time will be ignored datastore_entities.CommandAttempt.update_time._auto_now = False command_event_test_util.CreateCommandAttempt( self.command, 'attempt_id', state=common.CommandState.UNKNOWN) event = command_event_test_util.CreateTestCommandEvent( request_id, command_id, 'attempt_id', common.InvocationEventType.INVOCATION_STARTED, time=update_time) command_manager.UpdateCommandAttempt(event) attempt = command_manager.GetCommandAttempts(request_id, command_id)[0] attempt.update_time = update_time attempt.put() command_attempt_monitor.SyncCommandAttempt(request_id, command_id, 'attempt_id') mock_update.assert_not_called() sync.assert_has_calls([mock.call(attempt)])
def testSyncCommandAttempt_reset(self, mock_update): now = datetime.datetime.utcnow() update_time = ( now - datetime.timedelta( minutes=command_manager.MAX_COMMAND_EVENT_DELAY_MIN) * 2) _, request_id, _, command_id = self.command.key.flat() # disable auto_now or update_time will be ignored datastore_entities.CommandAttempt.update_time._auto_now = False command_event_test_util.CreateCommandAttempt( self.command, 'attempt_id', state=common.CommandState.UNKNOWN) event = command_event_test_util.CreateTestCommandEvent( request_id, command_id, 'attempt_id', common.InvocationEventType.INVOCATION_STARTED) command_manager.UpdateCommandAttempt(event) attempt = command_manager.GetCommandAttempts(request_id, command_id)[0] attempt.update_time = update_time attempt.put() command_attempt_monitor.SyncCommandAttempt( request_id, command_id, 'attempt_id') mock_update.assert_called_once_with( hamcrest.match_equality( hamcrest.all_of( hamcrest.has_property( 'task_id', 'task_id'), hamcrest.has_property( 'error', 'A task reset by command attempt monitor.'), hamcrest.has_property( 'attempt_state', common.CommandState.ERROR))))
def testProcessCommandEvent_multiDevice( self, mock_notify, mock_command_event_type_count): """Should populate multiple devices.""" _, request_id, _, command_id = self.command.key.flat() command_manager.ScheduleTasks([self.command]) tasks = command_manager.GetActiveTasks(self.command) self.assertEqual(len(tasks), 1) command_task_store.LeaseTask(tasks[0].task_id) command_event_test_util.CreateCommandAttempt( self.command, "0", common.CommandState.UNKNOWN, task=tasks[0]) queried_command = command_manager.GetCommand(request_id, command_id) self.assertNotEqual(common.CommandState.RUNNING, queried_command.state) event = command_event_test_util.CreateTestCommandEvent( request_id, command_id, "0", "InvocationStarted", task=tasks[0], device_serials=["d1", "d2"]) command_event_handler.ProcessCommandEvent(event) queried_command = command_manager.GetCommand(request_id, command_id) self.assertEqual(common.CommandState.RUNNING, queried_command.state) tasks = command_manager.GetActiveTasks(self.command) self.assertEqual(len(tasks), 1) self.assertEqual(tasks[0].attempt_index, 0) self.assertEqual(tasks[0].leasable, False) attempts = command_manager.GetCommandAttempts(request_id, command_id) self.assertEqual(1, len(attempts)) self.assertEqual(["d1", "d2"], attempts[0].device_serials) mock_notify.assert_called_with(request_id) expected_metric_fields = { metric.METRIC_FIELD_HOSTNAME: "hostname", metric.METRIC_FIELD_TYPE: "InvocationStarted" } mock_command_event_type_count.Increment.assert_called_once_with( expected_metric_fields)
def testInit_configurationError(self): event = command_event_test_util.CreateTestCommandEvent( REQUEST_ID, COMMAND_ID, ATTEMPT_ID, common.InvocationEventType.CONFIGURATION_ERROR, data={"error_status": "CUSTOMER_ISSUE"}) self.assertEqual("CUSTOMER_ISSUE", event.error_status) self.assertEqual(common.CommandState.FATAL, event.attempt_state)
def testInit_configurationError_hostIssue(self): event = command_event_test_util.CreateTestCommandEvent( REQUEST_ID, COMMAND_ID, ATTEMPT_ID, common.InvocationEventType.CONFIGURATION_ERROR, data={"error_status": "INFRA_FAILURE"}) self.assertEqual("INFRA_FAILURE", event.error_status) self.assertEqual(common.CommandState.ERROR, event.attempt_state)
def testInit_testRunProgress(self): event = command_event_test_util.CreateTestCommandEvent( REQUEST_ID, COMMAND_ID, ATTEMPT_ID, common.InvocationEventType.TEST_RUN_IN_PROGRESS) self.assertEqual(command_event_test_util.TIMESTAMP, event.attempt_start_time) self.assertEqual(common.CommandState.RUNNING, event.attempt_state)
def testInit_invocationEnded(self): event = command_event_test_util.CreateTestCommandEvent( REQUEST_ID, COMMAND_ID, ATTEMPT_ID, common.InvocationEventType.INVOCATION_ENDED) self.assertEqual(command_event_test_util.TIMESTAMP, event.attempt_start_time) self.assertEqual(common.CommandState.RUNNING, event.attempt_state)
def testProcessCommandEvent_nonFinal_delete(self, attempt_metric): # Test ProcessCommandEvent for a non-final state with deletion command = self._CreateCommand(run_count=2) command_manager.ScheduleTasks([command]) _, request_id, _, command_id = command.key.flat() tasks = command_manager.GetActiveTasks(command) self.assertEqual(len(tasks), 2) command_task_store.LeaseTask(tasks[0].task_id) command_event_test_util.CreateCommandAttempt( command, "attempt0", common.CommandState.UNKNOWN, task=tasks[0]) event = command_event_test_util.CreateTestCommandEvent( request_id, command_id, "attempt0", common.InvocationEventType.INVOCATION_COMPLETED, task=tasks[0], time=TIMESTAMP) commander.ProcessCommandEvent(event) tasks = command_manager.GetActiveTasks(command) self.assertEqual(len(tasks), 1) self.assertEqual(tasks[0].task_id, "%s-%s-1" % (request_id, command_id)) self.assertEqual(tasks[0].leasable, True) self.assertEqual(tasks[0].run_index, 1) self.assertEqual(tasks[0].attempt_index, 0) command = command.key.get(use_cache=False) self.assertEqual(common.CommandState.QUEUED, command.state) request = command.key.parent().get(use_cache=False) self.assertEqual(common.RequestState.QUEUED, request.state) attempt_metric.assert_called_once_with(cluster_id=command.cluster, run_target=command.run_target, hostname="hostname", state="COMPLETED")
def testProcessCommandEvent_TFShutdown( self, mock_notify, mock_command_event_type_count): """Mark command events that were terminated by TF shutdown as CANCELED.""" _, request_id, _, command_id = self.command.key.flat() command_manager.ScheduleTasks([self.command]) error = "RunInterruptedException: Tradefed is shutting down." for i in range(3): tasks = command_manager.GetActiveTasks(self.command) self.assertEqual(len(tasks), 1) command_task_store.LeaseTask(tasks[0].task_id) command_event_test_util.CreateCommandAttempt( self.command, str(i), common.CommandState.UNKNOWN, task=tasks[0]) event = command_event_test_util.CreateTestCommandEvent( request_id, command_id, str(i), "InvocationCompleted", task=tasks[0], data={"error": error}) command_event_handler.ProcessCommandEvent(event) # After three cancelled attempts, the command should still be queued. queried_command = self.command.key.get() self.assertEqual(common.CommandState.QUEUED, queried_command.state) tasks = command_manager.GetActiveTasks(self.command) self.assertEqual(len(tasks), 1) attempts = command_manager.GetCommandAttempts(request_id, command_id) self.assertEqual(3, len(attempts)) mock_notify.assert_called_with(request_id) expected_metric_fields = { metric.METRIC_FIELD_HOSTNAME: "hostname", metric.METRIC_FIELD_TYPE: "InvocationCompleted" } mock_command_event_type_count.Increment.assert_has_calls( [mock.call(expected_metric_fields)] * 3)
def testProcessCommandEvent_final(self, plugin, attempt_metric): # Test ProcessCommandEvent for a final state command = self._CreateCommand() command_manager.ScheduleTasks([command]) _, request_id, _, command_id = command.key.flat() tasks = command_manager.GetActiveTasks(command) self.assertEqual(len(tasks), 1) command_task_store.LeaseTask(tasks[0].task_id) attempt = command_event_test_util.CreateCommandAttempt( command, "attempt0", common.CommandState.UNKNOWN, task=tasks[0]) event = command_event_test_util.CreateTestCommandEvent( request_id, command_id, "attempt0", common.InvocationEventType.INVOCATION_COMPLETED, task=tasks[0], time=TIMESTAMP) commander.ProcessCommandEvent(event) tasks = command_manager.GetActiveTasks(command) self.assertEqual(len(tasks), 0) command = command.key.get(use_cache=False) self.assertEqual(common.CommandState.COMPLETED, command.state) request = command.key.parent().get(use_cache=False) self.assertEqual(common.RequestState.COMPLETED, request.state) attempt_metric.assert_called_once_with(cluster_id=command.cluster, run_target=command.run_target, hostname="hostname", state="COMPLETED") plugin.assert_has_calls([ mock.call.OnCreateCommands([ plugin_base.CommandInfo(command_id=COMMAND_ID, command_line="command_line1", run_count=1, shard_count=1, shard_index=0) ], { "ants_invocation_id": "i123", "command_ants_work_unit_id": "w123" }, {}), mock.call.OnProcessCommandEvent( command, hamcrest.match_equality( hamcrest.all_of( hamcrest.has_property("command_id", attempt.command_id), hamcrest.has_property("attempt_id", attempt.attempt_id), hamcrest.has_property("task_id", attempt.task_id), )), event_data={ "total_test_count": 1000, "device_lost_detected": 0, "failed_test_run_count": 10, "passed_test_count": 900, "failed_test_count": 100, "summary": "summary" }), ])
def testInit_tfShutDown(self, get_config): get_config.return_value = ("error_reason", "error_type") error = "RunInterruptedException: Tradefed is shutting down." event = command_event_test_util.CreateTestCommandEvent( REQUEST_ID, COMMAND_ID, ATTEMPT_ID, common.InvocationEventType.INVOCATION_COMPLETED, error=error) self.assertEqual(common.CommandState.CANCELED, event.attempt_state)
def testInit_withError(self, get_config): get_config.return_value = ("error_reason", "error_type") event = command_event_test_util.CreateTestCommandEvent( REQUEST_ID, COMMAND_ID, ATTEMPT_ID, common.InvocationEventType.INVOCATION_COMPLETED, error="error") self.assertEqual(common.CommandState.ERROR, event.attempt_state) self.assertEqual("error", event.error) self.assertEqual("error_reason", event.error_reason) self.assertEqual("error_type", event.error_type) self.assertEqual(command_event_test_util.TIMESTAMP, event.attempt_end_time)
def testInit_allocationFail(self, get_config): get_config.return_value = ("error_reason", "error_type") event = command_event_test_util.CreateTestCommandEvent( REQUEST_ID, COMMAND_ID, ATTEMPT_ID, common.InvocationEventType.ALLOCATION_FAILED) self.assertEqual(common.CommandState.CANCELED, event.attempt_state) self.assertEqual( "Device allocation failed: Device did not meet command requirements", event.error) self.assertEqual("error_reason", event.error_reason) self.assertEqual("error_type", event.error_type)
def _AddCommandEvent(request_id=REQUEST_ID, time=TIMESTAP_INT): """Helper to add a command event.""" event = command_event_test_util.CreateTestCommandEvent( request_id, 123, 'attempt0', common.InvocationEventType.INVOCATION_COMPLETED, time=time) request_sync_monitor.StoreCommandEvent(event) return event
def testLogCommandEventMetrics_noInvocationTiming( self, event_type_metric, command_timing_metric): _, request_id, _, command_id = self.command.key.flat() invocation_completed_event = command_event_test_util.CreateTestCommandEvent( request_id, command_id, "0000000", "InvocationCompleted") command_event_handler.LogCommandEventMetrics( command=self.command, event=invocation_completed_event) expected_metric_fields = { metric.METRIC_FIELD_HOSTNAME: "hostname", metric.METRIC_FIELD_TYPE: "InvocationCompleted" } event_type_metric.Increment.assert_called_once_with(expected_metric_fields) command_timing_metric.assert_not_called()
def testProcessCommandEvent_invocationCompleted_multipleRunsWithErrors( self, mock_notify): """Should complete command state for InvocationCompleted events. This tests a case when a run count is greater than command_manager.MAX_TASK_COUNT. Args: mock_notify: mock function to notify request state changes. """ run_count = 100 _, request_id, _, command_id = self.command.key.flat() self.command.run_count = run_count self.command.put() command_manager.ScheduleTasks([self.command]) max_error_count = ( command_manager.MAX_ERROR_COUNT_BASE + int(run_count * command_manager.MAX_ERROR_COUNT_RATIO)) for i in range(max_error_count): tasks = command_manager.GetActiveTasks(self.command) self.assertEqual( len(tasks), min(run_count - i, command_manager.MAX_TASK_COUNT)) next_leasable_task = next((t for t in tasks if t.leasable), None) command_task_store.LeaseTask(next_leasable_task.task_id) command_event_test_util.CreateCommandAttempt( self.command, str(i), common.CommandState.UNKNOWN, task=next_leasable_task) # It should be marked as error at the max error count attempts. queried_command = command_manager.GetCommand(request_id, command_id) self.assertNotEqual(common.CommandState.ERROR, queried_command.state) event = command_event_test_util.CreateTestCommandEvent( request_id, command_id, str(i), "InvocationCompleted", task=next_leasable_task, error="error") command_event_handler.ProcessCommandEvent(event) queried_command = command_manager.GetCommand(request_id, command_id) self.assertEqual(common.CommandState.ERROR, queried_command.state) tasks = command_manager.GetActiveTasks(self.command) self.assertEqual(len(tasks), 0) command_attempts = command_manager.GetCommandAttempts( request_id, command_id) self.assertEqual(len(command_attempts), max_error_count) run_attempt_pairs = [ (attempt.run_index, attempt.attempt_index) for attempt in command_attempts] self.assertEqual(len(set(run_attempt_pairs)), len(command_attempts)) mock_notify.assert_called_with(request_id)
def testLogCommandEventMetrics_badEvent( self, event_type_metric, command_timing_metric): _, request_id, _, command_id = self.command.key.flat() invocation_completed_event = command_event_test_util.CreateTestCommandEvent( request_id, command_id, "0000000", "InvocationCompleted", data={"fetch_build_time_millis": "abcd"}) # Not a number # ValueError should be caught and this should not crash command_event_handler.LogCommandEventMetrics( command=self.command, event=invocation_completed_event) expected_metric_fields = { metric.METRIC_FIELD_HOSTNAME: "hostname", metric.METRIC_FIELD_TYPE: "InvocationCompleted" } event_type_metric.Increment.assert_called_once_with(expected_metric_fields) command_timing_metric.assert_not_called()
def testProcessCommandEvent_withRequestSync(self, mock_store_event, mock_event_legacy_count): _, request_id, _, command_id = self.command.key.flat() sync_key = ndb.Key( datastore_entities.RequestSyncStatus, request_id, namespace=common.NAMESPACE) sync_status = datastore_entities.RequestSyncStatus( key=sync_key, request_id=request_id) sync_status.put() event = command_event_test_util.CreateTestCommandEvent( request_id, command_id, "0000000", "InvocationCompleted") command_event_handler.ProcessCommandEvent(event) mock_store_event.assert_called_once_with(event) mock_event_legacy_count.assert_not_called()
def testProcessCommandEvent_notUpdated(self, plugin, attempt_metric): """Test ProcessCommandEvent with no update.""" command = self._CreateCommand() command_manager.ScheduleTasks([command]) tasks = command_manager.GetActiveTasks(command) self.assertEqual(len(tasks), 1) command_task_store.LeaseTask(tasks[0].task_id) command = command.key.get(use_cache=False) self.assertEqual(common.CommandState.QUEUED, command.state) request = command.key.parent().get(use_cache=False) self.assertEqual(common.RequestState.QUEUED, request.state) _, request_id, _, command_id = command.key.flat() event = command_event_test_util.CreateTestCommandEvent( request_id, command_id, "attempt0", common.InvocationEventType.INVOCATION_STARTED, time=TIMESTAMP) commander.ProcessCommandEvent(event) tasks = command_manager.GetActiveTasks(command) self.assertEqual(len(tasks), 1) self.assertEqual(tasks[0].leasable, False) self.assertEqual(tasks[0].attempt_index, 0) command = command.key.get(use_cache=False) self.assertEqual(common.CommandState.QUEUED, command.state) request = command.key.parent().get(use_cache=False) self.assertEqual(common.RequestState.QUEUED, request.state) attempt_metric.assert_not_called() plugin.assert_has_calls([ mock.call.OnCreateCommands([ plugin_base.CommandInfo(command_id=COMMAND_ID, command_line="command_line1", run_count=1, shard_count=1, shard_index=0) ], { "ants_invocation_id": "i123", "command_ants_work_unit_id": "w123" }, {}), ])
def testProcessCommandEvent_invocationCompleted_multipleRuns_20( self, mock_notify): """Should complete command state for InvocationCompleted events. This tests a case when a run count is greater than command_manager.MAX_TASK_COUNT. Args: mock_notify: mock function to notify request state changes. """ run_count = 100 _, request_id, _, command_id = self.command.key.flat() self.command.run_count = run_count self.command.put() command_manager.ScheduleTasks([self.command]) for i in range(run_count): tasks = command_manager.GetActiveTasks(self.command) self.assertEqual( len(tasks), min(run_count - i, command_manager.MAX_TASK_COUNT)) next_leasable_task = next((t for t in tasks if t.leasable), None) command_task_store.LeaseTask(next_leasable_task.task_id) queried_command = command_manager.GetCommand(request_id, command_id) self.assertNotEqual(common.CommandState.COMPLETED, queried_command.state) command_event_test_util.CreateCommandAttempt( self.command, str(i), common.CommandState.UNKNOWN, task=next_leasable_task) event = command_event_test_util.CreateTestCommandEvent( request_id, command_id, str(i), "InvocationCompleted", task=next_leasable_task) command_event_handler.ProcessCommandEvent(event) queried_command = command_manager.GetCommand(request_id, command_id) self.assertEqual(common.CommandState.COMPLETED, queried_command.state) tasks = command_manager.GetActiveTasks(self.command) self.assertEqual(len(tasks), 0) command_attempts = command_manager.GetCommandAttempts( request_id, command_id) self.assertEqual(len(command_attempts), run_count) self.assertSetEqual( set(attempt.run_index for attempt in command_attempts), set(range(run_count))) mock_notify.assert_called_with(request_id)
def testProcessCommandEvent_allocationFailed(self, mock_command_event_type_count, mock_event_legacy_count): """Should reschedule tasks that send AllocationFailed events.""" _, request_id, _, command_id = self.command.key.flat() command_manager.ScheduleTasks([self.command]) # Command should stay queued even after MAX_CANCELED_COUNT_BASE different # attempts result in an AllocationFailed error. num_attempts = command_manager.MAX_CANCELED_COUNT_BASE for i in range(num_attempts): tasks = command_manager.GetActiveTasks(self.command) self.assertEqual(len(tasks), 1) command_task_store.LeaseTask(tasks[0].task_id) command_event_test_util.CreateCommandAttempt( self.command, str(i), common.CommandState.UNKNOWN, task=tasks[0]) queried_command = command_manager.GetCommand(request_id, command_id) self.assertNotEqual(common.CommandState.ERROR, queried_command.state) event = command_event_test_util.CreateTestCommandEvent( request_id, command_id, str(i), "AllocationFailed", task=tasks[0]) command_event_handler.ProcessCommandEvent(event) queried_command = command_manager.GetCommand(request_id, command_id) self.assertEqual(common.CommandState.CANCELED, queried_command.state) tasks = command_manager.GetActiveTasks(self.command) self.assertEqual(len(tasks), 0) command_attempts = command_manager.GetCommandAttempts( request_id, command_id) self.assertEqual( len(command_attempts), command_manager.MAX_CANCELED_COUNT_BASE) self.assertSetEqual( set(attempt.attempt_index for attempt in command_attempts), set(range(command_manager.MAX_CANCELED_COUNT_BASE))) expected_metric_fields = { metric.METRIC_FIELD_HOSTNAME: "hostname", metric.METRIC_FIELD_TYPE: "AllocationFailed" } mock_command_event_type_count.Increment.assert_has_calls( [mock.call(expected_metric_fields)] * num_attempts) mock_event_legacy_count.Increment.assert_has_calls([mock.call({})] * num_attempts)
def testProcessCommandEvent_unknownEvent(self, mock_notify): """Should update command state for unknown events.""" _, request_id, _, command_id = self.command.key.flat() command_manager.ScheduleTasks([self.command]) tasks = command_manager.GetActiveTasks(self.command) self.assertEqual(len(tasks), 1) command_task_store.LeaseTask(tasks[0].task_id) command_event_test_util.CreateCommandAttempt( self.command, "0", common.CommandState.UNKNOWN, task=tasks[0]) queried_command = command_manager.GetCommand(request_id, command_id) self.assertEqual(common.CommandState.QUEUED, queried_command.state) event = command_event_test_util.CreateTestCommandEvent( request_id, command_id, "0", "SomeRandomInexistentType", task=tasks[0]) command_event_handler.ProcessCommandEvent(event) queried_command = command_manager.GetCommand(request_id, command_id) self.assertEqual(common.CommandState.QUEUED, queried_command.state) tasks = command_manager.GetActiveTasks(self.command) self.assertEqual(len(tasks), 1) mock_notify.not_called()
def testInit(self): event = command_event_test_util.CreateTestCommandEvent( REQUEST_ID, COMMAND_ID, ATTEMPT_ID, common.InvocationEventType.INVOCATION_COMPLETED, invocation_status={ "test_group_statuses": [{ "name": "test1" }, { "name": "test2" }] }, device_serials=["s1", "s2"], device_lost_detected=1) self.assertEqual("%s-%s-0" % (REQUEST_ID, COMMAND_ID), event.task_id) self.assertEqual(REQUEST_ID, event.request_id) self.assertEqual(COMMAND_ID, event.command_id) self.assertEqual(ATTEMPT_ID, event.attempt_id) self.assertEqual(common.InvocationEventType.INVOCATION_COMPLETED, event.type) self.assertEqual(command_event_test_util.TIMESTAMP, event.time) self.assertEqual("hostname", event.hostname) self.assertEqual(["s1", "s2"], event.device_serials) self.assertEqual("summary", event.summary) self.assertEqual(1000, event.total_test_count) self.assertEqual(100, event.failed_test_count) self.assertEqual(900, event.passed_test_count) self.assertEqual(10, event.failed_test_run_count) self.assertEqual(1, event.device_lost_detected) self.assertEqual(command_event_test_util.TIMESTAMP, event.attempt_end_time) self.assertEqual(common.CommandState.COMPLETED, event.attempt_state) self.assertIsNone(event.error) self.assertIsNone(event.error_reason) self.assertIsNone(event.error_type) self.assertEqual( "test1", event.invocation_status.test_group_statuses[0].name) self.assertEqual( "test2", event.invocation_status.test_group_statuses[1].name)
def testProcessCommandEvent_InvocationStarted( self, mock_notify, mock_command_event_type_count): """Should update command state for InvocationStarted events. State should become RUNNING if it isn't already. Args: mock_notify: mock function to notify request state changes. mock_command_event_type_count: mock command event type count metric """ _, request_id, _, command_id = self.command.key.flat() command_manager.ScheduleTasks([self.command]) tasks = command_manager.GetActiveTasks(self.command) self.assertEqual(len(tasks), 1) command_task_store.LeaseTask(tasks[0].task_id) command_event_test_util.CreateCommandAttempt( self.command, "0", common.CommandState.UNKNOWN, task=tasks[0]) queried_command = command_manager.GetCommand(request_id, command_id) self.assertNotEqual(common.CommandState.RUNNING, queried_command.state) event = command_event_test_util.CreateTestCommandEvent( request_id, command_id, "0", "InvocationStarted", task=tasks[0]) command_event_handler.ProcessCommandEvent(event) queried_command = command_manager.GetCommand(request_id, command_id) self.assertEqual(common.CommandState.RUNNING, queried_command.state) tasks = command_manager.GetActiveTasks(self.command) self.assertEqual(len(tasks), 1) self.assertEqual(tasks[0].attempt_index, 0) self.assertEqual(tasks[0].leasable, False) attempts = command_manager.GetCommandAttempts(request_id, command_id) self.assertEqual(1, len(attempts)) self.assertEqual(["0123456789ABCDEF"], attempts[0].device_serials) mock_notify.assert_called_with(request_id) expected_metric_fields = { metric.METRIC_FIELD_HOSTNAME: "hostname", metric.METRIC_FIELD_TYPE: "InvocationStarted" } mock_command_event_type_count.Increment.assert_called_once_with( expected_metric_fields)
def testLogCommandEventMetrics_logInvocationComplete( self, event_type_metric, command_timing_metric): _, request_id, _, command_id = self.command.key.flat() self.command.start_time = TIMESTAMP self.command.put() invocation_completed_event = command_event_test_util.CreateTestCommandEvent( request_id, command_id, "0000000", "InvocationCompleted") command_event_handler.LogCommandEventMetrics( command=self.command, event=invocation_completed_event) expected_metric_fields = { metric.METRIC_FIELD_HOSTNAME: "hostname", metric.METRIC_FIELD_TYPE: "InvocationCompleted", } event_type_metric.Increment.assert_called_once_with(expected_metric_fields) command_timing_metric.assert_called_once_with( cluster_id="foobar", run_target="foo", command_action=metric.CommandAction.INVOCATION_COMPLETED, hostname="hostname", create_timestamp=TIMESTAMP)
def testLogCommandEventMetrics_withSetupTiming( self, event_type_metric, command_timing_metric): _, request_id, _, command_id = self.command.key.flat() invocation_completed_event = command_event_test_util.CreateTestCommandEvent( request_id, command_id, "0000000", "InvocationCompleted", data={"setup_time_millis": "10500"}) # 10.5 seconds command_event_handler.LogCommandEventMetrics( command=self.command, event=invocation_completed_event) expected_metric_fields = { metric.METRIC_FIELD_HOSTNAME: "hostname", metric.METRIC_FIELD_TYPE: "InvocationCompleted", metric.METRIC_FIELD_HOSTNAME: "hostname" } event_type_metric.Increment.assert_called_once_with(expected_metric_fields) command_timing_metric.assert_called_once_with( cluster_id="foobar", run_target="foo", command_action=metric.CommandAction.INVOCATION_SETUP, hostname="hostname", latency_secs=10.5)
def testProcessCommandEvent_fetchFailed( self, mock_notify, mock_command_event_type_count): """Should error commands from FetchFailed events.""" _, request_id, _, command_id = self.command.key.flat() command_manager.ScheduleTasks([self.command]) # It should be marked as error at the MAX_ERROR_COUNT_BASE attempt. for i in range(command_manager.MAX_ERROR_COUNT_BASE): tasks = command_manager.GetActiveTasks(self.command) self.assertEqual(len(tasks), 1) self.assertEqual(tasks[0].attempt_index, i) command_task_store.LeaseTask(tasks[0].task_id) command_event_test_util.CreateCommandAttempt( self.command, str(i), common.CommandState.UNKNOWN, task=tasks[0]) queried_command = command_manager.GetCommand(request_id, command_id) self.assertNotEqual(common.CommandState.ERROR, queried_command.state) event = command_event_test_util.CreateTestCommandEvent( request_id, command_id, str(i), "FetchFailed", task=tasks[0]) command_event_handler.ProcessCommandEvent(event) queried_command = command_manager.GetCommand(request_id, command_id) self.assertEqual(common.CommandState.ERROR, queried_command.state) tasks = command_manager.GetActiveTasks(self.command) self.assertEqual(len(tasks), 0) command_attempts = command_manager.GetCommandAttempts( request_id, command_id) self.assertEqual( len(command_attempts), command_manager.MAX_ERROR_COUNT_BASE) self.assertSetEqual( set(attempt.attempt_index for attempt in command_attempts), set(range(command_manager.MAX_ERROR_COUNT_BASE))) mock_notify.assert_called_with(request_id) expected_metric_fields = { metric.METRIC_FIELD_HOSTNAME: "hostname", metric.METRIC_FIELD_TYPE: "FetchFailed" } mock_command_event_type_count.Increment.assert_has_calls( [mock.call(expected_metric_fields) ] * command_manager.MAX_ERROR_COUNT_BASE)
def testProcessCommandEvent_invocationCompleted( self, mock_notify, mock_command_event_type_count): """Should complete command state for InvocationCompleted events. Args: mock_notify: mock function to notify request state changes. mock_command_event_type_count: mock command event type count metric """ _, request_id, _, command_id = self.command.key.flat() command_manager.ScheduleTasks([self.command]) tasks = command_manager.GetActiveTasks(self.command) self.assertEqual(len(tasks), 1) command_task_store.LeaseTask(tasks[0].task_id) command_event_test_util.CreateCommandAttempt( self.command, "0", common.CommandState.UNKNOWN, task=tasks[0]) queried_command = command_manager.GetCommand(request_id, command_id) self.assertNotEqual(common.CommandState.COMPLETED, queried_command.state) event = command_event_test_util.CreateTestCommandEvent( request_id, command_id, "0", "InvocationCompleted", task=tasks[0]) command_event_handler.ProcessCommandEvent(event) queried_command = command_manager.GetCommand(request_id, command_id) self.assertEqual(common.CommandState.COMPLETED, queried_command.state) tasks = command_manager.GetActiveTasks(self.command) self.assertEqual(len(tasks), 0) attempts = command_manager.GetCommandAttempts(request_id, command_id) self.assertEqual(1, len(attempts)) self.assertEqual("summary", attempts[0].summary) self.assertEqual(1000, attempts[0].total_test_count) self.assertEqual(100, attempts[0].failed_test_count) self.assertEqual(10, attempts[0].failed_test_run_count) mock_notify.assert_called_with(request_id) expected_metric_fields = { metric.METRIC_FIELD_HOSTNAME: "hostname", metric.METRIC_FIELD_TYPE: "InvocationCompleted" } mock_command_event_type_count.Increment.assert_called_once_with( expected_metric_fields)
def testProcessCommandEvent_fatalEvent( self, mock_notify, mock_command_event_type_count): """Should not reschedule a configuration error, request should error out. Args: mock_notify: mock function to notify request state changes. mock_command_event_type_count: mock command event type count metric """ _, request_id, _, command_id = self.command.key.flat() command_manager.ScheduleTasks([self.command]) tasks = command_manager.GetActiveTasks(self.command) self.assertEqual(len(tasks), 1) command_task_store.LeaseTask(tasks[0].task_id) command_event_test_util.CreateCommandAttempt( self.command, "0", common.CommandState.UNKNOWN, task=tasks[0]) queried_command = command_manager.GetCommand(request_id, command_id) self.assertNotEqual(common.CommandState.FATAL, queried_command.state) event = command_event_test_util.CreateTestCommandEvent( request_id, command_id, "0", "ConfigurationError", data={"error_status": "CUSTOMER_ISSUE"}, task=tasks[0]) command_event_handler.ProcessCommandEvent(event) queried_command = command_manager.GetCommand(request_id, command_id) self.assertEqual(common.CommandState.FATAL, queried_command.state) tasks = command_manager.GetActiveTasks(self.command) self.assertEqual(len(tasks), 0) attempts = command_manager.GetCommandAttempts(request_id, command_id) self.assertEqual(1, len(attempts)) mock_notify.assert_called_with(request_id) expected_metric_fields = { metric.METRIC_FIELD_HOSTNAME: "hostname", metric.METRIC_FIELD_TYPE: "ConfigurationError" } mock_command_event_type_count.Increment.assert_called_once_with( expected_metric_fields)