def testLeaseTask(self): # lease works for the first time self.assertTrue(command_task_store.LeaseTask('task_id1')) # lease fails for the second time self.assertFalse(command_task_store.LeaseTask('task_id1')) task = command_task_store._Key('task_id1').get() self.assertFalse(task.leasable) self.assertIsNotNone(task.lease_timestamp)
def testProcessCommandEvent_nonFinal_delete(self, attempt_metric): # Test ProcessCommandEvent for a non-final state with deletion command = self._CreateCommand(run_count=2) command_manager.ScheduleTasks([command]) _, request_id, _, command_id = command.key.flat() tasks = command_manager.GetActiveTasks(command) self.assertEqual(len(tasks), 2) command_task_store.LeaseTask(tasks[0].task_id) command_event_test_util.CreateCommandAttempt( command, "attempt0", common.CommandState.UNKNOWN, task=tasks[0]) event = command_event_test_util.CreateTestCommandEvent( request_id, command_id, "attempt0", common.InvocationEventType.INVOCATION_COMPLETED, task=tasks[0], time=TIMESTAMP) commander.ProcessCommandEvent(event) tasks = command_manager.GetActiveTasks(command) self.assertEqual(len(tasks), 1) self.assertEqual(tasks[0].task_id, "%s-%s-1" % (request_id, command_id)) self.assertEqual(tasks[0].leasable, True) self.assertEqual(tasks[0].run_index, 1) self.assertEqual(tasks[0].attempt_index, 0) command = command.key.get(use_cache=False) self.assertEqual(common.CommandState.QUEUED, command.state) request = command.key.parent().get(use_cache=False) self.assertEqual(common.RequestState.QUEUED, request.state) attempt_metric.assert_called_once_with(cluster_id=command.cluster, run_target=command.run_target, hostname="hostname", state="COMPLETED")
def testProcessCommandEvent_multiDevice( self, mock_notify, mock_command_event_type_count): """Should populate multiple devices.""" _, request_id, _, command_id = self.command.key.flat() command_manager.ScheduleTasks([self.command]) tasks = command_manager.GetActiveTasks(self.command) self.assertEqual(len(tasks), 1) command_task_store.LeaseTask(tasks[0].task_id) command_event_test_util.CreateCommandAttempt( self.command, "0", common.CommandState.UNKNOWN, task=tasks[0]) queried_command = command_manager.GetCommand(request_id, command_id) self.assertNotEqual(common.CommandState.RUNNING, queried_command.state) event = command_event_test_util.CreateTestCommandEvent( request_id, command_id, "0", "InvocationStarted", task=tasks[0], device_serials=["d1", "d2"]) command_event_handler.ProcessCommandEvent(event) queried_command = command_manager.GetCommand(request_id, command_id) self.assertEqual(common.CommandState.RUNNING, queried_command.state) tasks = command_manager.GetActiveTasks(self.command) self.assertEqual(len(tasks), 1) self.assertEqual(tasks[0].attempt_index, 0) self.assertEqual(tasks[0].leasable, False) attempts = command_manager.GetCommandAttempts(request_id, command_id) self.assertEqual(1, len(attempts)) self.assertEqual(["d1", "d2"], attempts[0].device_serials) mock_notify.assert_called_with(request_id) expected_metric_fields = { metric.METRIC_FIELD_HOSTNAME: "hostname", metric.METRIC_FIELD_TYPE: "InvocationStarted" } mock_command_event_type_count.Increment.assert_called_once_with( expected_metric_fields)
def testProcessCommandEvent_ignoreOutOfDateEvent( self, mock_notify, mock_command_event_type_count): """Should complete command state for InvocationCompleted events.""" _, request_id, _, command_id = self.command.key.flat() command_manager.ScheduleTasks([self.command]) tasks = command_manager.GetActiveTasks(self.command) command_task_store.LeaseTask(tasks[0].task_id) command_event_test_util.CreateCommandAttempt( self.command, "0", common.CommandState.UNKNOWN, task=tasks[0]) invocation_completed_event = command_event_test_util.CreateTestCommandEvent( request_id, command_id, "0", "InvocationCompleted", task=tasks[0], time=TIMESTAMP_INT + 60) invocation_started_event = command_event_test_util.CreateTestCommandEvent( request_id, command_id, "0", "InvocationStarted", task=tasks[0], time=TIMESTAMP_INT + 30) queried_command = command_manager.GetCommand(request_id, command_id) self.assertEqual(common.CommandState.QUEUED, queried_command.state) command_event_handler.ProcessCommandEvent(invocation_completed_event) queried_command = command_manager.GetCommand(request_id, command_id) command_attempt = command_manager.GetCommandAttempts( request_id, command_id)[0] self.assertEqual( TIMESTAMP + datetime.timedelta(seconds=60), command_attempt.last_event_time) self.assertEqual(common.CommandState.COMPLETED, queried_command.state) # The second event is ignored. command_event_handler.ProcessCommandEvent(invocation_started_event) queried_command = command_manager.GetCommand(request_id, command_id) command_attempt = command_manager.GetCommandAttempts( request_id, command_id)[0] self.assertEqual( TIMESTAMP + datetime.timedelta(seconds=60), command_attempt.last_event_time) self.assertEqual(common.CommandState.COMPLETED, queried_command.state) tasks = command_manager.GetActiveTasks(self.command) self.assertEqual(len(tasks), 0) attempts = command_manager.GetCommandAttempts( request_id, command_id) self.assertEqual(1, len(attempts)) self.assertEqual("summary", attempts[0].summary) self.assertEqual(1000, attempts[0].total_test_count) self.assertEqual(100, attempts[0].failed_test_count) mock_notify.assert_called_with(request_id) # Metrics should still be logged for out of order events started_metric_fields = { metric.METRIC_FIELD_HOSTNAME: "hostname", metric.METRIC_FIELD_TYPE: "InvocationStarted" } completed_metric_fields = { metric.METRIC_FIELD_HOSTNAME: "hostname", metric.METRIC_FIELD_TYPE: "InvocationCompleted" } mock_command_event_type_count.Increment.assert_has_calls( [mock.call(completed_metric_fields), mock.call(started_metric_fields)])
def testProcessCommandEvent_final(self, plugin, attempt_metric): # Test ProcessCommandEvent for a final state command = self._CreateCommand() command_manager.ScheduleTasks([command]) _, request_id, _, command_id = command.key.flat() tasks = command_manager.GetActiveTasks(command) self.assertEqual(len(tasks), 1) command_task_store.LeaseTask(tasks[0].task_id) attempt = command_event_test_util.CreateCommandAttempt( command, "attempt0", common.CommandState.UNKNOWN, task=tasks[0]) event = command_event_test_util.CreateTestCommandEvent( request_id, command_id, "attempt0", common.InvocationEventType.INVOCATION_COMPLETED, task=tasks[0], time=TIMESTAMP) commander.ProcessCommandEvent(event) tasks = command_manager.GetActiveTasks(command) self.assertEqual(len(tasks), 0) command = command.key.get(use_cache=False) self.assertEqual(common.CommandState.COMPLETED, command.state) request = command.key.parent().get(use_cache=False) self.assertEqual(common.RequestState.COMPLETED, request.state) attempt_metric.assert_called_once_with(cluster_id=command.cluster, run_target=command.run_target, hostname="hostname", state="COMPLETED") plugin.assert_has_calls([ mock.call.OnCreateCommands([ plugin_base.CommandInfo(command_id=COMMAND_ID, command_line="command_line1", run_count=1, shard_count=1, shard_index=0) ], { "ants_invocation_id": "i123", "command_ants_work_unit_id": "w123" }, {}), mock.call.OnProcessCommandEvent( command, hamcrest.match_equality( hamcrest.all_of( hamcrest.has_property("command_id", attempt.command_id), hamcrest.has_property("attempt_id", attempt.attempt_id), hamcrest.has_property("task_id", attempt.task_id), )), event_data={ "total_test_count": 1000, "device_lost_detected": 0, "failed_test_run_count": 10, "passed_test_count": 900, "failed_test_count": 100, "summary": "summary" }), ])
def testProcessCommandEvent_TFShutdown( self, mock_notify, mock_command_event_type_count): """Mark command events that were terminated by TF shutdown as CANCELED.""" _, request_id, _, command_id = self.command.key.flat() command_manager.ScheduleTasks([self.command]) error = "RunInterruptedException: Tradefed is shutting down." for i in range(3): tasks = command_manager.GetActiveTasks(self.command) self.assertEqual(len(tasks), 1) command_task_store.LeaseTask(tasks[0].task_id) command_event_test_util.CreateCommandAttempt( self.command, str(i), common.CommandState.UNKNOWN, task=tasks[0]) event = command_event_test_util.CreateTestCommandEvent( request_id, command_id, str(i), "InvocationCompleted", task=tasks[0], data={"error": error}) command_event_handler.ProcessCommandEvent(event) # After three cancelled attempts, the command should still be queued. queried_command = self.command.key.get() self.assertEqual(common.CommandState.QUEUED, queried_command.state) tasks = command_manager.GetActiveTasks(self.command) self.assertEqual(len(tasks), 1) attempts = command_manager.GetCommandAttempts(request_id, command_id) self.assertEqual(3, len(attempts)) mock_notify.assert_called_with(request_id) expected_metric_fields = { metric.METRIC_FIELD_HOSTNAME: "hostname", metric.METRIC_FIELD_TYPE: "InvocationCompleted" } mock_command_event_type_count.Increment.assert_has_calls( [mock.call(expected_metric_fields)] * 3)
def testRescheduleTask(self): self.assertTrue(command_task_store.LeaseTask('task_id1')) task = command_task_store._Key('task_id1').get() old_schedule_timestamp = task.schedule_timestamp self.assertFalse(task.leasable) command_task_store.RescheduleTask('task_id1', 1, 2) task = command_task_store._Key('task_id1').get() self.assertTrue(task.leasable) self.assertEqual(1, task.run_index) self.assertEqual(2, task.attempt_index) self.assertIsNotNone(task.schedule_timestamp) self.assertNotEqual(old_schedule_timestamp, task.schedule_timestamp)
def testProcessCommandEvent_invocationCompleted_multipleRunsWithErrors( self, mock_notify): """Should complete command state for InvocationCompleted events. This tests a case when a run count is greater than command_manager.MAX_TASK_COUNT. Args: mock_notify: mock function to notify request state changes. """ run_count = 100 _, request_id, _, command_id = self.command.key.flat() self.command.run_count = run_count self.command.put() command_manager.ScheduleTasks([self.command]) max_error_count = ( command_manager.MAX_ERROR_COUNT_BASE + int(run_count * command_manager.MAX_ERROR_COUNT_RATIO)) for i in range(max_error_count): tasks = command_manager.GetActiveTasks(self.command) self.assertEqual( len(tasks), min(run_count - i, command_manager.MAX_TASK_COUNT)) next_leasable_task = next((t for t in tasks if t.leasable), None) command_task_store.LeaseTask(next_leasable_task.task_id) command_event_test_util.CreateCommandAttempt( self.command, str(i), common.CommandState.UNKNOWN, task=next_leasable_task) # It should be marked as error at the max error count attempts. queried_command = command_manager.GetCommand(request_id, command_id) self.assertNotEqual(common.CommandState.ERROR, queried_command.state) event = command_event_test_util.CreateTestCommandEvent( request_id, command_id, str(i), "InvocationCompleted", task=next_leasable_task, error="error") command_event_handler.ProcessCommandEvent(event) queried_command = command_manager.GetCommand(request_id, command_id) self.assertEqual(common.CommandState.ERROR, queried_command.state) tasks = command_manager.GetActiveTasks(self.command) self.assertEqual(len(tasks), 0) command_attempts = command_manager.GetCommandAttempts( request_id, command_id) self.assertEqual(len(command_attempts), max_error_count) run_attempt_pairs = [ (attempt.run_index, attempt.attempt_index) for attempt in command_attempts] self.assertEqual(len(set(run_attempt_pairs)), len(command_attempts)) mock_notify.assert_called_with(request_id)
def testProcessCommandEvent_notUpdated(self, plugin, attempt_metric): """Test ProcessCommandEvent with no update.""" command = self._CreateCommand() command_manager.ScheduleTasks([command]) tasks = command_manager.GetActiveTasks(command) self.assertEqual(len(tasks), 1) command_task_store.LeaseTask(tasks[0].task_id) command = command.key.get(use_cache=False) self.assertEqual(common.CommandState.QUEUED, command.state) request = command.key.parent().get(use_cache=False) self.assertEqual(common.RequestState.QUEUED, request.state) _, request_id, _, command_id = command.key.flat() event = command_event_test_util.CreateTestCommandEvent( request_id, command_id, "attempt0", common.InvocationEventType.INVOCATION_STARTED, time=TIMESTAMP) commander.ProcessCommandEvent(event) tasks = command_manager.GetActiveTasks(command) self.assertEqual(len(tasks), 1) self.assertEqual(tasks[0].leasable, False) self.assertEqual(tasks[0].attempt_index, 0) command = command.key.get(use_cache=False) self.assertEqual(common.CommandState.QUEUED, command.state) request = command.key.parent().get(use_cache=False) self.assertEqual(common.RequestState.QUEUED, request.state) attempt_metric.assert_not_called() plugin.assert_has_calls([ mock.call.OnCreateCommands([ plugin_base.CommandInfo(command_id=COMMAND_ID, command_line="command_line1", run_count=1, shard_count=1, shard_index=0) ], { "ants_invocation_id": "i123", "command_ants_work_unit_id": "w123" }, {}), ])
def testProcessCommandEvent_invocationCompleted_multipleRuns_20( self, mock_notify): """Should complete command state for InvocationCompleted events. This tests a case when a run count is greater than command_manager.MAX_TASK_COUNT. Args: mock_notify: mock function to notify request state changes. """ run_count = 100 _, request_id, _, command_id = self.command.key.flat() self.command.run_count = run_count self.command.put() command_manager.ScheduleTasks([self.command]) for i in range(run_count): tasks = command_manager.GetActiveTasks(self.command) self.assertEqual( len(tasks), min(run_count - i, command_manager.MAX_TASK_COUNT)) next_leasable_task = next((t for t in tasks if t.leasable), None) command_task_store.LeaseTask(next_leasable_task.task_id) queried_command = command_manager.GetCommand(request_id, command_id) self.assertNotEqual(common.CommandState.COMPLETED, queried_command.state) command_event_test_util.CreateCommandAttempt( self.command, str(i), common.CommandState.UNKNOWN, task=next_leasable_task) event = command_event_test_util.CreateTestCommandEvent( request_id, command_id, str(i), "InvocationCompleted", task=next_leasable_task) command_event_handler.ProcessCommandEvent(event) queried_command = command_manager.GetCommand(request_id, command_id) self.assertEqual(common.CommandState.COMPLETED, queried_command.state) tasks = command_manager.GetActiveTasks(self.command) self.assertEqual(len(tasks), 0) command_attempts = command_manager.GetCommandAttempts( request_id, command_id) self.assertEqual(len(command_attempts), run_count) self.assertSetEqual( set(attempt.run_index for attempt in command_attempts), set(range(run_count))) mock_notify.assert_called_with(request_id)
def testProcessCommandEvent_allocationFailed(self, mock_command_event_type_count, mock_event_legacy_count): """Should reschedule tasks that send AllocationFailed events.""" _, request_id, _, command_id = self.command.key.flat() command_manager.ScheduleTasks([self.command]) # Command should stay queued even after MAX_CANCELED_COUNT_BASE different # attempts result in an AllocationFailed error. num_attempts = command_manager.MAX_CANCELED_COUNT_BASE for i in range(num_attempts): tasks = command_manager.GetActiveTasks(self.command) self.assertEqual(len(tasks), 1) command_task_store.LeaseTask(tasks[0].task_id) command_event_test_util.CreateCommandAttempt( self.command, str(i), common.CommandState.UNKNOWN, task=tasks[0]) queried_command = command_manager.GetCommand(request_id, command_id) self.assertNotEqual(common.CommandState.ERROR, queried_command.state) event = command_event_test_util.CreateTestCommandEvent( request_id, command_id, str(i), "AllocationFailed", task=tasks[0]) command_event_handler.ProcessCommandEvent(event) queried_command = command_manager.GetCommand(request_id, command_id) self.assertEqual(common.CommandState.CANCELED, queried_command.state) tasks = command_manager.GetActiveTasks(self.command) self.assertEqual(len(tasks), 0) command_attempts = command_manager.GetCommandAttempts( request_id, command_id) self.assertEqual( len(command_attempts), command_manager.MAX_CANCELED_COUNT_BASE) self.assertSetEqual( set(attempt.attempt_index for attempt in command_attempts), set(range(command_manager.MAX_CANCELED_COUNT_BASE))) expected_metric_fields = { metric.METRIC_FIELD_HOSTNAME: "hostname", metric.METRIC_FIELD_TYPE: "AllocationFailed" } mock_command_event_type_count.Increment.assert_has_calls( [mock.call(expected_metric_fields)] * num_attempts) mock_event_legacy_count.Increment.assert_has_calls([mock.call({})] * num_attempts)
def testProcessCommandEvent_unknownEvent(self, mock_notify): """Should update command state for unknown events.""" _, request_id, _, command_id = self.command.key.flat() command_manager.ScheduleTasks([self.command]) tasks = command_manager.GetActiveTasks(self.command) self.assertEqual(len(tasks), 1) command_task_store.LeaseTask(tasks[0].task_id) command_event_test_util.CreateCommandAttempt( self.command, "0", common.CommandState.UNKNOWN, task=tasks[0]) queried_command = command_manager.GetCommand(request_id, command_id) self.assertEqual(common.CommandState.QUEUED, queried_command.state) event = command_event_test_util.CreateTestCommandEvent( request_id, command_id, "0", "SomeRandomInexistentType", task=tasks[0]) command_event_handler.ProcessCommandEvent(event) queried_command = command_manager.GetCommand(request_id, command_id) self.assertEqual(common.CommandState.QUEUED, queried_command.state) tasks = command_manager.GetActiveTasks(self.command) self.assertEqual(len(tasks), 1) mock_notify.not_called()
def testProcessCommandEvent_InvocationStarted( self, mock_notify, mock_command_event_type_count): """Should update command state for InvocationStarted events. State should become RUNNING if it isn't already. Args: mock_notify: mock function to notify request state changes. mock_command_event_type_count: mock command event type count metric """ _, request_id, _, command_id = self.command.key.flat() command_manager.ScheduleTasks([self.command]) tasks = command_manager.GetActiveTasks(self.command) self.assertEqual(len(tasks), 1) command_task_store.LeaseTask(tasks[0].task_id) command_event_test_util.CreateCommandAttempt( self.command, "0", common.CommandState.UNKNOWN, task=tasks[0]) queried_command = command_manager.GetCommand(request_id, command_id) self.assertNotEqual(common.CommandState.RUNNING, queried_command.state) event = command_event_test_util.CreateTestCommandEvent( request_id, command_id, "0", "InvocationStarted", task=tasks[0]) command_event_handler.ProcessCommandEvent(event) queried_command = command_manager.GetCommand(request_id, command_id) self.assertEqual(common.CommandState.RUNNING, queried_command.state) tasks = command_manager.GetActiveTasks(self.command) self.assertEqual(len(tasks), 1) self.assertEqual(tasks[0].attempt_index, 0) self.assertEqual(tasks[0].leasable, False) attempts = command_manager.GetCommandAttempts(request_id, command_id) self.assertEqual(1, len(attempts)) self.assertEqual(["0123456789ABCDEF"], attempts[0].device_serials) mock_notify.assert_called_with(request_id) expected_metric_fields = { metric.METRIC_FIELD_HOSTNAME: "hostname", metric.METRIC_FIELD_TYPE: "InvocationStarted" } mock_command_event_type_count.Increment.assert_called_once_with( expected_metric_fields)
def testProcessCommandEvent_invocationCompleted( self, mock_notify, mock_command_event_type_count): """Should complete command state for InvocationCompleted events. Args: mock_notify: mock function to notify request state changes. mock_command_event_type_count: mock command event type count metric """ _, request_id, _, command_id = self.command.key.flat() command_manager.ScheduleTasks([self.command]) tasks = command_manager.GetActiveTasks(self.command) self.assertEqual(len(tasks), 1) command_task_store.LeaseTask(tasks[0].task_id) command_event_test_util.CreateCommandAttempt( self.command, "0", common.CommandState.UNKNOWN, task=tasks[0]) queried_command = command_manager.GetCommand(request_id, command_id) self.assertNotEqual(common.CommandState.COMPLETED, queried_command.state) event = command_event_test_util.CreateTestCommandEvent( request_id, command_id, "0", "InvocationCompleted", task=tasks[0]) command_event_handler.ProcessCommandEvent(event) queried_command = command_manager.GetCommand(request_id, command_id) self.assertEqual(common.CommandState.COMPLETED, queried_command.state) tasks = command_manager.GetActiveTasks(self.command) self.assertEqual(len(tasks), 0) attempts = command_manager.GetCommandAttempts(request_id, command_id) self.assertEqual(1, len(attempts)) self.assertEqual("summary", attempts[0].summary) self.assertEqual(1000, attempts[0].total_test_count) self.assertEqual(100, attempts[0].failed_test_count) self.assertEqual(10, attempts[0].failed_test_run_count) mock_notify.assert_called_with(request_id) expected_metric_fields = { metric.METRIC_FIELD_HOSTNAME: "hostname", metric.METRIC_FIELD_TYPE: "InvocationCompleted" } mock_command_event_type_count.Increment.assert_called_once_with( expected_metric_fields)
def testProcessCommandEvent_fetchFailed( self, mock_notify, mock_command_event_type_count): """Should error commands from FetchFailed events.""" _, request_id, _, command_id = self.command.key.flat() command_manager.ScheduleTasks([self.command]) # It should be marked as error at the MAX_ERROR_COUNT_BASE attempt. for i in range(command_manager.MAX_ERROR_COUNT_BASE): tasks = command_manager.GetActiveTasks(self.command) self.assertEqual(len(tasks), 1) self.assertEqual(tasks[0].attempt_index, i) command_task_store.LeaseTask(tasks[0].task_id) command_event_test_util.CreateCommandAttempt( self.command, str(i), common.CommandState.UNKNOWN, task=tasks[0]) queried_command = command_manager.GetCommand(request_id, command_id) self.assertNotEqual(common.CommandState.ERROR, queried_command.state) event = command_event_test_util.CreateTestCommandEvent( request_id, command_id, str(i), "FetchFailed", task=tasks[0]) command_event_handler.ProcessCommandEvent(event) queried_command = command_manager.GetCommand(request_id, command_id) self.assertEqual(common.CommandState.ERROR, queried_command.state) tasks = command_manager.GetActiveTasks(self.command) self.assertEqual(len(tasks), 0) command_attempts = command_manager.GetCommandAttempts( request_id, command_id) self.assertEqual( len(command_attempts), command_manager.MAX_ERROR_COUNT_BASE) self.assertSetEqual( set(attempt.attempt_index for attempt in command_attempts), set(range(command_manager.MAX_ERROR_COUNT_BASE))) mock_notify.assert_called_with(request_id) expected_metric_fields = { metric.METRIC_FIELD_HOSTNAME: "hostname", metric.METRIC_FIELD_TYPE: "FetchFailed" } mock_command_event_type_count.Increment.assert_has_calls( [mock.call(expected_metric_fields) ] * command_manager.MAX_ERROR_COUNT_BASE)
def testProcessCommandEvent_fatalEvent( self, mock_notify, mock_command_event_type_count): """Should not reschedule a configuration error, request should error out. Args: mock_notify: mock function to notify request state changes. mock_command_event_type_count: mock command event type count metric """ _, request_id, _, command_id = self.command.key.flat() command_manager.ScheduleTasks([self.command]) tasks = command_manager.GetActiveTasks(self.command) self.assertEqual(len(tasks), 1) command_task_store.LeaseTask(tasks[0].task_id) command_event_test_util.CreateCommandAttempt( self.command, "0", common.CommandState.UNKNOWN, task=tasks[0]) queried_command = command_manager.GetCommand(request_id, command_id) self.assertNotEqual(common.CommandState.FATAL, queried_command.state) event = command_event_test_util.CreateTestCommandEvent( request_id, command_id, "0", "ConfigurationError", data={"error_status": "CUSTOMER_ISSUE"}, task=tasks[0]) command_event_handler.ProcessCommandEvent(event) queried_command = command_manager.GetCommand(request_id, command_id) self.assertEqual(common.CommandState.FATAL, queried_command.state) tasks = command_manager.GetActiveTasks(self.command) self.assertEqual(len(tasks), 0) attempts = command_manager.GetCommandAttempts(request_id, command_id) self.assertEqual(1, len(attempts)) mock_notify.assert_called_with(request_id) expected_metric_fields = { metric.METRIC_FIELD_HOSTNAME: "hostname", metric.METRIC_FIELD_TYPE: "ConfigurationError" } mock_command_event_type_count.Increment.assert_called_once_with( expected_metric_fields)
def testProcessCommandEvent_notUpdatedButFinal(self, plugin, attempt_metric): command = self._CreateCommand() command_manager.ScheduleTasks([command]) _, request_id, _, command_id = command.key.flat() # Setup to ensure we are properly in the error state for i in range(command_manager.MAX_ERROR_COUNT_BASE): tasks = command_manager.GetActiveTasks(command) self.assertEqual(len(tasks), 1) command_task_store.LeaseTask(tasks[0].task_id) attempt = command_event_test_util.CreateCommandAttempt( command, str(i), common.CommandState.RUNNING, task=tasks[0]) event = command_event_test_util.CreateTestCommandEvent( request_id, command_id, str(i), common.InvocationEventType.INVOCATION_COMPLETED, error="error", task=tasks[0], time=TIMESTAMP) commander.ProcessCommandEvent(event) tasks = command_manager.GetActiveTasks(command) self.assertEqual(len(tasks), 0) command = command.key.get(use_cache=False) self.assertEqual(common.CommandState.ERROR, command.state) request = command.key.parent().get(use_cache=False) self.assertEqual(common.RequestState.ERROR, request.state) attempt_metric.reset_mock() plugin.reset_mock() # Process last event again to ensure that we don't update commander.ProcessCommandEvent(event) tasks = command_manager.GetActiveTasks(command) self.assertEqual(len(tasks), 0) command = command.key.get(use_cache=False) self.assertEqual(common.CommandState.ERROR, command.state) request = command.key.parent().get(use_cache=False) self.assertEqual(common.RequestState.ERROR, request.state) attempt_metric.assert_called_once_with(cluster_id=command.cluster, run_target=command.run_target, hostname="hostname", state="ERROR") plugin.assert_has_calls([ mock.call.OnProcessCommandEvent( command, hamcrest.match_equality( hamcrest.all_of( hamcrest.has_property("command_id", attempt.command_id), hamcrest.has_property("attempt_id", attempt.attempt_id), hamcrest.has_property("task_id", attempt.task_id), )), event_data={ "summary": "summary", "total_test_count": 1000, "failed_test_count": 100, "passed_test_count": 900, "failed_test_run_count": 10, "device_lost_detected": 0, "error": "error" }), ])
def testLeaseTask_notExistTask(self): # lease works for the first time self.assertFalse(command_task_store.LeaseTask('task_id_not_exist'))
def _LeaseHostTasksForCluster( self, matcher, cluster, host, num_tasks=None): leased_tasks = [] leasable_tasks = command_task_store.GetLeasableTasks( cluster, matcher.GetRunTargets()) for task in leasable_tasks: matched_devices = matcher.Match(task) if not matched_devices: continue try: # b/27136167: Touch command to prevent coordinator from cancelling # during task lease. command = command_manager.Touch(task.request_id, task.command_id) # task comes from GetLeasableTasks is eventual consistent. # LeaseTask return the strong consistent task, so the data is more # accurate. leased_task = command_task_store.LeaseTask(task.task_id) if not leased_task: continue data_consistent = self._EnsureCommandConsistency( task.request_id, task.command_id, task.task_id) except grpc.RpcError as e: # Datastore entities can only be written to once per second. If we fail # to update the command or task, log the error, and try leasing other # tasks. logging.warning("Error leasing task %s: %s", task.task_id, e) continue if not data_consistent: continue matcher.RemoveDeviceGroups(matched_devices) logging.debug("lease task %s to run on %s", str(leased_task.task_id), tuple(m.device_serial for m in matched_devices)) plugin_data_dict = copy.copy(leased_task.plugin_data) plugin_data_dict[_HOSTNAME_KEY] = host.hostname plugin_data_dict[_LAB_NAME_KEY] = host.lab_name or common.UNKNOWN_LAB_NAME plugin_data_dict[_HOST_GROUP_KEY] = host.host_group if leased_task.schedule_timestamp: plugin_data_dict[_TFC_COMMAND_ATTEMPT_QUEUE_START_TIMESTAMP_KEY] = ( common.DatetimeToAntsTimestampProperty( leased_task.schedule_timestamp)) plugin_data_dict[_TFC_COMMAND_ATTEMPT_QUEUE_END_TIMESTAMP_KEY] = ( common.DatetimeToAntsTimestampProperty(leased_task.lease_timestamp or common.Now())) plugin_data_ = api_messages.MapToKeyValuePairMessages(plugin_data_dict) leased_tasks.append( CommandTask( request_id=leased_task.request_id, command_id=leased_task.command_id, task_id=leased_task.task_id, command_line=leased_task.command_line, request_type=leased_task.request_type, device_serials=[match.device_serial for match in matched_devices], shard_count=leased_task.shard_count, shard_index=leased_task.shard_index, run_index=leased_task.run_index, attempt_index=leased_task.attempt_index, plugin_data=plugin_data_)) for run_target in leased_task.run_targets: metric.RecordCommandTimingMetric( cluster_id=cluster, run_target=run_target, create_timestamp=command.create_time, command_action=metric.CommandAction.LEASE, count=True) if matcher.IsEmpty(): break if num_tasks is not None and len(leased_tasks) >= num_tasks: break return leased_tasks
def testProcessCommandEvent_pendingCommands(self, attempt_metric, monitor): # Test ProcessCommandEvent for a non-final state with deletion request_id = "1001" command_infos = [ datastore_entities.CommandInfo(command_line="command_line %04d" % i, cluster="cluster %04d" % i, run_target="run_target %04d" % i, run_count=1, shard_count=1) for i in range(10) ] request = datastore_test_util.CreateRequest( request_id=request_id, user="******", command_infos=command_infos, max_concurrent_tasks=5, plugin_data={ "FOO": "foo", "BAR": "'bar", }) commands = command_manager.CreateCommands(request_id=request_id, command_infos=command_infos, priority=request.priority, shard_indexes=[0] * len(command_infos)) command_manager.ScheduleTasks(commands[:5]) _, request_id, _, command_id = commands[0].key.flat() pending_commands = command_manager.GetCommands( request_id, common.CommandState.UNKNOWN) self.assertEqual(5, len(pending_commands)) queued_commands = command_manager.GetCommands( request_id, common.CommandState.QUEUED) self.assertEqual(5, len(queued_commands)) tasks = command_manager.GetActiveTasks(commands[0]) self.assertEqual(1, len(tasks)) command_task_store.LeaseTask(tasks[0].task_id) command_event_test_util.CreateCommandAttempt( commands[0], "attempt0", common.CommandState.UNKNOWN, task=tasks[0]) event = command_event_test_util.CreateTestCommandEvent( request_id, command_id, "attempt0", common.InvocationEventType.INVOCATION_COMPLETED, task=tasks[0], time=TIMESTAMP) commander.ProcessCommandEvent(event) tasks = command_manager.GetActiveTasks(commands[0]) self.assertEqual(0, len(tasks)) command = commands[0].key.get(use_cache=False) self.assertEqual(common.CommandState.COMPLETED, command.state) attempt_metric.assert_called_once_with(cluster_id=command.cluster, run_target=command.run_target, hostname="hostname", state="COMPLETED") next_command = pending_commands[0] monitor.assert_called_once_with([next_command]) next_command = pending_commands[0].key.get(use_cache=False) self.assertEqual(common.CommandState.QUEUED, next_command.state) pending_commands = command_manager.GetCommands( request_id, common.CommandState.UNKNOWN) self.assertEqual(4, len(pending_commands)) queued_commands = command_manager.GetCommands( request_id, common.CommandState.QUEUED) self.assertEqual(5, len(queued_commands)) completed_commands = command_manager.GetCommands( request_id, common.CommandState.COMPLETED) self.assertEqual(1, len(completed_commands))
def testGetLeasableTasks_noLeasableTasks(self): command_task_store.LeaseTask('task_id1') command_task_store.LeaseTask('task_id2') tasks = list(command_task_store.GetLeasableTasks( 'cluster', ['run_target1'])) self.assertEqual(0, len(tasks))