def testLeaseTask(self):
# lease works for the first time
self.assertTrue(command_task_store.LeaseTask('task_id1'))
# lease fails for the second time
self.assertFalse(command_task_store.LeaseTask('task_id1'))
task = command_task_store._Key('task_id1').get()
self.assertFalse(task.leasable)
self.assertIsNotNone(task.lease_timestamp)
def testProcessCommandEvent_nonFinal_delete(self, attempt_metric):
# Test ProcessCommandEvent for a non-final state with deletion
command = self._CreateCommand(run_count=2)
command_manager.ScheduleTasks([command])
_, request_id, _, command_id = command.key.flat()
tasks = command_manager.GetActiveTasks(command)
self.assertEqual(len(tasks), 2)
command_task_store.LeaseTask(tasks[0].task_id)
command_event_test_util.CreateCommandAttempt(
command, "attempt0", common.CommandState.UNKNOWN, task=tasks[0])
event = command_event_test_util.CreateTestCommandEvent(
request_id,
command_id,
"attempt0",
common.InvocationEventType.INVOCATION_COMPLETED,
task=tasks[0],
time=TIMESTAMP)
commander.ProcessCommandEvent(event)
tasks = command_manager.GetActiveTasks(command)
self.assertEqual(len(tasks), 1)
self.assertEqual(tasks[0].task_id,
"%s-%s-1" % (request_id, command_id))
self.assertEqual(tasks[0].leasable, True)
self.assertEqual(tasks[0].run_index, 1)
self.assertEqual(tasks[0].attempt_index, 0)
command = command.key.get(use_cache=False)
self.assertEqual(common.CommandState.QUEUED, command.state)
request = command.key.parent().get(use_cache=False)
self.assertEqual(common.RequestState.QUEUED, request.state)
attempt_metric.assert_called_once_with(cluster_id=command.cluster,
run_target=command.run_target,
hostname="hostname",
state="COMPLETED")
def testProcessCommandEvent_multiDevice(
self, mock_notify, mock_command_event_type_count):
"""Should populate multiple devices."""
_, request_id, _, command_id = self.command.key.flat()
command_manager.ScheduleTasks([self.command])
tasks = command_manager.GetActiveTasks(self.command)
self.assertEqual(len(tasks), 1)
command_task_store.LeaseTask(tasks[0].task_id)
command_event_test_util.CreateCommandAttempt(
self.command, "0", common.CommandState.UNKNOWN, task=tasks[0])
queried_command = command_manager.GetCommand(request_id, command_id)
self.assertNotEqual(common.CommandState.RUNNING, queried_command.state)
event = command_event_test_util.CreateTestCommandEvent(
request_id, command_id, "0", "InvocationStarted",
task=tasks[0], device_serials=["d1", "d2"])
command_event_handler.ProcessCommandEvent(event)
queried_command = command_manager.GetCommand(request_id, command_id)
self.assertEqual(common.CommandState.RUNNING, queried_command.state)
tasks = command_manager.GetActiveTasks(self.command)
self.assertEqual(len(tasks), 1)
self.assertEqual(tasks[0].attempt_index, 0)
self.assertEqual(tasks[0].leasable, False)
attempts = command_manager.GetCommandAttempts(request_id, command_id)
self.assertEqual(1, len(attempts))
self.assertEqual(["d1", "d2"], attempts[0].device_serials)
mock_notify.assert_called_with(request_id)
expected_metric_fields = {
metric.METRIC_FIELD_HOSTNAME: "hostname",
metric.METRIC_FIELD_TYPE: "InvocationStarted"
}
mock_command_event_type_count.Increment.assert_called_once_with(
expected_metric_fields)
def testProcessCommandEvent_ignoreOutOfDateEvent(
self, mock_notify, mock_command_event_type_count):
"""Should complete command state for InvocationCompleted events."""
_, request_id, _, command_id = self.command.key.flat()
command_manager.ScheduleTasks([self.command])
tasks = command_manager.GetActiveTasks(self.command)
command_task_store.LeaseTask(tasks[0].task_id)
command_event_test_util.CreateCommandAttempt(
self.command, "0", common.CommandState.UNKNOWN, task=tasks[0])
invocation_completed_event = command_event_test_util.CreateTestCommandEvent(
request_id, command_id, "0", "InvocationCompleted",
task=tasks[0], time=TIMESTAMP_INT + 60)
invocation_started_event = command_event_test_util.CreateTestCommandEvent(
request_id, command_id, "0", "InvocationStarted",
task=tasks[0], time=TIMESTAMP_INT + 30)
queried_command = command_manager.GetCommand(request_id, command_id)
self.assertEqual(common.CommandState.QUEUED, queried_command.state)
command_event_handler.ProcessCommandEvent(invocation_completed_event)
queried_command = command_manager.GetCommand(request_id, command_id)
command_attempt = command_manager.GetCommandAttempts(
request_id, command_id)[0]
self.assertEqual(
TIMESTAMP + datetime.timedelta(seconds=60),
command_attempt.last_event_time)
self.assertEqual(common.CommandState.COMPLETED, queried_command.state)
# The second event is ignored.
command_event_handler.ProcessCommandEvent(invocation_started_event)
queried_command = command_manager.GetCommand(request_id, command_id)
command_attempt = command_manager.GetCommandAttempts(
request_id, command_id)[0]
self.assertEqual(
TIMESTAMP + datetime.timedelta(seconds=60),
command_attempt.last_event_time)
self.assertEqual(common.CommandState.COMPLETED, queried_command.state)
tasks = command_manager.GetActiveTasks(self.command)
self.assertEqual(len(tasks), 0)
attempts = command_manager.GetCommandAttempts(
request_id, command_id)
self.assertEqual(1, len(attempts))
self.assertEqual("summary", attempts[0].summary)
self.assertEqual(1000, attempts[0].total_test_count)
self.assertEqual(100, attempts[0].failed_test_count)
mock_notify.assert_called_with(request_id)
# Metrics should still be logged for out of order events
started_metric_fields = {
metric.METRIC_FIELD_HOSTNAME: "hostname",
metric.METRIC_FIELD_TYPE: "InvocationStarted"
}
completed_metric_fields = {
metric.METRIC_FIELD_HOSTNAME: "hostname",
metric.METRIC_FIELD_TYPE: "InvocationCompleted"
}
mock_command_event_type_count.Increment.assert_has_calls(
[mock.call(completed_metric_fields),
mock.call(started_metric_fields)])
def testProcessCommandEvent_final(self, plugin, attempt_metric):
# Test ProcessCommandEvent for a final state
command = self._CreateCommand()
command_manager.ScheduleTasks([command])
_, request_id, _, command_id = command.key.flat()
tasks = command_manager.GetActiveTasks(command)
self.assertEqual(len(tasks), 1)
command_task_store.LeaseTask(tasks[0].task_id)
attempt = command_event_test_util.CreateCommandAttempt(
command, "attempt0", common.CommandState.UNKNOWN, task=tasks[0])
event = command_event_test_util.CreateTestCommandEvent(
request_id,
command_id,
"attempt0",
common.InvocationEventType.INVOCATION_COMPLETED,
task=tasks[0],
time=TIMESTAMP)
commander.ProcessCommandEvent(event)
tasks = command_manager.GetActiveTasks(command)
self.assertEqual(len(tasks), 0)
command = command.key.get(use_cache=False)
self.assertEqual(common.CommandState.COMPLETED, command.state)
request = command.key.parent().get(use_cache=False)
self.assertEqual(common.RequestState.COMPLETED, request.state)
attempt_metric.assert_called_once_with(cluster_id=command.cluster,
run_target=command.run_target,
hostname="hostname",
state="COMPLETED")
plugin.assert_has_calls([
mock.call.OnCreateCommands([
plugin_base.CommandInfo(command_id=COMMAND_ID,
command_line="command_line1",
run_count=1,
shard_count=1,
shard_index=0)
], {
"ants_invocation_id": "i123",
"command_ants_work_unit_id": "w123"
}, {}),
mock.call.OnProcessCommandEvent(
command,
hamcrest.match_equality(
hamcrest.all_of(
hamcrest.has_property("command_id",
attempt.command_id),
hamcrest.has_property("attempt_id",
attempt.attempt_id),
hamcrest.has_property("task_id", attempt.task_id),
)),
event_data={
"total_test_count": 1000,
"device_lost_detected": 0,
"failed_test_run_count": 10,
"passed_test_count": 900,
"failed_test_count": 100,
"summary": "summary"
}),
])
def testProcessCommandEvent_TFShutdown(
self, mock_notify, mock_command_event_type_count):
"""Mark command events that were terminated by TF shutdown as CANCELED."""
_, request_id, _, command_id = self.command.key.flat()
command_manager.ScheduleTasks([self.command])
error = "RunInterruptedException: Tradefed is shutting down."
for i in range(3):
tasks = command_manager.GetActiveTasks(self.command)
self.assertEqual(len(tasks), 1)
command_task_store.LeaseTask(tasks[0].task_id)
command_event_test_util.CreateCommandAttempt(
self.command, str(i), common.CommandState.UNKNOWN, task=tasks[0])
event = command_event_test_util.CreateTestCommandEvent(
request_id, command_id, str(i), "InvocationCompleted",
task=tasks[0], data={"error": error})
command_event_handler.ProcessCommandEvent(event)
# After three cancelled attempts, the command should still be queued.
queried_command = self.command.key.get()
self.assertEqual(common.CommandState.QUEUED, queried_command.state)
tasks = command_manager.GetActiveTasks(self.command)
self.assertEqual(len(tasks), 1)
attempts = command_manager.GetCommandAttempts(request_id, command_id)
self.assertEqual(3, len(attempts))
mock_notify.assert_called_with(request_id)
expected_metric_fields = {
metric.METRIC_FIELD_HOSTNAME: "hostname",
metric.METRIC_FIELD_TYPE: "InvocationCompleted"
}
mock_command_event_type_count.Increment.assert_has_calls(
[mock.call(expected_metric_fields)] * 3)
def testRescheduleTask(self):
self.assertTrue(command_task_store.LeaseTask('task_id1'))
task = command_task_store._Key('task_id1').get()
old_schedule_timestamp = task.schedule_timestamp
self.assertFalse(task.leasable)
command_task_store.RescheduleTask('task_id1', 1, 2)
task = command_task_store._Key('task_id1').get()
self.assertTrue(task.leasable)
self.assertEqual(1, task.run_index)
self.assertEqual(2, task.attempt_index)
self.assertIsNotNone(task.schedule_timestamp)
self.assertNotEqual(old_schedule_timestamp, task.schedule_timestamp)
def testProcessCommandEvent_invocationCompleted_multipleRunsWithErrors(
self, mock_notify):
"""Should complete command state for InvocationCompleted events.
This tests a case when a run count is greater than
command_manager.MAX_TASK_COUNT.
Args:
mock_notify: mock function to notify request state changes.
"""
run_count = 100
_, request_id, _, command_id = self.command.key.flat()
self.command.run_count = run_count
self.command.put()
command_manager.ScheduleTasks([self.command])
max_error_count = (
command_manager.MAX_ERROR_COUNT_BASE +
int(run_count * command_manager.MAX_ERROR_COUNT_RATIO))
for i in range(max_error_count):
tasks = command_manager.GetActiveTasks(self.command)
self.assertEqual(
len(tasks), min(run_count - i, command_manager.MAX_TASK_COUNT))
next_leasable_task = next((t for t in tasks if t.leasable), None)
command_task_store.LeaseTask(next_leasable_task.task_id)
command_event_test_util.CreateCommandAttempt(
self.command, str(i), common.CommandState.UNKNOWN,
task=next_leasable_task)
# It should be marked as error at the max error count attempts.
queried_command = command_manager.GetCommand(request_id, command_id)
self.assertNotEqual(common.CommandState.ERROR, queried_command.state)
event = command_event_test_util.CreateTestCommandEvent(
request_id, command_id, str(i), "InvocationCompleted",
task=next_leasable_task, error="error")
command_event_handler.ProcessCommandEvent(event)
queried_command = command_manager.GetCommand(request_id, command_id)
self.assertEqual(common.CommandState.ERROR, queried_command.state)
tasks = command_manager.GetActiveTasks(self.command)
self.assertEqual(len(tasks), 0)
command_attempts = command_manager.GetCommandAttempts(
request_id, command_id)
self.assertEqual(len(command_attempts), max_error_count)
run_attempt_pairs = [
(attempt.run_index, attempt.attempt_index)
for attempt in command_attempts]
self.assertEqual(len(set(run_attempt_pairs)), len(command_attempts))
mock_notify.assert_called_with(request_id)
def testProcessCommandEvent_notUpdated(self, plugin, attempt_metric):
"""Test ProcessCommandEvent with no update."""
command = self._CreateCommand()
command_manager.ScheduleTasks([command])
tasks = command_manager.GetActiveTasks(command)
self.assertEqual(len(tasks), 1)
command_task_store.LeaseTask(tasks[0].task_id)
command = command.key.get(use_cache=False)
self.assertEqual(common.CommandState.QUEUED, command.state)
request = command.key.parent().get(use_cache=False)
self.assertEqual(common.RequestState.QUEUED, request.state)
_, request_id, _, command_id = command.key.flat()
event = command_event_test_util.CreateTestCommandEvent(
request_id,
command_id,
"attempt0",
common.InvocationEventType.INVOCATION_STARTED,
time=TIMESTAMP)
commander.ProcessCommandEvent(event)
tasks = command_manager.GetActiveTasks(command)
self.assertEqual(len(tasks), 1)
self.assertEqual(tasks[0].leasable, False)
self.assertEqual(tasks[0].attempt_index, 0)
command = command.key.get(use_cache=False)
self.assertEqual(common.CommandState.QUEUED, command.state)
request = command.key.parent().get(use_cache=False)
self.assertEqual(common.RequestState.QUEUED, request.state)
attempt_metric.assert_not_called()
plugin.assert_has_calls([
mock.call.OnCreateCommands([
plugin_base.CommandInfo(command_id=COMMAND_ID,
command_line="command_line1",
run_count=1,
shard_count=1,
shard_index=0)
], {
"ants_invocation_id": "i123",
"command_ants_work_unit_id": "w123"
}, {}),
])
def testProcessCommandEvent_invocationCompleted_multipleRuns_20(
self, mock_notify):
"""Should complete command state for InvocationCompleted events.
This tests a case when a run count is greater than
command_manager.MAX_TASK_COUNT.
Args:
mock_notify: mock function to notify request state changes.
"""
run_count = 100
_, request_id, _, command_id = self.command.key.flat()
self.command.run_count = run_count
self.command.put()
command_manager.ScheduleTasks([self.command])
for i in range(run_count):
tasks = command_manager.GetActiveTasks(self.command)
self.assertEqual(
len(tasks), min(run_count - i, command_manager.MAX_TASK_COUNT))
next_leasable_task = next((t for t in tasks if t.leasable), None)
command_task_store.LeaseTask(next_leasable_task.task_id)
queried_command = command_manager.GetCommand(request_id, command_id)
self.assertNotEqual(common.CommandState.COMPLETED, queried_command.state)
command_event_test_util.CreateCommandAttempt(
self.command, str(i), common.CommandState.UNKNOWN,
task=next_leasable_task)
event = command_event_test_util.CreateTestCommandEvent(
request_id, command_id, str(i), "InvocationCompleted",
task=next_leasable_task)
command_event_handler.ProcessCommandEvent(event)
queried_command = command_manager.GetCommand(request_id, command_id)
self.assertEqual(common.CommandState.COMPLETED, queried_command.state)
tasks = command_manager.GetActiveTasks(self.command)
self.assertEqual(len(tasks), 0)
command_attempts = command_manager.GetCommandAttempts(
request_id, command_id)
self.assertEqual(len(command_attempts), run_count)
self.assertSetEqual(
set(attempt.run_index for attempt in command_attempts),
set(range(run_count)))
mock_notify.assert_called_with(request_id)
def testProcessCommandEvent_allocationFailed(self,
mock_command_event_type_count,
mock_event_legacy_count):
"""Should reschedule tasks that send AllocationFailed events."""
_, request_id, _, command_id = self.command.key.flat()
command_manager.ScheduleTasks([self.command])
# Command should stay queued even after MAX_CANCELED_COUNT_BASE different
# attempts result in an AllocationFailed error.
num_attempts = command_manager.MAX_CANCELED_COUNT_BASE
for i in range(num_attempts):
tasks = command_manager.GetActiveTasks(self.command)
self.assertEqual(len(tasks), 1)
command_task_store.LeaseTask(tasks[0].task_id)
command_event_test_util.CreateCommandAttempt(
self.command, str(i), common.CommandState.UNKNOWN, task=tasks[0])
queried_command = command_manager.GetCommand(request_id, command_id)
self.assertNotEqual(common.CommandState.ERROR, queried_command.state)
event = command_event_test_util.CreateTestCommandEvent(
request_id, command_id, str(i), "AllocationFailed", task=tasks[0])
command_event_handler.ProcessCommandEvent(event)
queried_command = command_manager.GetCommand(request_id, command_id)
self.assertEqual(common.CommandState.CANCELED, queried_command.state)
tasks = command_manager.GetActiveTasks(self.command)
self.assertEqual(len(tasks), 0)
command_attempts = command_manager.GetCommandAttempts(
request_id, command_id)
self.assertEqual(
len(command_attempts), command_manager.MAX_CANCELED_COUNT_BASE)
self.assertSetEqual(
set(attempt.attempt_index for attempt in command_attempts),
set(range(command_manager.MAX_CANCELED_COUNT_BASE)))
expected_metric_fields = {
metric.METRIC_FIELD_HOSTNAME: "hostname",
metric.METRIC_FIELD_TYPE: "AllocationFailed"
}
mock_command_event_type_count.Increment.assert_has_calls(
[mock.call(expected_metric_fields)] * num_attempts)
mock_event_legacy_count.Increment.assert_has_calls([mock.call({})] *
num_attempts)
def testProcessCommandEvent_unknownEvent(self, mock_notify):
"""Should update command state for unknown events."""
_, request_id, _, command_id = self.command.key.flat()
command_manager.ScheduleTasks([self.command])
tasks = command_manager.GetActiveTasks(self.command)
self.assertEqual(len(tasks), 1)
command_task_store.LeaseTask(tasks[0].task_id)
command_event_test_util.CreateCommandAttempt(
self.command, "0", common.CommandState.UNKNOWN, task=tasks[0])
queried_command = command_manager.GetCommand(request_id, command_id)
self.assertEqual(common.CommandState.QUEUED, queried_command.state)
event = command_event_test_util.CreateTestCommandEvent(
request_id, command_id, "0", "SomeRandomInexistentType", task=tasks[0])
command_event_handler.ProcessCommandEvent(event)
queried_command = command_manager.GetCommand(request_id, command_id)
self.assertEqual(common.CommandState.QUEUED, queried_command.state)
tasks = command_manager.GetActiveTasks(self.command)
self.assertEqual(len(tasks), 1)
mock_notify.not_called()
def testProcessCommandEvent_InvocationStarted(
self, mock_notify, mock_command_event_type_count):
"""Should update command state for InvocationStarted events.
State should become RUNNING if it isn't already.
Args:
mock_notify: mock function to notify request state changes.
mock_command_event_type_count: mock command event type count metric
"""
_, request_id, _, command_id = self.command.key.flat()
command_manager.ScheduleTasks([self.command])
tasks = command_manager.GetActiveTasks(self.command)
self.assertEqual(len(tasks), 1)
command_task_store.LeaseTask(tasks[0].task_id)
command_event_test_util.CreateCommandAttempt(
self.command, "0", common.CommandState.UNKNOWN, task=tasks[0])
queried_command = command_manager.GetCommand(request_id, command_id)
self.assertNotEqual(common.CommandState.RUNNING, queried_command.state)
event = command_event_test_util.CreateTestCommandEvent(
request_id, command_id, "0", "InvocationStarted", task=tasks[0])
command_event_handler.ProcessCommandEvent(event)
queried_command = command_manager.GetCommand(request_id, command_id)
self.assertEqual(common.CommandState.RUNNING, queried_command.state)
tasks = command_manager.GetActiveTasks(self.command)
self.assertEqual(len(tasks), 1)
self.assertEqual(tasks[0].attempt_index, 0)
self.assertEqual(tasks[0].leasable, False)
attempts = command_manager.GetCommandAttempts(request_id, command_id)
self.assertEqual(1, len(attempts))
self.assertEqual(["0123456789ABCDEF"], attempts[0].device_serials)
mock_notify.assert_called_with(request_id)
expected_metric_fields = {
metric.METRIC_FIELD_HOSTNAME: "hostname",
metric.METRIC_FIELD_TYPE: "InvocationStarted"
}
mock_command_event_type_count.Increment.assert_called_once_with(
expected_metric_fields)
def testProcessCommandEvent_invocationCompleted(
self, mock_notify, mock_command_event_type_count):
"""Should complete command state for InvocationCompleted events.
Args:
mock_notify: mock function to notify request state changes.
mock_command_event_type_count: mock command event type count metric
"""
_, request_id, _, command_id = self.command.key.flat()
command_manager.ScheduleTasks([self.command])
tasks = command_manager.GetActiveTasks(self.command)
self.assertEqual(len(tasks), 1)
command_task_store.LeaseTask(tasks[0].task_id)
command_event_test_util.CreateCommandAttempt(
self.command, "0", common.CommandState.UNKNOWN, task=tasks[0])
queried_command = command_manager.GetCommand(request_id, command_id)
self.assertNotEqual(common.CommandState.COMPLETED, queried_command.state)
event = command_event_test_util.CreateTestCommandEvent(
request_id, command_id, "0", "InvocationCompleted", task=tasks[0])
command_event_handler.ProcessCommandEvent(event)
queried_command = command_manager.GetCommand(request_id, command_id)
self.assertEqual(common.CommandState.COMPLETED, queried_command.state)
tasks = command_manager.GetActiveTasks(self.command)
self.assertEqual(len(tasks), 0)
attempts = command_manager.GetCommandAttempts(request_id, command_id)
self.assertEqual(1, len(attempts))
self.assertEqual("summary", attempts[0].summary)
self.assertEqual(1000, attempts[0].total_test_count)
self.assertEqual(100, attempts[0].failed_test_count)
self.assertEqual(10, attempts[0].failed_test_run_count)
mock_notify.assert_called_with(request_id)
expected_metric_fields = {
metric.METRIC_FIELD_HOSTNAME: "hostname",
metric.METRIC_FIELD_TYPE: "InvocationCompleted"
}
mock_command_event_type_count.Increment.assert_called_once_with(
expected_metric_fields)
def testProcessCommandEvent_fetchFailed(
self, mock_notify, mock_command_event_type_count):
"""Should error commands from FetchFailed events."""
_, request_id, _, command_id = self.command.key.flat()
command_manager.ScheduleTasks([self.command])
# It should be marked as error at the MAX_ERROR_COUNT_BASE attempt.
for i in range(command_manager.MAX_ERROR_COUNT_BASE):
tasks = command_manager.GetActiveTasks(self.command)
self.assertEqual(len(tasks), 1)
self.assertEqual(tasks[0].attempt_index, i)
command_task_store.LeaseTask(tasks[0].task_id)
command_event_test_util.CreateCommandAttempt(
self.command, str(i), common.CommandState.UNKNOWN, task=tasks[0])
queried_command = command_manager.GetCommand(request_id, command_id)
self.assertNotEqual(common.CommandState.ERROR, queried_command.state)
event = command_event_test_util.CreateTestCommandEvent(
request_id, command_id, str(i), "FetchFailed", task=tasks[0])
command_event_handler.ProcessCommandEvent(event)
queried_command = command_manager.GetCommand(request_id, command_id)
self.assertEqual(common.CommandState.ERROR, queried_command.state)
tasks = command_manager.GetActiveTasks(self.command)
self.assertEqual(len(tasks), 0)
command_attempts = command_manager.GetCommandAttempts(
request_id, command_id)
self.assertEqual(
len(command_attempts), command_manager.MAX_ERROR_COUNT_BASE)
self.assertSetEqual(
set(attempt.attempt_index for attempt in command_attempts),
set(range(command_manager.MAX_ERROR_COUNT_BASE)))
mock_notify.assert_called_with(request_id)
expected_metric_fields = {
metric.METRIC_FIELD_HOSTNAME: "hostname",
metric.METRIC_FIELD_TYPE: "FetchFailed"
}
mock_command_event_type_count.Increment.assert_has_calls(
[mock.call(expected_metric_fields)
] * command_manager.MAX_ERROR_COUNT_BASE)
def testProcessCommandEvent_fatalEvent(
self, mock_notify, mock_command_event_type_count):
"""Should not reschedule a configuration error, request should error out.
Args:
mock_notify: mock function to notify request state changes.
mock_command_event_type_count: mock command event type count metric
"""
_, request_id, _, command_id = self.command.key.flat()
command_manager.ScheduleTasks([self.command])
tasks = command_manager.GetActiveTasks(self.command)
self.assertEqual(len(tasks), 1)
command_task_store.LeaseTask(tasks[0].task_id)
command_event_test_util.CreateCommandAttempt(
self.command, "0", common.CommandState.UNKNOWN, task=tasks[0])
queried_command = command_manager.GetCommand(request_id, command_id)
self.assertNotEqual(common.CommandState.FATAL, queried_command.state)
event = command_event_test_util.CreateTestCommandEvent(
request_id, command_id, "0", "ConfigurationError",
data={"error_status": "CUSTOMER_ISSUE"},
task=tasks[0])
command_event_handler.ProcessCommandEvent(event)
queried_command = command_manager.GetCommand(request_id, command_id)
self.assertEqual(common.CommandState.FATAL, queried_command.state)
tasks = command_manager.GetActiveTasks(self.command)
self.assertEqual(len(tasks), 0)
attempts = command_manager.GetCommandAttempts(request_id, command_id)
self.assertEqual(1, len(attempts))
mock_notify.assert_called_with(request_id)
expected_metric_fields = {
metric.METRIC_FIELD_HOSTNAME: "hostname",
metric.METRIC_FIELD_TYPE: "ConfigurationError"
}
mock_command_event_type_count.Increment.assert_called_once_with(
expected_metric_fields)
def testProcessCommandEvent_notUpdatedButFinal(self, plugin,
attempt_metric):
command = self._CreateCommand()
command_manager.ScheduleTasks([command])
_, request_id, _, command_id = command.key.flat()
# Setup to ensure we are properly in the error state
for i in range(command_manager.MAX_ERROR_COUNT_BASE):
tasks = command_manager.GetActiveTasks(command)
self.assertEqual(len(tasks), 1)
command_task_store.LeaseTask(tasks[0].task_id)
attempt = command_event_test_util.CreateCommandAttempt(
command, str(i), common.CommandState.RUNNING, task=tasks[0])
event = command_event_test_util.CreateTestCommandEvent(
request_id,
command_id,
str(i),
common.InvocationEventType.INVOCATION_COMPLETED,
error="error",
task=tasks[0],
time=TIMESTAMP)
commander.ProcessCommandEvent(event)
tasks = command_manager.GetActiveTasks(command)
self.assertEqual(len(tasks), 0)
command = command.key.get(use_cache=False)
self.assertEqual(common.CommandState.ERROR, command.state)
request = command.key.parent().get(use_cache=False)
self.assertEqual(common.RequestState.ERROR, request.state)
attempt_metric.reset_mock()
plugin.reset_mock()
# Process last event again to ensure that we don't update
commander.ProcessCommandEvent(event)
tasks = command_manager.GetActiveTasks(command)
self.assertEqual(len(tasks), 0)
command = command.key.get(use_cache=False)
self.assertEqual(common.CommandState.ERROR, command.state)
request = command.key.parent().get(use_cache=False)
self.assertEqual(common.RequestState.ERROR, request.state)
attempt_metric.assert_called_once_with(cluster_id=command.cluster,
run_target=command.run_target,
hostname="hostname",
state="ERROR")
plugin.assert_has_calls([
mock.call.OnProcessCommandEvent(
command,
hamcrest.match_equality(
hamcrest.all_of(
hamcrest.has_property("command_id",
attempt.command_id),
hamcrest.has_property("attempt_id",
attempt.attempt_id),
hamcrest.has_property("task_id", attempt.task_id),
)),
event_data={
"summary": "summary",
"total_test_count": 1000,
"failed_test_count": 100,
"passed_test_count": 900,
"failed_test_run_count": 10,
"device_lost_detected": 0,
"error": "error"
}),
])
def testLeaseTask_notExistTask(self):
# lease works for the first time
self.assertFalse(command_task_store.LeaseTask('task_id_not_exist'))
def _LeaseHostTasksForCluster(
self, matcher, cluster, host, num_tasks=None):
leased_tasks = []
leasable_tasks = command_task_store.GetLeasableTasks(
cluster, matcher.GetRunTargets())
for task in leasable_tasks:
matched_devices = matcher.Match(task)
if not matched_devices:
continue
try:
# b/27136167: Touch command to prevent coordinator from cancelling
# during task lease.
command = command_manager.Touch(task.request_id, task.command_id)
# task comes from GetLeasableTasks is eventual consistent.
# LeaseTask return the strong consistent task, so the data is more
# accurate.
leased_task = command_task_store.LeaseTask(task.task_id)
if not leased_task:
continue
data_consistent = self._EnsureCommandConsistency(
task.request_id, task.command_id, task.task_id)
except grpc.RpcError as e:
# Datastore entities can only be written to once per second. If we fail
# to update the command or task, log the error, and try leasing other
# tasks.
logging.warning("Error leasing task %s: %s", task.task_id, e)
continue
if not data_consistent:
continue
matcher.RemoveDeviceGroups(matched_devices)
logging.debug("lease task %s to run on %s",
str(leased_task.task_id),
tuple(m.device_serial for m in matched_devices))
plugin_data_dict = copy.copy(leased_task.plugin_data)
plugin_data_dict[_HOSTNAME_KEY] = host.hostname
plugin_data_dict[_LAB_NAME_KEY] = host.lab_name or common.UNKNOWN_LAB_NAME
plugin_data_dict[_HOST_GROUP_KEY] = host.host_group
if leased_task.schedule_timestamp:
plugin_data_dict[_TFC_COMMAND_ATTEMPT_QUEUE_START_TIMESTAMP_KEY] = (
common.DatetimeToAntsTimestampProperty(
leased_task.schedule_timestamp))
plugin_data_dict[_TFC_COMMAND_ATTEMPT_QUEUE_END_TIMESTAMP_KEY] = (
common.DatetimeToAntsTimestampProperty(leased_task.lease_timestamp or
common.Now()))
plugin_data_ = api_messages.MapToKeyValuePairMessages(plugin_data_dict)
leased_tasks.append(
CommandTask(
request_id=leased_task.request_id,
command_id=leased_task.command_id,
task_id=leased_task.task_id,
command_line=leased_task.command_line,
request_type=leased_task.request_type,
device_serials=[match.device_serial for match in matched_devices],
shard_count=leased_task.shard_count,
shard_index=leased_task.shard_index,
run_index=leased_task.run_index,
attempt_index=leased_task.attempt_index,
plugin_data=plugin_data_))
for run_target in leased_task.run_targets:
metric.RecordCommandTimingMetric(
cluster_id=cluster,
run_target=run_target,
create_timestamp=command.create_time,
command_action=metric.CommandAction.LEASE,
count=True)
if matcher.IsEmpty():
break
if num_tasks is not None and len(leased_tasks) >= num_tasks:
break
return leased_tasks
def testProcessCommandEvent_pendingCommands(self, attempt_metric, monitor):
# Test ProcessCommandEvent for a non-final state with deletion
request_id = "1001"
command_infos = [
datastore_entities.CommandInfo(command_line="command_line %04d" %
i,
cluster="cluster %04d" % i,
run_target="run_target %04d" % i,
run_count=1,
shard_count=1) for i in range(10)
]
request = datastore_test_util.CreateRequest(
request_id=request_id,
user="******",
command_infos=command_infos,
max_concurrent_tasks=5,
plugin_data={
"FOO": "foo",
"BAR": "'bar",
})
commands = command_manager.CreateCommands(request_id=request_id,
command_infos=command_infos,
priority=request.priority,
shard_indexes=[0] *
len(command_infos))
command_manager.ScheduleTasks(commands[:5])
_, request_id, _, command_id = commands[0].key.flat()
pending_commands = command_manager.GetCommands(
request_id, common.CommandState.UNKNOWN)
self.assertEqual(5, len(pending_commands))
queued_commands = command_manager.GetCommands(
request_id, common.CommandState.QUEUED)
self.assertEqual(5, len(queued_commands))
tasks = command_manager.GetActiveTasks(commands[0])
self.assertEqual(1, len(tasks))
command_task_store.LeaseTask(tasks[0].task_id)
command_event_test_util.CreateCommandAttempt(
commands[0],
"attempt0",
common.CommandState.UNKNOWN,
task=tasks[0])
event = command_event_test_util.CreateTestCommandEvent(
request_id,
command_id,
"attempt0",
common.InvocationEventType.INVOCATION_COMPLETED,
task=tasks[0],
time=TIMESTAMP)
commander.ProcessCommandEvent(event)
tasks = command_manager.GetActiveTasks(commands[0])
self.assertEqual(0, len(tasks))
command = commands[0].key.get(use_cache=False)
self.assertEqual(common.CommandState.COMPLETED, command.state)
attempt_metric.assert_called_once_with(cluster_id=command.cluster,
run_target=command.run_target,
hostname="hostname",
state="COMPLETED")
next_command = pending_commands[0]
monitor.assert_called_once_with([next_command])
next_command = pending_commands[0].key.get(use_cache=False)
self.assertEqual(common.CommandState.QUEUED, next_command.state)
pending_commands = command_manager.GetCommands(
request_id, common.CommandState.UNKNOWN)
self.assertEqual(4, len(pending_commands))
queued_commands = command_manager.GetCommands(
request_id, common.CommandState.QUEUED)
self.assertEqual(5, len(queued_commands))
completed_commands = command_manager.GetCommands(
request_id, common.CommandState.COMPLETED)
self.assertEqual(1, len(completed_commands))
def testGetLeasableTasks_noLeasableTasks(self):
command_task_store.LeaseTask('task_id1')
command_task_store.LeaseTask('task_id2')
tasks = list(command_task_store.GetLeasableTasks(
'cluster', ['run_target1']))
self.assertEqual(0, len(tasks))