def testProcessCommandEvent_ignoreOutOfDateEvent(
      self, mock_notify, mock_command_event_type_count):
    """Should complete command state for InvocationCompleted events."""
    _, request_id, _, command_id = self.command.key.flat()
    command_manager.ScheduleTasks([self.command])

    tasks = command_manager.GetActiveTasks(self.command)
    command_task_store.LeaseTask(tasks[0].task_id)
    command_event_test_util.CreateCommandAttempt(
        self.command, "0", common.CommandState.UNKNOWN, task=tasks[0])
    invocation_completed_event = command_event_test_util.CreateTestCommandEvent(
        request_id, command_id, "0", "InvocationCompleted",
        task=tasks[0], time=TIMESTAMP_INT + 60)
    invocation_started_event = command_event_test_util.CreateTestCommandEvent(
        request_id, command_id, "0", "InvocationStarted",
        task=tasks[0], time=TIMESTAMP_INT + 30)

    queried_command = command_manager.GetCommand(request_id, command_id)
    self.assertEqual(common.CommandState.QUEUED, queried_command.state)

    command_event_handler.ProcessCommandEvent(invocation_completed_event)
    queried_command = command_manager.GetCommand(request_id, command_id)
    command_attempt = command_manager.GetCommandAttempts(
        request_id, command_id)[0]
    self.assertEqual(
        TIMESTAMP + datetime.timedelta(seconds=60),
        command_attempt.last_event_time)
    self.assertEqual(common.CommandState.COMPLETED, queried_command.state)

    # The second event is ignored.
    command_event_handler.ProcessCommandEvent(invocation_started_event)
    queried_command = command_manager.GetCommand(request_id, command_id)
    command_attempt = command_manager.GetCommandAttempts(
        request_id, command_id)[0]
    self.assertEqual(
        TIMESTAMP + datetime.timedelta(seconds=60),
        command_attempt.last_event_time)
    self.assertEqual(common.CommandState.COMPLETED, queried_command.state)

    tasks = command_manager.GetActiveTasks(self.command)
    self.assertEqual(len(tasks), 0)
    attempts = command_manager.GetCommandAttempts(
        request_id, command_id)
    self.assertEqual(1, len(attempts))
    self.assertEqual("summary", attempts[0].summary)
    self.assertEqual(1000, attempts[0].total_test_count)
    self.assertEqual(100, attempts[0].failed_test_count)
    mock_notify.assert_called_with(request_id)
    # Metrics should still be logged for out of order events
    started_metric_fields = {
        metric.METRIC_FIELD_HOSTNAME: "hostname",
        metric.METRIC_FIELD_TYPE: "InvocationStarted"
    }
    completed_metric_fields = {
        metric.METRIC_FIELD_HOSTNAME: "hostname",
        metric.METRIC_FIELD_TYPE: "InvocationCompleted"
    }
    mock_command_event_type_count.Increment.assert_has_calls(
        [mock.call(completed_metric_fields),
         mock.call(started_metric_fields)])
  def testEnqueueCommandEvents_multipleEvents(self):
    self.request = request_manager.CreateRequest(
        request_id="9999",
        user="******",
        command_infos=[
            datastore_entities.CommandInfo(
                command_line="command_line",
                cluster="cluster",
                run_target="run_target",
                shard_count=2)
        ])
    command_1, command_2 = command_manager.CreateCommands(
        request_id=self.request.key.id(),
        command_infos=[
            datastore_entities.CommandInfo(
                command_line="long command line 0",
                cluster="foobar",
                run_target="foo",
                run_count=1,
                shard_count=2),
            datastore_entities.CommandInfo(
                command_line="long command line 1",
                cluster="foobar",
                run_target="foo",
                run_count=1,
                shard_count=2)
        ],
        shard_indexes=list(range(2)))
    _, request_id, _, command_1_id = command_1.key.flat()
    _, _, _, command_2_id = command_2.key.flat()
    command_event_test_util.CreateCommandAttempt(
        command_1, "aid", common.CommandState.QUEUED)
    command_event_test_util.CreateCommandAttempt(
        command_2, "aid", common.CommandState.QUEUED)

    event = command_event_test_util.CreateTestCommandEventJson(
        request_id, command_1_id, "aid", "InvocationStarted")
    event2 = command_event_test_util.CreateTestCommandEventJson(
        request_id, command_2_id, "aid", "InvocationStarted")
    event3 = command_event_test_util.CreateTestCommandEventJson(
        request_id, command_1_id, "aid", "InvocationCompleted")
    event4 = command_event_test_util.CreateTestCommandEventJson(
        request_id, command_2_id, "aid", "InvocationCompleted")
    command_event_handler.EnqueueCommandEvents([event, event2, event3, event4])

    tasks = self.mock_task_scheduler.GetTasks()
    self.assertEqual(len(tasks), 4)
    for task in tasks:
      self.testapp.post(
          command_event_handler.COMMAND_EVENT_HANDLER_PATH, task.payload)

    command_attempts = command_manager.GetCommandAttempts(
        request_id, command_1_id)
    self.assertEqual(len(command_attempts), 1)
    self.assertEqual(common.CommandState.COMPLETED, command_attempts[0].state)
    command_attempts = command_manager.GetCommandAttempts(
        request_id, command_2_id)
    self.assertEqual(len(command_attempts), 1)
    self.assertEqual(common.CommandState.COMPLETED, command_attempts[0].state)
Exemple #3
0
  def testSyncCommandAttempt_reset(self, mock_update):
    now = datetime.datetime.utcnow()
    update_time = (
        now - datetime.timedelta(
            minutes=command_manager.MAX_COMMAND_EVENT_DELAY_MIN) * 2)
    _, request_id, _, command_id = self.command.key.flat()
    # disable auto_now or update_time will be ignored
    datastore_entities.CommandAttempt.update_time._auto_now = False
    command_event_test_util.CreateCommandAttempt(
        self.command, 'attempt_id', state=common.CommandState.UNKNOWN)

    event = command_event_test_util.CreateTestCommandEvent(
        request_id,
        command_id,
        'attempt_id',
        common.InvocationEventType.INVOCATION_STARTED)
    command_manager.UpdateCommandAttempt(event)
    attempt = command_manager.GetCommandAttempts(request_id, command_id)[0]
    attempt.update_time = update_time
    attempt.put()
    command_attempt_monitor.SyncCommandAttempt(
        request_id, command_id, 'attempt_id')
    mock_update.assert_called_once_with(
        hamcrest.match_equality(
            hamcrest.all_of(
                hamcrest.has_property(
                    'task_id',
                    'task_id'),
                hamcrest.has_property(
                    'error',
                    'A task reset by command attempt monitor.'),
                hamcrest.has_property(
                    'attempt_state', common.CommandState.ERROR))))
Exemple #4
0
 def testSyncCommandAttempt_nonFinalState(self, mock_update, sync, mock_now):
   now = datetime.datetime.utcnow()
   mock_now.return_value = now
   update_time = (
       now - datetime.timedelta(
           minutes=command_manager.MAX_COMMAND_EVENT_DELAY_MIN - 1))
   _, request_id, _, command_id = self.command.key.flat()
   # disable auto_now or update_time will be ignored
   datastore_entities.CommandAttempt.update_time._auto_now = False
   command_event_test_util.CreateCommandAttempt(
       self.command, 'attempt_id', state=common.CommandState.UNKNOWN)
   event = command_event_test_util.CreateTestCommandEvent(
       request_id,
       command_id,
       'attempt_id',
       common.InvocationEventType.INVOCATION_STARTED,
       time=update_time)
   command_manager.UpdateCommandAttempt(event)
   attempt = command_manager.GetCommandAttempts(request_id, command_id)[0]
   attempt.update_time = update_time
   attempt.put()
   command_attempt_monitor.SyncCommandAttempt(request_id, command_id,
                                              'attempt_id')
   mock_update.assert_not_called()
   sync.assert_has_calls([mock.call(attempt)])
  def testProcessCommandEvent_multiDevice(
      self, mock_notify, mock_command_event_type_count):
    """Should populate multiple devices."""
    _, request_id, _, command_id = self.command.key.flat()
    command_manager.ScheduleTasks([self.command])

    tasks = command_manager.GetActiveTasks(self.command)
    self.assertEqual(len(tasks), 1)
    command_task_store.LeaseTask(tasks[0].task_id)
    command_event_test_util.CreateCommandAttempt(
        self.command, "0", common.CommandState.UNKNOWN, task=tasks[0])
    queried_command = command_manager.GetCommand(request_id, command_id)
    self.assertNotEqual(common.CommandState.RUNNING, queried_command.state)
    event = command_event_test_util.CreateTestCommandEvent(
        request_id, command_id, "0", "InvocationStarted",
        task=tasks[0], device_serials=["d1", "d2"])
    command_event_handler.ProcessCommandEvent(event)

    queried_command = command_manager.GetCommand(request_id, command_id)
    self.assertEqual(common.CommandState.RUNNING, queried_command.state)
    tasks = command_manager.GetActiveTasks(self.command)
    self.assertEqual(len(tasks), 1)
    self.assertEqual(tasks[0].attempt_index, 0)
    self.assertEqual(tasks[0].leasable, False)
    attempts = command_manager.GetCommandAttempts(request_id, command_id)
    self.assertEqual(1, len(attempts))
    self.assertEqual(["d1", "d2"], attempts[0].device_serials)
    mock_notify.assert_called_with(request_id)
    expected_metric_fields = {
        metric.METRIC_FIELD_HOSTNAME: "hostname",
        metric.METRIC_FIELD_TYPE: "InvocationStarted"
    }
    mock_command_event_type_count.Increment.assert_called_once_with(
        expected_metric_fields)
  def testEnqueueCommandEvents_malformedEvents(self):
    """A malformed event should not lose all events."""
    _, request_id, _, command_id = self.command.key.flat()
    command_event_test_util.CreateCommandAttempt(
        self.command, "aid", common.CommandState.QUEUED)
    event = command_event_test_util.CreateTestCommandEventJson(
        request_id, command_id, "aid", "InvocationStarted")
    malformed_event = {
        "data": {"total_test_count": 1, "exec_test_count": 1},
        "time": TIMESTAMP_INT,
        "type": "TestRunInProgress",
    }

    command_event_handler.EnqueueCommandEvents([event, malformed_event])
    tasks = self.mock_task_scheduler.GetTasks()
    self.assertEqual(len(tasks), 2)
    self.testapp.post(
        command_event_handler.COMMAND_EVENT_HANDLER_PATH, tasks[0].payload)
    with self.assertRaises(webtest.app.AppError):
      self.testapp.post(
          command_event_handler.COMMAND_EVENT_HANDLER_PATH, tasks[1].payload)

    command_attempts = command_manager.GetCommandAttempts(
        request_id, command_id)
    self.assertEqual(len(command_attempts), 1)
    self.assertEqual(common.CommandState.RUNNING, command_attempts[0].state)
  def testProcessCommandEvent_TFShutdown(
      self, mock_notify, mock_command_event_type_count):
    """Mark command events that were terminated by TF shutdown as CANCELED."""
    _, request_id, _, command_id = self.command.key.flat()
    command_manager.ScheduleTasks([self.command])
    error = "RunInterruptedException: Tradefed is shutting down."

    for i in range(3):
      tasks = command_manager.GetActiveTasks(self.command)
      self.assertEqual(len(tasks), 1)
      command_task_store.LeaseTask(tasks[0].task_id)
      command_event_test_util.CreateCommandAttempt(
          self.command, str(i), common.CommandState.UNKNOWN, task=tasks[0])
      event = command_event_test_util.CreateTestCommandEvent(
          request_id, command_id, str(i), "InvocationCompleted",
          task=tasks[0], data={"error": error})
      command_event_handler.ProcessCommandEvent(event)

    # After three cancelled attempts, the command should still be queued.
    queried_command = self.command.key.get()
    self.assertEqual(common.CommandState.QUEUED, queried_command.state)
    tasks = command_manager.GetActiveTasks(self.command)
    self.assertEqual(len(tasks), 1)
    attempts = command_manager.GetCommandAttempts(request_id, command_id)
    self.assertEqual(3, len(attempts))
    mock_notify.assert_called_with(request_id)
    expected_metric_fields = {
        metric.METRIC_FIELD_HOSTNAME: "hostname",
        metric.METRIC_FIELD_TYPE: "InvocationCompleted"
    }
    mock_command_event_type_count.Increment.assert_has_calls(
        [mock.call(expected_metric_fields)] * 3)
Exemple #8
0
  def testCreateCommandAttempt(self):
    request_id = 'request_id'
    command_id = 'command_id'
    leased_tasks = [
        command_task_api.CommandTask(
            task_id='task_id0',
            request_id=request_id,
            command_id=command_id,
            device_serials=['d1'],
            plugin_data=[
                api_messages.KeyValuePair(key='key0', value='value0'),
                api_messages.KeyValuePair(key='key1', value='value1'),
                api_messages.KeyValuePair(key='hostname', value='ahost'),
                api_messages.KeyValuePair(key='lab_name', value='alab'),
                api_messages.KeyValuePair(key='host_group', value='cluster'),
            ]),
        command_task_api.CommandTask(
            task_id='task_id1',
            request_id=request_id,
            command_id=command_id,
            device_serials=['d2', 'd3'],
            plugin_data=[
                api_messages.KeyValuePair(key='key2', value='value2'),
                api_messages.KeyValuePair(key='key3', value='value3'),
                api_messages.KeyValuePair(key='hostname', value='ahost'),
                api_messages.KeyValuePair(key='lab_name', value='alab'),
                api_messages.KeyValuePair(key='host_group', value='agroup'),
            ])
    ]
    command_task_api.CommandTaskApi()._CreateCommandAttempt(leased_tasks)
    attempts = command_manager.GetCommandAttempts(request_id='request_id',
                                                  command_id='command_id')

    self.assertEqual(2, len(attempts))
    self.assertEqual('command_id', attempts[0].command_id)
    self.assertEqual('task_id0', attempts[0].task_id)
    self.assertEqual('ahost', attempts[0].hostname)
    self.assertEqual(['d1'], attempts[0].device_serials)
    self.assertEqual(
        {'host_group': 'cluster', 'hostname': 'ahost', 'lab_name': 'alab',
         'key0': 'value0', 'key1': 'value1'},
        attempts[0].plugin_data)
    self.assertIsNotNone(attempts[0].last_event_time)
    self.assertEqual('command_id', attempts[1].command_id)
    self.assertEqual('task_id1', attempts[1].task_id)
    self.assertEqual('ahost', attempts[1].hostname)
    self.assertEqual(['d2', 'd3'], attempts[1].device_serials)
    self.assertEqual(
        {'host_group': 'agroup', 'hostname': 'ahost', 'lab_name': 'alab',
         'key2': 'value2', 'key3': 'value3'},
        attempts[1].plugin_data)
    self.assertIsNotNone(attempts[1].last_event_time)
  def testProcessCommandEvent_invocationCompleted_multipleRunsWithErrors(
      self, mock_notify):
    """Should complete command state for InvocationCompleted events.

    This tests a case when a run count is greater than
    command_manager.MAX_TASK_COUNT.

    Args:
      mock_notify: mock function to notify request state changes.
    """
    run_count = 100
    _, request_id, _, command_id = self.command.key.flat()
    self.command.run_count = run_count
    self.command.put()
    command_manager.ScheduleTasks([self.command])

    max_error_count = (
        command_manager.MAX_ERROR_COUNT_BASE +
        int(run_count * command_manager.MAX_ERROR_COUNT_RATIO))
    for i in range(max_error_count):
      tasks = command_manager.GetActiveTasks(self.command)
      self.assertEqual(
          len(tasks), min(run_count - i, command_manager.MAX_TASK_COUNT))
      next_leasable_task = next((t for t in tasks if t.leasable), None)
      command_task_store.LeaseTask(next_leasable_task.task_id)
      command_event_test_util.CreateCommandAttempt(
          self.command, str(i), common.CommandState.UNKNOWN,
          task=next_leasable_task)
      # It should be marked as error at the max error count attempts.
      queried_command = command_manager.GetCommand(request_id, command_id)
      self.assertNotEqual(common.CommandState.ERROR, queried_command.state)
      event = command_event_test_util.CreateTestCommandEvent(
          request_id, command_id, str(i), "InvocationCompleted",
          task=next_leasable_task, error="error")
      command_event_handler.ProcessCommandEvent(event)

    queried_command = command_manager.GetCommand(request_id, command_id)
    self.assertEqual(common.CommandState.ERROR, queried_command.state)
    tasks = command_manager.GetActiveTasks(self.command)
    self.assertEqual(len(tasks), 0)
    command_attempts = command_manager.GetCommandAttempts(
        request_id, command_id)
    self.assertEqual(len(command_attempts), max_error_count)
    run_attempt_pairs = [
        (attempt.run_index, attempt.attempt_index)
        for attempt in command_attempts]
    self.assertEqual(len(set(run_attempt_pairs)), len(command_attempts))
    mock_notify.assert_called_with(request_id)
  def testEnqueueCommandEvents(self):
    _, request_id, _, command_id = self.command.key.flat()
    command_event_test_util.CreateCommandAttempt(
        self.command, "aid", common.CommandState.QUEUED)
    event = command_event_test_util.CreateTestCommandEventJson(
        request_id, command_id, "aid", "InvocationCompleted")

    command_event_handler.EnqueueCommandEvents([event])
    tasks = self.mock_task_scheduler.GetTasks()
    self.assertEqual(len(tasks), 1)
    self.testapp.post(
        command_event_handler.COMMAND_EVENT_HANDLER_PATH, tasks[0].payload)

    command_attempts = command_manager.GetCommandAttempts(
        request_id, command_id)
    self.assertEqual(len(command_attempts), 1)
    self.assertEqual(common.CommandState.COMPLETED, command_attempts[0].state)
  def testProcessCommandEvent_invocationCompleted_multipleRuns_20(
      self, mock_notify):
    """Should complete command state for InvocationCompleted events.

    This tests a case when a run count is greater than
    command_manager.MAX_TASK_COUNT.

    Args:
      mock_notify: mock function to notify request state changes.
    """
    run_count = 100
    _, request_id, _, command_id = self.command.key.flat()
    self.command.run_count = run_count
    self.command.put()
    command_manager.ScheduleTasks([self.command])

    for i in range(run_count):
      tasks = command_manager.GetActiveTasks(self.command)
      self.assertEqual(
          len(tasks), min(run_count - i, command_manager.MAX_TASK_COUNT))
      next_leasable_task = next((t for t in tasks if t.leasable), None)
      command_task_store.LeaseTask(next_leasable_task.task_id)
      queried_command = command_manager.GetCommand(request_id, command_id)
      self.assertNotEqual(common.CommandState.COMPLETED, queried_command.state)
      command_event_test_util.CreateCommandAttempt(
          self.command, str(i), common.CommandState.UNKNOWN,
          task=next_leasable_task)
      event = command_event_test_util.CreateTestCommandEvent(
          request_id, command_id, str(i), "InvocationCompleted",
          task=next_leasable_task)
      command_event_handler.ProcessCommandEvent(event)

    queried_command = command_manager.GetCommand(request_id, command_id)
    self.assertEqual(common.CommandState.COMPLETED, queried_command.state)
    tasks = command_manager.GetActiveTasks(self.command)
    self.assertEqual(len(tasks), 0)
    command_attempts = command_manager.GetCommandAttempts(
        request_id, command_id)
    self.assertEqual(len(command_attempts), run_count)
    self.assertSetEqual(
        set(attempt.run_index for attempt in command_attempts),
        set(range(run_count)))
    mock_notify.assert_called_with(request_id)
  def testProcessCommandEvent_allocationFailed(self,
                                               mock_command_event_type_count,
                                               mock_event_legacy_count):
    """Should reschedule tasks that send AllocationFailed events."""
    _, request_id, _, command_id = self.command.key.flat()
    command_manager.ScheduleTasks([self.command])

    # Command should stay queued even after MAX_CANCELED_COUNT_BASE different
    # attempts result in an AllocationFailed error.
    num_attempts = command_manager.MAX_CANCELED_COUNT_BASE
    for i in range(num_attempts):
      tasks = command_manager.GetActiveTasks(self.command)
      self.assertEqual(len(tasks), 1)
      command_task_store.LeaseTask(tasks[0].task_id)
      command_event_test_util.CreateCommandAttempt(
          self.command, str(i), common.CommandState.UNKNOWN, task=tasks[0])
      queried_command = command_manager.GetCommand(request_id, command_id)
      self.assertNotEqual(common.CommandState.ERROR, queried_command.state)
      event = command_event_test_util.CreateTestCommandEvent(
          request_id, command_id, str(i), "AllocationFailed", task=tasks[0])
      command_event_handler.ProcessCommandEvent(event)

    queried_command = command_manager.GetCommand(request_id, command_id)
    self.assertEqual(common.CommandState.CANCELED, queried_command.state)
    tasks = command_manager.GetActiveTasks(self.command)
    self.assertEqual(len(tasks), 0)
    command_attempts = command_manager.GetCommandAttempts(
        request_id, command_id)
    self.assertEqual(
        len(command_attempts), command_manager.MAX_CANCELED_COUNT_BASE)
    self.assertSetEqual(
        set(attempt.attempt_index for attempt in command_attempts),
        set(range(command_manager.MAX_CANCELED_COUNT_BASE)))
    expected_metric_fields = {
        metric.METRIC_FIELD_HOSTNAME: "hostname",
        metric.METRIC_FIELD_TYPE: "AllocationFailed"
    }
    mock_command_event_type_count.Increment.assert_has_calls(
        [mock.call(expected_metric_fields)] * num_attempts)
    mock_event_legacy_count.Increment.assert_has_calls([mock.call({})] *
                                                       num_attempts)
  def testProcessCommandEvent_InvocationStarted(
      self, mock_notify, mock_command_event_type_count):
    """Should update command state for InvocationStarted events.

    State should become RUNNING if it isn't already.

    Args:
      mock_notify: mock function to notify request state changes.
      mock_command_event_type_count: mock command event type count metric
    """
    _, request_id, _, command_id = self.command.key.flat()
    command_manager.ScheduleTasks([self.command])

    tasks = command_manager.GetActiveTasks(self.command)
    self.assertEqual(len(tasks), 1)
    command_task_store.LeaseTask(tasks[0].task_id)
    command_event_test_util.CreateCommandAttempt(
        self.command, "0", common.CommandState.UNKNOWN, task=tasks[0])
    queried_command = command_manager.GetCommand(request_id, command_id)
    self.assertNotEqual(common.CommandState.RUNNING, queried_command.state)
    event = command_event_test_util.CreateTestCommandEvent(
        request_id, command_id, "0", "InvocationStarted", task=tasks[0])
    command_event_handler.ProcessCommandEvent(event)

    queried_command = command_manager.GetCommand(request_id, command_id)
    self.assertEqual(common.CommandState.RUNNING, queried_command.state)
    tasks = command_manager.GetActiveTasks(self.command)
    self.assertEqual(len(tasks), 1)
    self.assertEqual(tasks[0].attempt_index, 0)
    self.assertEqual(tasks[0].leasable, False)
    attempts = command_manager.GetCommandAttempts(request_id, command_id)
    self.assertEqual(1, len(attempts))
    self.assertEqual(["0123456789ABCDEF"], attempts[0].device_serials)
    mock_notify.assert_called_with(request_id)
    expected_metric_fields = {
        metric.METRIC_FIELD_HOSTNAME: "hostname",
        metric.METRIC_FIELD_TYPE: "InvocationStarted"
    }
    mock_command_event_type_count.Increment.assert_called_once_with(
        expected_metric_fields)
  def testProcessCommandEvent_invocationCompleted(
      self, mock_notify, mock_command_event_type_count):
    """Should complete command state for InvocationCompleted events.

    Args:
      mock_notify: mock function to notify request state changes.
      mock_command_event_type_count: mock command event type count metric
    """
    _, request_id, _, command_id = self.command.key.flat()
    command_manager.ScheduleTasks([self.command])

    tasks = command_manager.GetActiveTasks(self.command)
    self.assertEqual(len(tasks), 1)
    command_task_store.LeaseTask(tasks[0].task_id)
    command_event_test_util.CreateCommandAttempt(
        self.command, "0", common.CommandState.UNKNOWN, task=tasks[0])
    queried_command = command_manager.GetCommand(request_id, command_id)
    self.assertNotEqual(common.CommandState.COMPLETED, queried_command.state)
    event = command_event_test_util.CreateTestCommandEvent(
        request_id, command_id, "0", "InvocationCompleted", task=tasks[0])
    command_event_handler.ProcessCommandEvent(event)

    queried_command = command_manager.GetCommand(request_id, command_id)
    self.assertEqual(common.CommandState.COMPLETED, queried_command.state)
    tasks = command_manager.GetActiveTasks(self.command)
    self.assertEqual(len(tasks), 0)
    attempts = command_manager.GetCommandAttempts(request_id, command_id)
    self.assertEqual(1, len(attempts))
    self.assertEqual("summary", attempts[0].summary)
    self.assertEqual(1000, attempts[0].total_test_count)
    self.assertEqual(100, attempts[0].failed_test_count)
    self.assertEqual(10, attempts[0].failed_test_run_count)
    mock_notify.assert_called_with(request_id)
    expected_metric_fields = {
        metric.METRIC_FIELD_HOSTNAME: "hostname",
        metric.METRIC_FIELD_TYPE: "InvocationCompleted"
    }
    mock_command_event_type_count.Increment.assert_called_once_with(
        expected_metric_fields)
  def testProcessCommandEvent_fetchFailed(
      self, mock_notify, mock_command_event_type_count):
    """Should error commands from FetchFailed events."""
    _, request_id, _, command_id = self.command.key.flat()
    command_manager.ScheduleTasks([self.command])

    # It should be marked as error at the MAX_ERROR_COUNT_BASE attempt.
    for i in range(command_manager.MAX_ERROR_COUNT_BASE):
      tasks = command_manager.GetActiveTasks(self.command)
      self.assertEqual(len(tasks), 1)
      self.assertEqual(tasks[0].attempt_index, i)
      command_task_store.LeaseTask(tasks[0].task_id)
      command_event_test_util.CreateCommandAttempt(
          self.command, str(i), common.CommandState.UNKNOWN, task=tasks[0])
      queried_command = command_manager.GetCommand(request_id, command_id)
      self.assertNotEqual(common.CommandState.ERROR, queried_command.state)
      event = command_event_test_util.CreateTestCommandEvent(
          request_id, command_id, str(i), "FetchFailed", task=tasks[0])
      command_event_handler.ProcessCommandEvent(event)

    queried_command = command_manager.GetCommand(request_id, command_id)
    self.assertEqual(common.CommandState.ERROR, queried_command.state)
    tasks = command_manager.GetActiveTasks(self.command)
    self.assertEqual(len(tasks), 0)
    command_attempts = command_manager.GetCommandAttempts(
        request_id, command_id)
    self.assertEqual(
        len(command_attempts), command_manager.MAX_ERROR_COUNT_BASE)
    self.assertSetEqual(
        set(attempt.attempt_index for attempt in command_attempts),
        set(range(command_manager.MAX_ERROR_COUNT_BASE)))
    mock_notify.assert_called_with(request_id)
    expected_metric_fields = {
        metric.METRIC_FIELD_HOSTNAME: "hostname",
        metric.METRIC_FIELD_TYPE: "FetchFailed"
    }
    mock_command_event_type_count.Increment.assert_has_calls(
        [mock.call(expected_metric_fields)
        ] * command_manager.MAX_ERROR_COUNT_BASE)
  def testProcessCommandEvent_fatalEvent(
      self, mock_notify, mock_command_event_type_count):
    """Should not reschedule a configuration error, request should error out.

    Args:
      mock_notify: mock function to notify request state changes.
      mock_command_event_type_count: mock command event type count metric
    """
    _, request_id, _, command_id = self.command.key.flat()
    command_manager.ScheduleTasks([self.command])

    tasks = command_manager.GetActiveTasks(self.command)
    self.assertEqual(len(tasks), 1)
    command_task_store.LeaseTask(tasks[0].task_id)
    command_event_test_util.CreateCommandAttempt(
        self.command, "0", common.CommandState.UNKNOWN, task=tasks[0])
    queried_command = command_manager.GetCommand(request_id, command_id)
    self.assertNotEqual(common.CommandState.FATAL, queried_command.state)
    event = command_event_test_util.CreateTestCommandEvent(
        request_id, command_id, "0", "ConfigurationError",
        data={"error_status": "CUSTOMER_ISSUE"},
        task=tasks[0])
    command_event_handler.ProcessCommandEvent(event)

    queried_command = command_manager.GetCommand(request_id, command_id)
    self.assertEqual(common.CommandState.FATAL, queried_command.state)
    tasks = command_manager.GetActiveTasks(self.command)
    self.assertEqual(len(tasks), 0)
    attempts = command_manager.GetCommandAttempts(request_id, command_id)
    self.assertEqual(1, len(attempts))
    mock_notify.assert_called_with(request_id)
    expected_metric_fields = {
        metric.METRIC_FIELD_HOSTNAME: "hostname",
        metric.METRIC_FIELD_TYPE: "ConfigurationError"
    }
    mock_command_event_type_count.Increment.assert_called_once_with(
        expected_metric_fields)