def test_cron_handle_bot_died_second(self):
   # Test two tries internal_failure's leading to a BOT_DIED status.
   self.mock(random, 'getrandbits', lambda _: 0x88)
   now = utils.utcnow()
   data = _gen_request(
       properties=dict(dimensions={u'OS': u'Windows-3.1.1'}),
       created_ts=now,
       expiration_ts=now+datetime.timedelta(seconds=600))
   request = task_request.make_request(data, True)
   _result_summary = task_scheduler.schedule_request(request)
   bot_dimensions = {
     u'OS': [u'Windows', u'Windows-3.1.1'],
     u'hostname': u'localhost',
     u'foo': u'bar',
   }
   _request, run_result = task_scheduler.bot_reap_task(
       bot_dimensions, 'localhost', 'abc')
   self.assertEqual(1, run_result.try_number)
   self.assertEqual(task_result.State.RUNNING, run_result.state)
   self.mock_now(self.now + task_result.BOT_PING_TOLERANCE, 1)
   self.assertEqual(([], 1, 0), task_scheduler.cron_handle_bot_died('f.local'))
   now_1 = self.mock_now(self.now + task_result.BOT_PING_TOLERANCE, 2)
   # It must be a different bot.
   _request, run_result = task_scheduler.bot_reap_task(
       bot_dimensions, 'localhost-second', 'abc')
   now_2 = self.mock_now(self.now + 2 * task_result.BOT_PING_TOLERANCE, 3)
   self.assertEqual(
       (['1d69b9f088008812'], 0, 0),
       task_scheduler.cron_handle_bot_died('f.local'))
   self.assertEqual(([], 0, 0), task_scheduler.cron_handle_bot_died('f.local'))
   expected = {
     'abandoned_ts': now_2,
     'bot_dimensions': bot_dimensions,
     'bot_id': u'localhost-second',
     'bot_version': u'abc',
     'children_task_ids': [],
     'completed_ts': None,
     'costs_usd': [0., 0.],
     'cost_saved_usd': None,
     'created_ts': self.now,
     'deduped_from': None,
     'durations': [],
     'exit_codes': [],
     'failure': False,
     'id': '1d69b9f088008810',
     'internal_failure': True,
     'modified_ts': now_2,
     'name': u'Request name',
     'outputs_ref': None,
     'properties_hash': None,
     'server_versions': [u'v1a'],
     'started_ts': now_1,
     'state': task_result.State.BOT_DIED,
     'tags': [u'OS:Windows-3.1.1', u'priority:50', u'tag:1', u'user:Jesus'],
     'try_number': 2,
     'user': u'Jesus',
   }
   self.assertEqual(expected, run_result.result_summary_key.get().to_dict())
Exemple #2
0
 def test_cron_handle_bot_died_second(self):
   # Test two tries internal_failure's leading to a BOT_DIED status.
   self.mock(random, 'getrandbits', lambda _: 0x88)
   now = utils.utcnow()
   data = _gen_request(
       properties=dict(dimensions={u'OS': u'Windows-3.1.1'}),
       created_ts=now,
       expiration_ts=now+datetime.timedelta(seconds=600))
   request = task_request.make_request(data, True)
   _result_summary = task_scheduler.schedule_request(request)
   bot_dimensions = {
     u'OS': [u'Windows', u'Windows-3.1.1'],
     u'hostname': u'localhost',
     u'foo': u'bar',
   }
   _request, run_result = task_scheduler.bot_reap_task(
       bot_dimensions, 'localhost', 'abc')
   self.assertEqual(1, run_result.try_number)
   self.assertEqual(task_result.State.RUNNING, run_result.state)
   self.mock_now(self.now + task_result.BOT_PING_TOLERANCE, 1)
   self.assertEqual((0, 1, 0), task_scheduler.cron_handle_bot_died())
   now_1 = self.mock_now(self.now + task_result.BOT_PING_TOLERANCE, 2)
   # It must be a different bot.
   _request, run_result = task_scheduler.bot_reap_task(
       bot_dimensions, 'localhost-second', 'abc')
   now_2 = self.mock_now(self.now + 2 * task_result.BOT_PING_TOLERANCE, 3)
   self.assertEqual((1, 0, 0), task_scheduler.cron_handle_bot_died())
   self.assertEqual((0, 0, 0), task_scheduler.cron_handle_bot_died())
   expected = {
     'abandoned_ts': now_2,
     'bot_dimensions': bot_dimensions,
     'bot_id': u'localhost-second',
     'bot_version': u'abc',
     'children_task_ids': [],
     'completed_ts': None,
     'costs_usd': [0., 0.],
     'cost_saved_usd': None,
     'created_ts': self.now,
     'deduped_from': None,
     'durations': [],
     'exit_codes': [],
     'failure': False,
     'id': '1d69b9f088008810',
     'internal_failure': True,
     'modified_ts': now_2,
     'name': u'Request name',
     'outputs_ref': None,
     'properties_hash': None,
     'server_versions': [u'v1a'],
     'started_ts': now_1,
     'state': task_result.State.BOT_DIED,
     'tags': [u'OS:Windows-3.1.1', u'priority:50', u'tag:1', u'user:Jesus'],
     'try_number': 2,
     'user': u'Jesus',
   }
   self.assertEqual(expected, run_result.result_summary_key.get().to_dict())
 def get(self):
   try:
     task_scheduler.cron_handle_bot_died()
   except db.NeedIndexError as e:
     # When a fresh new instance is deployed, it takes a few minutes for the
     # composite indexes to be created even if they are empty. Ignore the case
     # where the index is defined but still being created by AppEngine.
     if not str(e).startswith(
         'NeedIndexError: The index for this query is not ready to serve.'):
       raise
   self.response.headers['Content-Type'] = 'text/plain; charset=utf-8'
   self.response.out.write('Success.')
Exemple #4
0
 def get(self):
   try:
     task_scheduler.cron_handle_bot_died(self.request.host_url)
   except datastore_errors.NeedIndexError as e:
     # When a fresh new instance is deployed, it takes a few minutes for the
     # composite indexes to be created even if they are empty. Ignore the case
     # where the index is defined but still being created by AppEngine.
     if not str(e).startswith(
         'NeedIndexError: The index for this query is not ready to serve.'):
       raise
   self.response.headers['Content-Type'] = 'text/plain; charset=utf-8'
   self.response.out.write('Success.')
  def test_cron_abort_expired_task_to_run_retry(self):
    self.mock(random, 'getrandbits', lambda _: 0x88)
    now = utils.utcnow()
    data = _gen_request(
        properties=dict(dimensions={u'OS': u'Windows-3.1.1'}),
        created_ts=now,
        expiration_ts=now+datetime.timedelta(seconds=600))
    request = task_request.make_request(data, True)
    result_summary = task_scheduler.schedule_request(request)

    # Fake first try bot died.
    bot_dimensions = {
      u'OS': [u'Windows', u'Windows-3.1.1'],
      u'hostname': u'localhost',
      u'foo': u'bar',
    }
    _request, run_result = task_scheduler.bot_reap_task(
        bot_dimensions, 'localhost', 'abc')
    now_1 = self.mock_now(self.now + task_result.BOT_PING_TOLERANCE, 1)
    self.assertEqual((0, 1, 0), task_scheduler.cron_handle_bot_died())
    self.assertEqual(task_result.State.BOT_DIED, run_result.key.get().state)
    self.assertEqual(
        task_result.State.PENDING, run_result.result_summary_key.get().state)

    # BOT_DIED is kept instead of EXPIRED.
    abandoned_ts = self.mock_now(self.now, request.expiration_secs+1)
    self.assertEqual(1, task_scheduler.cron_abort_expired_task_to_run())
    self.assertEqual(1, len(task_result.TaskRunResult.query().fetch()))
    expected = {
      'abandoned_ts': abandoned_ts,
      'bot_dimensions': bot_dimensions,
      'bot_id': u'localhost',
      'bot_version': u'abc',
      'children_task_ids': [],
      'completed_ts': None,
      'costs_usd': [0.],
      'cost_saved_usd': None,
      'created_ts': self.now,
      'deduped_from': None,
      'durations': [],
      'exit_codes': [],
      'failure': False,
      'id': '1d69b9f088008810',
      'internal_failure': True,
      'modified_ts': abandoned_ts,
      'name': u'Request name',
      'outputs_ref': None,
      'properties_hash': None,
      'server_versions': [u'v1a'],
      'started_ts': self.now,
      'state': task_result.State.BOT_DIED,
      'tags': [u'OS:Windows-3.1.1', u'priority:50', u'tag:1', u'user:Jesus'],
      'try_number': 1,
      'user': u'Jesus',
    }
    self.assertEqual(expected, result_summary.key.get().to_dict())
Exemple #6
0
  def test_cron_abort_expired_task_to_run_retry(self):
    self.mock(random, 'getrandbits', lambda _: 0x88)
    now = utils.utcnow()
    data = _gen_request(
        properties=dict(dimensions={u'OS': u'Windows-3.1.1'}),
        created_ts=now,
        expiration_ts=now+datetime.timedelta(seconds=600))
    request = task_request.make_request(data, True)
    result_summary = task_scheduler.schedule_request(request)

    # Fake first try bot died.
    bot_dimensions = {
      u'OS': [u'Windows', u'Windows-3.1.1'],
      u'hostname': u'localhost',
      u'foo': u'bar',
    }
    _request, run_result = task_scheduler.bot_reap_task(
        bot_dimensions, 'localhost', 'abc')
    now_1 = self.mock_now(self.now + task_result.BOT_PING_TOLERANCE, 1)
    self.assertEqual((0, 1, 0), task_scheduler.cron_handle_bot_died())
    self.assertEqual(task_result.State.BOT_DIED, run_result.key.get().state)
    self.assertEqual(
        task_result.State.PENDING, run_result.result_summary_key.get().state)

    # BOT_DIED is kept instead of EXPIRED.
    abandoned_ts = self.mock_now(self.now, request.expiration_secs+1)
    self.assertEqual(1, task_scheduler.cron_abort_expired_task_to_run())
    self.assertEqual(1, len(task_result.TaskRunResult.query().fetch()))
    expected = {
      'abandoned_ts': abandoned_ts,
      'bot_dimensions': bot_dimensions,
      'bot_id': u'localhost',
      'bot_version': u'abc',
      'children_task_ids': [],
      'completed_ts': None,
      'costs_usd': [0.],
      'cost_saved_usd': None,
      'created_ts': self.now,
      'deduped_from': None,
      'durations': [],
      'exit_codes': [],
      'failure': False,
      'id': '1d69b9f088008810',
      'internal_failure': True,
      'modified_ts': abandoned_ts,
      'name': u'Request name',
      'outputs_ref': None,
      'properties_hash': None,
      'server_versions': [u'v1a'],
      'started_ts': self.now,
      'state': task_result.State.BOT_DIED,
      'tags': [u'OS:Windows-3.1.1', u'priority:50', u'tag:1', u'user:Jesus'],
      'try_number': 1,
      'user': u'Jesus',
    }
    self.assertEqual(expected, result_summary.key.get().to_dict())
 def test_cron_handle_bot_died_ignored_expired(self):
   self.mock(random, 'getrandbits', lambda _: 0x88)
   data = _gen_request_data(
       properties=dict(dimensions={u'OS': u'Windows-3.1.1'}),
       scheduling_expiration_secs=600)
   request = task_request.make_request(data)
   _result_summary = task_scheduler.schedule_request(request)
   bot_dimensions = {
     u'OS': [u'Windows', u'Windows-3.1.1'],
     u'hostname': u'localhost',
     u'foo': u'bar',
   }
   _request, run_result = task_scheduler.bot_reap_task(
       bot_dimensions, 'localhost', 'abc')
   self.assertEqual(1, run_result.try_number)
   self.assertEqual(task_result.State.RUNNING, run_result.state)
   self.mock_now(self.now + task_result.BOT_PING_TOLERANCE, 601)
   self.assertEqual((1, 0, 0), task_scheduler.cron_handle_bot_died())
 def test_cron_handle_bot_died_ignored_expired(self):
     self.mock(random, 'getrandbits', lambda _: 0x88)
     data = _gen_request_data(
         properties=dict(dimensions={u'OS': u'Windows-3.1.1'}),
         scheduling_expiration_secs=600)
     request = task_request.make_request(data)
     _result_summary = task_scheduler.schedule_request(request)
     bot_dimensions = {
         u'OS': [u'Windows', u'Windows-3.1.1'],
         u'hostname': u'localhost',
         u'foo': u'bar',
     }
     _request, run_result = task_scheduler.bot_reap_task(
         bot_dimensions, 'localhost', 'abc')
     self.assertEqual(1, run_result.try_number)
     self.assertEqual(task_result.State.RUNNING, run_result.state)
     self.mock_now(self.now + task_result.BOT_PING_TOLERANCE, 601)
     self.assertEqual((1, 0, 0), task_scheduler.cron_handle_bot_died())
 def test_cron_handle_bot_died_ignored_expired(self):
   self.mock(random, 'getrandbits', lambda _: 0x88)
   now = utils.utcnow()
   data = _gen_request(
       properties=dict(dimensions={u'OS': u'Windows-3.1.1'}),
       created_ts=now,
       expiration_ts=now+datetime.timedelta(seconds=600))
   request = task_request.make_request(data, True)
   _result_summary = task_scheduler.schedule_request(request)
   bot_dimensions = {
     u'OS': [u'Windows', u'Windows-3.1.1'],
     u'hostname': u'localhost',
     u'foo': u'bar',
   }
   _request, run_result = task_scheduler.bot_reap_task(
       bot_dimensions, 'localhost', 'abc')
   self.assertEqual(1, run_result.try_number)
   self.assertEqual(task_result.State.RUNNING, run_result.state)
   self.mock_now(self.now + task_result.BOT_PING_TOLERANCE, 601)
   self.assertEqual(
       (['1d69b9f088008811'], 0, 0),
       task_scheduler.cron_handle_bot_died('f.local'))
  def test_cron_handle_bot_died_same_bot_denied(self):
    # Test first retry, then success.
    self.mock(random, 'getrandbits', lambda _: 0x88)
    now = utils.utcnow()
    data = _gen_request(
        properties=dict(dimensions={u'OS': u'Windows-3.1.1'}),
        created_ts=now,
        expiration_ts=now+datetime.timedelta(seconds=600))
    request = task_request.make_request(data, True)
    _result_summary = task_scheduler.schedule_request(request)
    bot_dimensions = {
      u'OS': [u'Windows', u'Windows-3.1.1'],
      u'hostname': u'localhost',
      u'foo': u'bar',
    }
    _request, run_result = task_scheduler.bot_reap_task(
        bot_dimensions, 'localhost', 'abc')
    self.assertEqual(1, run_result.try_number)
    self.assertEqual(task_result.State.RUNNING, run_result.state)
    now_1 = self.mock_now(self.now + task_result.BOT_PING_TOLERANCE, 1)
    self.assertEqual(([], 1, 0), task_scheduler.cron_handle_bot_died('f.local'))

    # Refresh and compare:
    expected = {
      'abandoned_ts': now_1,
      'bot_dimensions': bot_dimensions,
      'bot_id': u'localhost',
      'bot_version': u'abc',
      'children_task_ids': [],
      'completed_ts': None,
      'cost_usd': 0.,
      'durations': [],
      'exit_codes': [],
      'failure': False,
      'id': '1d69b9f088008811',
      'internal_failure': True,
      'modified_ts': now_1,
      'outputs_ref': None,
      'server_versions': [u'v1a'],
      'started_ts': self.now,
      'state': task_result.State.BOT_DIED,
      'try_number': 1,
    }
    self.assertEqual(expected, run_result.key.get().to_dict())
    expected = {
      'abandoned_ts': None,
      'bot_dimensions': bot_dimensions,
      'bot_id': u'localhost',
      'bot_version': u'abc',
      'children_task_ids': [],
      'completed_ts': None,
      'costs_usd': [0.],
      'cost_saved_usd': None,
      'created_ts': self.now,
      'deduped_from': None,
      'durations': [],
      'exit_codes': [],
      'failure': False,
      'id': '1d69b9f088008810',
      'internal_failure': False,
      'modified_ts': now_1,
      'name': u'Request name',
      'outputs_ref': None,
      'properties_hash': None,
      'server_versions': [u'v1a'],
      'started_ts': None,
      'state': task_result.State.PENDING,
      'tags': [u'OS:Windows-3.1.1', u'priority:50', u'tag:1', u'user:Jesus'],
      'try_number': 1,
      'user': u'Jesus',
    }
    self.assertEqual(expected, run_result.result_summary_key.get().to_dict())

    # Task was retried but the same bot polls again, it's denied the task.
    now_2 = self.mock_now(self.now + task_result.BOT_PING_TOLERANCE, 2)
    request, run_result = task_scheduler.bot_reap_task(
        bot_dimensions, 'localhost', 'abc')
    self.assertEqual(None, request)
    self.assertEqual(None, run_result)
    logging.info('%s', [t.to_dict() for t in task_to_run.TaskToRun.query()])
Exemple #11
0
  def test_cron_handle_bot_died(self):
    # Test first retry, then success.
    self.mock(random, 'getrandbits', lambda _: 0x88)
    data = _gen_request_data(
        properties=dict(dimensions={u'OS': u'Windows-3.1.1'}),
        scheduling_expiration_secs=600)
    request = task_request.make_request(data)
    _result_summary = task_scheduler.schedule_request(request)
    bot_dimensions = {
      u'OS': [u'Windows', u'Windows-3.1.1'],
      u'hostname': u'localhost',
      u'foo': u'bar',
    }
    _request, run_result = task_scheduler.bot_reap_task(
        bot_dimensions, 'localhost', 'abc')
    self.assertEqual(1, run_result.try_number)
    self.assertEqual(task_result.State.RUNNING, run_result.state)
    now_1 = self.mock_now(self.now + task_result.BOT_PING_TOLERANCE, 1)
    self.assertEqual((0, 1, 0), task_scheduler.cron_handle_bot_died())

    # Refresh and compare:
    expected = {
      'abandoned_ts': now_1,
      'bot_id': u'localhost',
      'bot_version': u'abc',
      'children_task_ids': [],
      'completed_ts': None,
      'cost_usd': 0.,
      'durations': [],
      'exit_codes': [],
      'failure': False,
      'id': '1d69b9f088008811',
      'internal_failure': True,
      'modified_ts': now_1,
      'server_versions': [u'v1a'],
      'started_ts': self.now,
      'state': task_result.State.BOT_DIED,
      'try_number': 1,
    }
    self.assertEqual(expected, run_result.key.get().to_dict())
    expected = {
      'abandoned_ts': None,
      'bot_id': u'localhost',
      'bot_version': u'abc',
      'children_task_ids': [],
      'completed_ts': None,
      'costs_usd': [0.],
      'cost_saved_usd': None,
      'created_ts': self.now,
      'deduped_from': None,
      'durations': [],
      'exit_codes': [],
      'failure': False,
      'id': '1d69b9f088008810',
      'internal_failure': False,
      'modified_ts': now_1,
      'name': u'Request name',
      'properties_hash': None,
      'server_versions': [u'v1a'],
      'started_ts': None,
      'state': task_result.State.PENDING,
      'try_number': 1,
      'user': u'Jesus',
    }
    self.assertEqual(expected, run_result.result_summary_key.get().to_dict())

    # Task was retried.
    now_2 = self.mock_now(self.now + task_result.BOT_PING_TOLERANCE, 2)
    _request, run_result = task_scheduler.bot_reap_task(
        bot_dimensions, 'localhost-second', 'abc')
    logging.info('%s', [t.to_dict() for t in task_to_run.TaskToRun.query()])
    self.assertEqual(2, run_result.try_number)
    self.assertEqual(
        (True, True),
        task_scheduler.bot_update_task(
            run_result.key, 'localhost-second', 'Foo1', 0, 0, 0.1, False, False,
            0.1))
    expected = {
      'abandoned_ts': None,
      'bot_id': u'localhost-second',
      'bot_version': u'abc',
      'children_task_ids': [],
      'completed_ts': now_2,
      'costs_usd': [0., 0.1],
      'cost_saved_usd': None,
      'created_ts': self.now,
      'deduped_from': None,
      'durations': [0.1],
      'exit_codes': [0],
      'failure': False,
      'id': '1d69b9f088008810',
      'internal_failure': False,
      'modified_ts': now_2,
      'name': u'Request name',
      'properties_hash': None,
      'server_versions': [u'v1a'],
      'started_ts': now_2,
      'state': task_result.State.COMPLETED,
      'try_number': 2,
      'user': u'Jesus',
    }
    self.assertEqual(expected, run_result.result_summary_key.get().to_dict())
    self.assertEqual(0.1, run_result.key.get().cost_usd)
Exemple #12
0
 def run_cron(self):
     task_scheduler.cron_handle_bot_died()
Exemple #13
0
 def get(self):
     task_scheduler.cron_handle_bot_died()
     self.response.headers['Content-Type'] = 'text/plain; charset=utf-8'
     self.response.out.write('Success.')
Exemple #14
0
 def get(self):
   task_scheduler.cron_handle_bot_died()
   self.response.headers['Content-Type'] = 'text/plain; charset=utf-8'
   self.response.out.write('Success.')
    def test_cron_handle_bot_died_same_bot_denied(self):
        # Test first retry, then success.
        self.mock(random, 'getrandbits', lambda _: 0x88)
        data = _gen_request_data(
            properties=dict(dimensions={u'OS': u'Windows-3.1.1'}),
            scheduling_expiration_secs=600)
        request = task_request.make_request(data)
        _result_summary = task_scheduler.schedule_request(request)
        bot_dimensions = {
            u'OS': [u'Windows', u'Windows-3.1.1'],
            u'hostname': u'localhost',
            u'foo': u'bar',
        }
        _request, run_result = task_scheduler.bot_reap_task(
            bot_dimensions, 'localhost', 'abc')
        self.assertEqual(1, run_result.try_number)
        self.assertEqual(task_result.State.RUNNING, run_result.state)
        now_1 = self.mock_now(self.now + task_result.BOT_PING_TOLERANCE, 1)
        self.assertEqual((0, 1, 0), task_scheduler.cron_handle_bot_died())

        # Refresh and compare:
        expected = {
            'abandoned_ts': now_1,
            'bot_id': u'localhost',
            'bot_version': u'abc',
            'children_task_ids': [],
            'completed_ts': None,
            'cost_usd': 0.,
            'durations': [],
            'exit_codes': [],
            'failure': False,
            'id': '1d69b9f088008811',
            'internal_failure': True,
            'modified_ts': now_1,
            'server_versions': [u'v1a'],
            'started_ts': self.now,
            'state': task_result.State.BOT_DIED,
            'try_number': 1,
        }
        self.assertEqual(expected, run_result.key.get().to_dict())
        expected = {
            'abandoned_ts': None,
            'bot_id': u'localhost',
            'bot_version': u'abc',
            'children_task_ids': [],
            'completed_ts': None,
            'costs_usd': [0.],
            'cost_saved_usd': None,
            'created_ts': self.now,
            'deduped_from': None,
            'durations': [],
            'exit_codes': [],
            'failure': False,
            'id': '1d69b9f088008810',
            'internal_failure': False,
            'modified_ts': now_1,
            'name': u'Request name',
            'properties_hash': None,
            'server_versions': [u'v1a'],
            'started_ts': None,
            'state': task_result.State.PENDING,
            'try_number': 1,
            'user': u'Jesus',
        }
        self.assertEqual(expected,
                         run_result.result_summary_key.get().to_dict())

        # Task was retried but the same bot polls again, it's denied the task.
        now_2 = self.mock_now(self.now + task_result.BOT_PING_TOLERANCE, 2)
        request, run_result = task_scheduler.bot_reap_task(
            bot_dimensions, 'localhost', 'abc')
        self.assertEqual(None, request)
        self.assertEqual(None, run_result)
        logging.info('%s',
                     [t.to_dict() for t in task_to_run.TaskToRun.query()])
Exemple #16
0
  def test_cron_handle_bot_died(self):
    # Test first retry, then success.
    self.mock(random, 'getrandbits', lambda _: 0x88)
    now = utils.utcnow()
    data = _gen_request(
        properties=dict(dimensions={u'OS': u'Windows-3.1.1'}),
        created_ts=now,
        expiration_ts=now+datetime.timedelta(seconds=600))
    request = task_request.make_request(data, True)
    _result_summary = task_scheduler.schedule_request(request)
    bot_dimensions = {
      u'OS': [u'Windows', u'Windows-3.1.1'],
      u'hostname': u'localhost',
      u'foo': u'bar',
    }
    _request, run_result = task_scheduler.bot_reap_task(
        bot_dimensions, 'localhost', 'abc')
    self.assertEqual(1, run_result.try_number)
    self.assertEqual(task_result.State.RUNNING, run_result.state)
    now_1 = self.mock_now(self.now + task_result.BOT_PING_TOLERANCE, 1)
    self.assertEqual((0, 1, 0), task_scheduler.cron_handle_bot_died())

    # Refresh and compare:
    expected = {
      'abandoned_ts': now_1,
      'bot_dimensions': bot_dimensions,
      'bot_id': u'localhost',
      'bot_version': u'abc',
      'children_task_ids': [],
      'completed_ts': None,
      'cost_usd': 0.,
      'durations': [],
      'exit_codes': [],
      'failure': False,
      'id': '1d69b9f088008811',
      'internal_failure': True,
      'modified_ts': now_1,
      'outputs_ref': None,
      'server_versions': [u'v1a'],
      'started_ts': self.now,
      'state': task_result.State.BOT_DIED,
      'try_number': 1,
    }
    self.assertEqual(expected, run_result.key.get().to_dict())
    expected = {
      'abandoned_ts': None,
      'bot_dimensions': bot_dimensions,
      'bot_id': u'localhost',
      'bot_version': u'abc',
      'children_task_ids': [],
      'completed_ts': None,
      'costs_usd': [0.],
      'cost_saved_usd': None,
      'created_ts': self.now,
      'deduped_from': None,
      'durations': [],
      'exit_codes': [],
      'failure': False,
      'id': '1d69b9f088008810',
      'internal_failure': False,
      'modified_ts': now_1,
      'name': u'Request name',
      'outputs_ref': None,
      'properties_hash': None,
      'server_versions': [u'v1a'],
      'started_ts': None,
      'state': task_result.State.PENDING,
      'tags': [u'OS:Windows-3.1.1', u'priority:50', u'tag:1', u'user:Jesus'],
      'try_number': 1,
      'user': u'Jesus',
    }
    self.assertEqual(expected, run_result.result_summary_key.get().to_dict())

    # Task was retried.
    now_2 = self.mock_now(self.now + task_result.BOT_PING_TOLERANCE, 2)
    _request, run_result = task_scheduler.bot_reap_task(
        bot_dimensions, 'localhost-second', 'abc')
    logging.info('%s', [t.to_dict() for t in task_to_run.TaskToRun.query()])
    self.assertEqual(2, run_result.try_number)
    self.assertEqual(
        (True, True),
        task_scheduler.bot_update_task(
            run_result.key, 'localhost-second', 'Foo1', 0, 0, 0.1, False, False,
            0.1, None))
    expected = {
      'abandoned_ts': None,
      'bot_dimensions': bot_dimensions,
      'bot_id': u'localhost-second',
      'bot_version': u'abc',
      'children_task_ids': [],
      'completed_ts': now_2,
      'costs_usd': [0., 0.1],
      'cost_saved_usd': None,
      'created_ts': self.now,
      'deduped_from': None,
      'durations': [0.1],
      'exit_codes': [0],
      'failure': False,
      'id': '1d69b9f088008810',
      'internal_failure': False,
      'modified_ts': now_2,
      'name': u'Request name',
      'outputs_ref': None,
      'properties_hash': None,
      'server_versions': [u'v1a'],
      'started_ts': now_2,
      'state': task_result.State.COMPLETED,
      'tags': [u'OS:Windows-3.1.1', u'priority:50', u'tag:1', u'user:Jesus'],
      'try_number': 2,
      'user': u'Jesus',
    }
    self.assertEqual(expected, run_result.result_summary_key.get().to_dict())
    self.assertEqual(0.1, run_result.key.get().cost_usd)