def test_cron_handle_bot_died_second(self): # Test two tries internal_failure's leading to a BOT_DIED status. self.mock(random, 'getrandbits', lambda _: 0x88) now = utils.utcnow() data = _gen_request( properties=dict(dimensions={u'OS': u'Windows-3.1.1'}), created_ts=now, expiration_ts=now+datetime.timedelta(seconds=600)) request = task_request.make_request(data, True) _result_summary = task_scheduler.schedule_request(request) bot_dimensions = { u'OS': [u'Windows', u'Windows-3.1.1'], u'hostname': u'localhost', u'foo': u'bar', } _request, run_result = task_scheduler.bot_reap_task( bot_dimensions, 'localhost', 'abc') self.assertEqual(1, run_result.try_number) self.assertEqual(task_result.State.RUNNING, run_result.state) self.mock_now(self.now + task_result.BOT_PING_TOLERANCE, 1) self.assertEqual(([], 1, 0), task_scheduler.cron_handle_bot_died('f.local')) now_1 = self.mock_now(self.now + task_result.BOT_PING_TOLERANCE, 2) # It must be a different bot. _request, run_result = task_scheduler.bot_reap_task( bot_dimensions, 'localhost-second', 'abc') now_2 = self.mock_now(self.now + 2 * task_result.BOT_PING_TOLERANCE, 3) self.assertEqual( (['1d69b9f088008812'], 0, 0), task_scheduler.cron_handle_bot_died('f.local')) self.assertEqual(([], 0, 0), task_scheduler.cron_handle_bot_died('f.local')) expected = { 'abandoned_ts': now_2, 'bot_dimensions': bot_dimensions, 'bot_id': u'localhost-second', 'bot_version': u'abc', 'children_task_ids': [], 'completed_ts': None, 'costs_usd': [0., 0.], 'cost_saved_usd': None, 'created_ts': self.now, 'deduped_from': None, 'durations': [], 'exit_codes': [], 'failure': False, 'id': '1d69b9f088008810', 'internal_failure': True, 'modified_ts': now_2, 'name': u'Request name', 'outputs_ref': None, 'properties_hash': None, 'server_versions': [u'v1a'], 'started_ts': now_1, 'state': task_result.State.BOT_DIED, 'tags': [u'OS:Windows-3.1.1', u'priority:50', u'tag:1', u'user:Jesus'], 'try_number': 2, 'user': u'Jesus', } self.assertEqual(expected, run_result.result_summary_key.get().to_dict())
def test_cron_handle_bot_died_second(self): # Test two tries internal_failure's leading to a BOT_DIED status. self.mock(random, 'getrandbits', lambda _: 0x88) now = utils.utcnow() data = _gen_request( properties=dict(dimensions={u'OS': u'Windows-3.1.1'}), created_ts=now, expiration_ts=now+datetime.timedelta(seconds=600)) request = task_request.make_request(data, True) _result_summary = task_scheduler.schedule_request(request) bot_dimensions = { u'OS': [u'Windows', u'Windows-3.1.1'], u'hostname': u'localhost', u'foo': u'bar', } _request, run_result = task_scheduler.bot_reap_task( bot_dimensions, 'localhost', 'abc') self.assertEqual(1, run_result.try_number) self.assertEqual(task_result.State.RUNNING, run_result.state) self.mock_now(self.now + task_result.BOT_PING_TOLERANCE, 1) self.assertEqual((0, 1, 0), task_scheduler.cron_handle_bot_died()) now_1 = self.mock_now(self.now + task_result.BOT_PING_TOLERANCE, 2) # It must be a different bot. _request, run_result = task_scheduler.bot_reap_task( bot_dimensions, 'localhost-second', 'abc') now_2 = self.mock_now(self.now + 2 * task_result.BOT_PING_TOLERANCE, 3) self.assertEqual((1, 0, 0), task_scheduler.cron_handle_bot_died()) self.assertEqual((0, 0, 0), task_scheduler.cron_handle_bot_died()) expected = { 'abandoned_ts': now_2, 'bot_dimensions': bot_dimensions, 'bot_id': u'localhost-second', 'bot_version': u'abc', 'children_task_ids': [], 'completed_ts': None, 'costs_usd': [0., 0.], 'cost_saved_usd': None, 'created_ts': self.now, 'deduped_from': None, 'durations': [], 'exit_codes': [], 'failure': False, 'id': '1d69b9f088008810', 'internal_failure': True, 'modified_ts': now_2, 'name': u'Request name', 'outputs_ref': None, 'properties_hash': None, 'server_versions': [u'v1a'], 'started_ts': now_1, 'state': task_result.State.BOT_DIED, 'tags': [u'OS:Windows-3.1.1', u'priority:50', u'tag:1', u'user:Jesus'], 'try_number': 2, 'user': u'Jesus', } self.assertEqual(expected, run_result.result_summary_key.get().to_dict())
def get(self): try: task_scheduler.cron_handle_bot_died() except db.NeedIndexError as e: # When a fresh new instance is deployed, it takes a few minutes for the # composite indexes to be created even if they are empty. Ignore the case # where the index is defined but still being created by AppEngine. if not str(e).startswith( 'NeedIndexError: The index for this query is not ready to serve.'): raise self.response.headers['Content-Type'] = 'text/plain; charset=utf-8' self.response.out.write('Success.')
def get(self): try: task_scheduler.cron_handle_bot_died(self.request.host_url) except datastore_errors.NeedIndexError as e: # When a fresh new instance is deployed, it takes a few minutes for the # composite indexes to be created even if they are empty. Ignore the case # where the index is defined but still being created by AppEngine. if not str(e).startswith( 'NeedIndexError: The index for this query is not ready to serve.'): raise self.response.headers['Content-Type'] = 'text/plain; charset=utf-8' self.response.out.write('Success.')
def test_cron_abort_expired_task_to_run_retry(self): self.mock(random, 'getrandbits', lambda _: 0x88) now = utils.utcnow() data = _gen_request( properties=dict(dimensions={u'OS': u'Windows-3.1.1'}), created_ts=now, expiration_ts=now+datetime.timedelta(seconds=600)) request = task_request.make_request(data, True) result_summary = task_scheduler.schedule_request(request) # Fake first try bot died. bot_dimensions = { u'OS': [u'Windows', u'Windows-3.1.1'], u'hostname': u'localhost', u'foo': u'bar', } _request, run_result = task_scheduler.bot_reap_task( bot_dimensions, 'localhost', 'abc') now_1 = self.mock_now(self.now + task_result.BOT_PING_TOLERANCE, 1) self.assertEqual((0, 1, 0), task_scheduler.cron_handle_bot_died()) self.assertEqual(task_result.State.BOT_DIED, run_result.key.get().state) self.assertEqual( task_result.State.PENDING, run_result.result_summary_key.get().state) # BOT_DIED is kept instead of EXPIRED. abandoned_ts = self.mock_now(self.now, request.expiration_secs+1) self.assertEqual(1, task_scheduler.cron_abort_expired_task_to_run()) self.assertEqual(1, len(task_result.TaskRunResult.query().fetch())) expected = { 'abandoned_ts': abandoned_ts, 'bot_dimensions': bot_dimensions, 'bot_id': u'localhost', 'bot_version': u'abc', 'children_task_ids': [], 'completed_ts': None, 'costs_usd': [0.], 'cost_saved_usd': None, 'created_ts': self.now, 'deduped_from': None, 'durations': [], 'exit_codes': [], 'failure': False, 'id': '1d69b9f088008810', 'internal_failure': True, 'modified_ts': abandoned_ts, 'name': u'Request name', 'outputs_ref': None, 'properties_hash': None, 'server_versions': [u'v1a'], 'started_ts': self.now, 'state': task_result.State.BOT_DIED, 'tags': [u'OS:Windows-3.1.1', u'priority:50', u'tag:1', u'user:Jesus'], 'try_number': 1, 'user': u'Jesus', } self.assertEqual(expected, result_summary.key.get().to_dict())
def test_cron_handle_bot_died_ignored_expired(self): self.mock(random, 'getrandbits', lambda _: 0x88) data = _gen_request_data( properties=dict(dimensions={u'OS': u'Windows-3.1.1'}), scheduling_expiration_secs=600) request = task_request.make_request(data) _result_summary = task_scheduler.schedule_request(request) bot_dimensions = { u'OS': [u'Windows', u'Windows-3.1.1'], u'hostname': u'localhost', u'foo': u'bar', } _request, run_result = task_scheduler.bot_reap_task( bot_dimensions, 'localhost', 'abc') self.assertEqual(1, run_result.try_number) self.assertEqual(task_result.State.RUNNING, run_result.state) self.mock_now(self.now + task_result.BOT_PING_TOLERANCE, 601) self.assertEqual((1, 0, 0), task_scheduler.cron_handle_bot_died())
def test_cron_handle_bot_died_ignored_expired(self): self.mock(random, 'getrandbits', lambda _: 0x88) now = utils.utcnow() data = _gen_request( properties=dict(dimensions={u'OS': u'Windows-3.1.1'}), created_ts=now, expiration_ts=now+datetime.timedelta(seconds=600)) request = task_request.make_request(data, True) _result_summary = task_scheduler.schedule_request(request) bot_dimensions = { u'OS': [u'Windows', u'Windows-3.1.1'], u'hostname': u'localhost', u'foo': u'bar', } _request, run_result = task_scheduler.bot_reap_task( bot_dimensions, 'localhost', 'abc') self.assertEqual(1, run_result.try_number) self.assertEqual(task_result.State.RUNNING, run_result.state) self.mock_now(self.now + task_result.BOT_PING_TOLERANCE, 601) self.assertEqual( (['1d69b9f088008811'], 0, 0), task_scheduler.cron_handle_bot_died('f.local'))
def test_cron_handle_bot_died_same_bot_denied(self): # Test first retry, then success. self.mock(random, 'getrandbits', lambda _: 0x88) now = utils.utcnow() data = _gen_request( properties=dict(dimensions={u'OS': u'Windows-3.1.1'}), created_ts=now, expiration_ts=now+datetime.timedelta(seconds=600)) request = task_request.make_request(data, True) _result_summary = task_scheduler.schedule_request(request) bot_dimensions = { u'OS': [u'Windows', u'Windows-3.1.1'], u'hostname': u'localhost', u'foo': u'bar', } _request, run_result = task_scheduler.bot_reap_task( bot_dimensions, 'localhost', 'abc') self.assertEqual(1, run_result.try_number) self.assertEqual(task_result.State.RUNNING, run_result.state) now_1 = self.mock_now(self.now + task_result.BOT_PING_TOLERANCE, 1) self.assertEqual(([], 1, 0), task_scheduler.cron_handle_bot_died('f.local')) # Refresh and compare: expected = { 'abandoned_ts': now_1, 'bot_dimensions': bot_dimensions, 'bot_id': u'localhost', 'bot_version': u'abc', 'children_task_ids': [], 'completed_ts': None, 'cost_usd': 0., 'durations': [], 'exit_codes': [], 'failure': False, 'id': '1d69b9f088008811', 'internal_failure': True, 'modified_ts': now_1, 'outputs_ref': None, 'server_versions': [u'v1a'], 'started_ts': self.now, 'state': task_result.State.BOT_DIED, 'try_number': 1, } self.assertEqual(expected, run_result.key.get().to_dict()) expected = { 'abandoned_ts': None, 'bot_dimensions': bot_dimensions, 'bot_id': u'localhost', 'bot_version': u'abc', 'children_task_ids': [], 'completed_ts': None, 'costs_usd': [0.], 'cost_saved_usd': None, 'created_ts': self.now, 'deduped_from': None, 'durations': [], 'exit_codes': [], 'failure': False, 'id': '1d69b9f088008810', 'internal_failure': False, 'modified_ts': now_1, 'name': u'Request name', 'outputs_ref': None, 'properties_hash': None, 'server_versions': [u'v1a'], 'started_ts': None, 'state': task_result.State.PENDING, 'tags': [u'OS:Windows-3.1.1', u'priority:50', u'tag:1', u'user:Jesus'], 'try_number': 1, 'user': u'Jesus', } self.assertEqual(expected, run_result.result_summary_key.get().to_dict()) # Task was retried but the same bot polls again, it's denied the task. now_2 = self.mock_now(self.now + task_result.BOT_PING_TOLERANCE, 2) request, run_result = task_scheduler.bot_reap_task( bot_dimensions, 'localhost', 'abc') self.assertEqual(None, request) self.assertEqual(None, run_result) logging.info('%s', [t.to_dict() for t in task_to_run.TaskToRun.query()])
def test_cron_handle_bot_died(self): # Test first retry, then success. self.mock(random, 'getrandbits', lambda _: 0x88) data = _gen_request_data( properties=dict(dimensions={u'OS': u'Windows-3.1.1'}), scheduling_expiration_secs=600) request = task_request.make_request(data) _result_summary = task_scheduler.schedule_request(request) bot_dimensions = { u'OS': [u'Windows', u'Windows-3.1.1'], u'hostname': u'localhost', u'foo': u'bar', } _request, run_result = task_scheduler.bot_reap_task( bot_dimensions, 'localhost', 'abc') self.assertEqual(1, run_result.try_number) self.assertEqual(task_result.State.RUNNING, run_result.state) now_1 = self.mock_now(self.now + task_result.BOT_PING_TOLERANCE, 1) self.assertEqual((0, 1, 0), task_scheduler.cron_handle_bot_died()) # Refresh and compare: expected = { 'abandoned_ts': now_1, 'bot_id': u'localhost', 'bot_version': u'abc', 'children_task_ids': [], 'completed_ts': None, 'cost_usd': 0., 'durations': [], 'exit_codes': [], 'failure': False, 'id': '1d69b9f088008811', 'internal_failure': True, 'modified_ts': now_1, 'server_versions': [u'v1a'], 'started_ts': self.now, 'state': task_result.State.BOT_DIED, 'try_number': 1, } self.assertEqual(expected, run_result.key.get().to_dict()) expected = { 'abandoned_ts': None, 'bot_id': u'localhost', 'bot_version': u'abc', 'children_task_ids': [], 'completed_ts': None, 'costs_usd': [0.], 'cost_saved_usd': None, 'created_ts': self.now, 'deduped_from': None, 'durations': [], 'exit_codes': [], 'failure': False, 'id': '1d69b9f088008810', 'internal_failure': False, 'modified_ts': now_1, 'name': u'Request name', 'properties_hash': None, 'server_versions': [u'v1a'], 'started_ts': None, 'state': task_result.State.PENDING, 'try_number': 1, 'user': u'Jesus', } self.assertEqual(expected, run_result.result_summary_key.get().to_dict()) # Task was retried. now_2 = self.mock_now(self.now + task_result.BOT_PING_TOLERANCE, 2) _request, run_result = task_scheduler.bot_reap_task( bot_dimensions, 'localhost-second', 'abc') logging.info('%s', [t.to_dict() for t in task_to_run.TaskToRun.query()]) self.assertEqual(2, run_result.try_number) self.assertEqual( (True, True), task_scheduler.bot_update_task( run_result.key, 'localhost-second', 'Foo1', 0, 0, 0.1, False, False, 0.1)) expected = { 'abandoned_ts': None, 'bot_id': u'localhost-second', 'bot_version': u'abc', 'children_task_ids': [], 'completed_ts': now_2, 'costs_usd': [0., 0.1], 'cost_saved_usd': None, 'created_ts': self.now, 'deduped_from': None, 'durations': [0.1], 'exit_codes': [0], 'failure': False, 'id': '1d69b9f088008810', 'internal_failure': False, 'modified_ts': now_2, 'name': u'Request name', 'properties_hash': None, 'server_versions': [u'v1a'], 'started_ts': now_2, 'state': task_result.State.COMPLETED, 'try_number': 2, 'user': u'Jesus', } self.assertEqual(expected, run_result.result_summary_key.get().to_dict()) self.assertEqual(0.1, run_result.key.get().cost_usd)
def run_cron(self): task_scheduler.cron_handle_bot_died()
def get(self): task_scheduler.cron_handle_bot_died() self.response.headers['Content-Type'] = 'text/plain; charset=utf-8' self.response.out.write('Success.')
def test_cron_handle_bot_died_same_bot_denied(self): # Test first retry, then success. self.mock(random, 'getrandbits', lambda _: 0x88) data = _gen_request_data( properties=dict(dimensions={u'OS': u'Windows-3.1.1'}), scheduling_expiration_secs=600) request = task_request.make_request(data) _result_summary = task_scheduler.schedule_request(request) bot_dimensions = { u'OS': [u'Windows', u'Windows-3.1.1'], u'hostname': u'localhost', u'foo': u'bar', } _request, run_result = task_scheduler.bot_reap_task( bot_dimensions, 'localhost', 'abc') self.assertEqual(1, run_result.try_number) self.assertEqual(task_result.State.RUNNING, run_result.state) now_1 = self.mock_now(self.now + task_result.BOT_PING_TOLERANCE, 1) self.assertEqual((0, 1, 0), task_scheduler.cron_handle_bot_died()) # Refresh and compare: expected = { 'abandoned_ts': now_1, 'bot_id': u'localhost', 'bot_version': u'abc', 'children_task_ids': [], 'completed_ts': None, 'cost_usd': 0., 'durations': [], 'exit_codes': [], 'failure': False, 'id': '1d69b9f088008811', 'internal_failure': True, 'modified_ts': now_1, 'server_versions': [u'v1a'], 'started_ts': self.now, 'state': task_result.State.BOT_DIED, 'try_number': 1, } self.assertEqual(expected, run_result.key.get().to_dict()) expected = { 'abandoned_ts': None, 'bot_id': u'localhost', 'bot_version': u'abc', 'children_task_ids': [], 'completed_ts': None, 'costs_usd': [0.], 'cost_saved_usd': None, 'created_ts': self.now, 'deduped_from': None, 'durations': [], 'exit_codes': [], 'failure': False, 'id': '1d69b9f088008810', 'internal_failure': False, 'modified_ts': now_1, 'name': u'Request name', 'properties_hash': None, 'server_versions': [u'v1a'], 'started_ts': None, 'state': task_result.State.PENDING, 'try_number': 1, 'user': u'Jesus', } self.assertEqual(expected, run_result.result_summary_key.get().to_dict()) # Task was retried but the same bot polls again, it's denied the task. now_2 = self.mock_now(self.now + task_result.BOT_PING_TOLERANCE, 2) request, run_result = task_scheduler.bot_reap_task( bot_dimensions, 'localhost', 'abc') self.assertEqual(None, request) self.assertEqual(None, run_result) logging.info('%s', [t.to_dict() for t in task_to_run.TaskToRun.query()])
def test_cron_handle_bot_died(self): # Test first retry, then success. self.mock(random, 'getrandbits', lambda _: 0x88) now = utils.utcnow() data = _gen_request( properties=dict(dimensions={u'OS': u'Windows-3.1.1'}), created_ts=now, expiration_ts=now+datetime.timedelta(seconds=600)) request = task_request.make_request(data, True) _result_summary = task_scheduler.schedule_request(request) bot_dimensions = { u'OS': [u'Windows', u'Windows-3.1.1'], u'hostname': u'localhost', u'foo': u'bar', } _request, run_result = task_scheduler.bot_reap_task( bot_dimensions, 'localhost', 'abc') self.assertEqual(1, run_result.try_number) self.assertEqual(task_result.State.RUNNING, run_result.state) now_1 = self.mock_now(self.now + task_result.BOT_PING_TOLERANCE, 1) self.assertEqual((0, 1, 0), task_scheduler.cron_handle_bot_died()) # Refresh and compare: expected = { 'abandoned_ts': now_1, 'bot_dimensions': bot_dimensions, 'bot_id': u'localhost', 'bot_version': u'abc', 'children_task_ids': [], 'completed_ts': None, 'cost_usd': 0., 'durations': [], 'exit_codes': [], 'failure': False, 'id': '1d69b9f088008811', 'internal_failure': True, 'modified_ts': now_1, 'outputs_ref': None, 'server_versions': [u'v1a'], 'started_ts': self.now, 'state': task_result.State.BOT_DIED, 'try_number': 1, } self.assertEqual(expected, run_result.key.get().to_dict()) expected = { 'abandoned_ts': None, 'bot_dimensions': bot_dimensions, 'bot_id': u'localhost', 'bot_version': u'abc', 'children_task_ids': [], 'completed_ts': None, 'costs_usd': [0.], 'cost_saved_usd': None, 'created_ts': self.now, 'deduped_from': None, 'durations': [], 'exit_codes': [], 'failure': False, 'id': '1d69b9f088008810', 'internal_failure': False, 'modified_ts': now_1, 'name': u'Request name', 'outputs_ref': None, 'properties_hash': None, 'server_versions': [u'v1a'], 'started_ts': None, 'state': task_result.State.PENDING, 'tags': [u'OS:Windows-3.1.1', u'priority:50', u'tag:1', u'user:Jesus'], 'try_number': 1, 'user': u'Jesus', } self.assertEqual(expected, run_result.result_summary_key.get().to_dict()) # Task was retried. now_2 = self.mock_now(self.now + task_result.BOT_PING_TOLERANCE, 2) _request, run_result = task_scheduler.bot_reap_task( bot_dimensions, 'localhost-second', 'abc') logging.info('%s', [t.to_dict() for t in task_to_run.TaskToRun.query()]) self.assertEqual(2, run_result.try_number) self.assertEqual( (True, True), task_scheduler.bot_update_task( run_result.key, 'localhost-second', 'Foo1', 0, 0, 0.1, False, False, 0.1, None)) expected = { 'abandoned_ts': None, 'bot_dimensions': bot_dimensions, 'bot_id': u'localhost-second', 'bot_version': u'abc', 'children_task_ids': [], 'completed_ts': now_2, 'costs_usd': [0., 0.1], 'cost_saved_usd': None, 'created_ts': self.now, 'deduped_from': None, 'durations': [0.1], 'exit_codes': [0], 'failure': False, 'id': '1d69b9f088008810', 'internal_failure': False, 'modified_ts': now_2, 'name': u'Request name', 'outputs_ref': None, 'properties_hash': None, 'server_versions': [u'v1a'], 'started_ts': now_2, 'state': task_result.State.COMPLETED, 'tags': [u'OS:Windows-3.1.1', u'priority:50', u'tag:1', u'user:Jesus'], 'try_number': 2, 'user': u'Jesus', } self.assertEqual(expected, run_result.result_summary_key.get().to_dict()) self.assertEqual(0.1, run_result.key.get().cost_usd)