def test_set_from_run_result_two_server_versions(self):
request = _gen_request()
result_summary = task_result.new_result_summary(request)
to_run = task_to_run.new_task_to_run(request, 1, 0)
run_result = task_result.new_run_result(request, to_run, 'localhost',
'abc', {})
run_result.started_ts = utils.utcnow()
self.assertTrue(result_summary.need_update_from_run_result(run_result))
result_summary.modified_ts = utils.utcnow()
run_result.modified_ts = utils.utcnow()
ndb.transaction(lambda: ndb.put_multi((result_summary, run_result)))
self.assertTrue(result_summary.need_update_from_run_result(run_result))
ndb.transaction(
lambda: result_summary.set_from_run_result(run_result, request))
ndb.transaction(lambda: ndb.put_multi([result_summary]))
run_result.signal_server_version('new-version')
run_result.modified_ts = utils.utcnow()
ndb.transaction(
lambda: result_summary.set_from_run_result(run_result, request))
ndb.transaction(lambda: ndb.put_multi((result_summary, run_result)))
self.assertEqual(['v1a', 'new-version'],
run_result.key.get().server_versions)
self.assertEqual(['v1a', 'new-version'],
result_summary.key.get().server_versions)
def test_set_from_run_result_two_tries(self):
request = task_request.make_request(_gen_request(), True)
result_summary = task_result.new_result_summary(request)
run_result_1 = task_result.new_run_result(
request, 1, 'localhost', 'abc', {})
run_result_2 = task_result.new_run_result(
request, 2, 'localhost', 'abc', {})
self.assertTrue(result_summary.need_update_from_run_result(run_result_1))
run_result_2.modified_ts = utils.utcnow()
result_summary.modified_ts = utils.utcnow()
ndb.transaction(lambda: ndb.put_multi((result_summary, run_result_2)))
self.assertTrue(result_summary.need_update_from_run_result(run_result_1))
run_result_1.modified_ts = utils.utcnow()
result_summary.set_from_run_result(run_result_1, request)
ndb.transaction(lambda: ndb.put_multi((result_summary, run_result_1)))
result_summary = result_summary.key.get()
self.assertFalse(result_summary.need_update_from_run_result(run_result_1))
self.assertTrue(result_summary.need_update_from_run_result(run_result_2))
run_result_2.modified_ts = utils.utcnow()
result_summary.set_from_run_result(run_result_2, request)
ndb.transaction(lambda: ndb.put_multi((result_summary, run_result_2)))
result_summary = result_summary.key.get()
self.assertEqual(2, result_summary.try_number)
self.assertFalse(result_summary.need_update_from_run_result(run_result_1))
def test_new_result_summary(self):
request = _gen_request()
actual = task_result.new_result_summary(request)
actual.modified_ts = self.now
# Trigger _pre_put_hook().
actual.put()
expected = self._gen_summary(modified_ts=self.now)
self.assertEqual(expected, actual.to_dict())
self.assertEqual(50, actual.request.priority)
self.assertEqual(True, actual.can_be_canceled)
actual.state = task_result.State.RUNNING
self.assertEqual(True, actual.can_be_canceled)
actual.state = task_result.State.TIMED_OUT
actual.duration = 0.1
self.assertEqual(False, actual.can_be_canceled)
actual.children_task_ids = [
'1d69ba3ea8008810',
'3d69ba3ea8008810',
'2d69ba3ea8008810',
]
actual.modified_ts = utils.utcnow()
ndb.transaction(actual.put)
expected = [
u'1d69ba3ea8008810', u'2d69ba3ea8008810', u'3d69ba3ea8008810'
]
self.assertEqual(expected, actual.key.get().children_task_ids)
def test_set_from_run_result_two_tries(self):
request = mkreq(_gen_request())
result_summary = task_result.new_result_summary(request)
run_result_1 = task_result.new_run_result(
request, 1, 'localhost', 'abc', {})
run_result_2 = task_result.new_run_result(
request, 2, 'localhost', 'abc', {})
self.assertTrue(result_summary.need_update_from_run_result(run_result_1))
run_result_2.modified_ts = utils.utcnow()
result_summary.modified_ts = utils.utcnow()
ndb.transaction(lambda: ndb.put_multi((result_summary, run_result_2)))
self.assertTrue(result_summary.need_update_from_run_result(run_result_1))
run_result_1.modified_ts = utils.utcnow()
result_summary.set_from_run_result(run_result_1, request)
ndb.transaction(lambda: ndb.put_multi((result_summary, run_result_1)))
result_summary = result_summary.key.get()
self.assertFalse(result_summary.need_update_from_run_result(run_result_1))
self.assertTrue(result_summary.need_update_from_run_result(run_result_2))
run_result_2.modified_ts = utils.utcnow()
result_summary.set_from_run_result(run_result_2, request)
ndb.transaction(lambda: ndb.put_multi((result_summary, run_result_2)))
result_summary = result_summary.key.get()
self.assertEqual(2, result_summary.try_number)
self.assertFalse(result_summary.need_update_from_run_result(run_result_1))
def setUp(self):
super(TestOutput, self).setUp()
request = task_request.make_request(_gen_request(), True)
result_summary = task_result.new_result_summary(request)
result_summary.modified_ts = utils.utcnow()
ndb.transaction(result_summary.put)
self.run_result = task_result.new_run_result(request, 1, "localhost", "abc", {})
self.run_result.modified_ts = utils.utcnow()
result_summary.set_from_run_result(self.run_result, request)
ndb.transaction(lambda: ndb.put_multi((result_summary, self.run_result)))
self.run_result = self.run_result.key.get()
def setUp(self): super(TestOutput, self).setUp() request = task_request.make_request(_gen_request_data()) result_summary = task_result.new_result_summary(request) result_summary.modified_ts = utils.utcnow() ndb.transaction(result_summary.put) self.run_result = task_result.new_run_result(request, 1, 'localhost', 'abc') self.run_result.modified_ts = utils.utcnow() result_summary.set_from_run_result(self.run_result, request) ndb.transaction(lambda: ndb.put_multi((result_summary, self.run_result))) self.run_result = self.run_result.key.get()
def test_new_result_summary(self):
request = task_request.make_request(_gen_request(), True)
actual = task_result.new_result_summary(request)
expected = {
'abandoned_ts': None,
'bot_dimensions': None,
'bot_id': None,
'bot_version': None,
'children_task_ids': [],
'completed_ts': None,
'costs_usd': [],
'cost_saved_usd': None,
'created_ts': self.now,
'deduped_from': None,
'duration': None,
'exit_code': None,
'failure': False,
'id': '1d69b9f088008810',
'internal_failure': False,
'modified_ts': None,
'name': u'Request name',
'outputs_ref': None,
'properties_hash': None,
'server_versions': [],
'started_ts': None,
'state': task_result.State.PENDING,
'tags': [
u'pool:default',
u'priority:50',
u'tag:1',
u'user:Jesus',
],
'try_number': None,
'user': u'Jesus',
}
self.assertEqual(expected, actual.to_dict())
self.assertEqual(50, actual.request.priority)
self.assertEqual(True, actual.can_be_canceled)
actual.state = task_result.State.RUNNING
self.assertEqual(False, actual.can_be_canceled)
actual.children_task_ids = [
'1d69ba3ea8008810',
'3d69ba3ea8008810',
'2d69ba3ea8008810',
]
actual.modified_ts = utils.utcnow()
ndb.transaction(actual.put)
expected = [
u'1d69ba3ea8008810', u'2d69ba3ea8008810', u'3d69ba3ea8008810'
]
self.assertEqual(expected, actual.key.get().children_task_ids)
def test_set_from_run_result(self):
request = mkreq(_gen_request())
result_summary = task_result.new_result_summary(request)
run_result = task_result.new_run_result(request, 1, 'localhost', 'abc', {})
self.assertTrue(result_summary.need_update_from_run_result(run_result))
result_summary.modified_ts = utils.utcnow()
run_result.modified_ts = utils.utcnow()
ndb.transaction(lambda: ndb.put_multi((result_summary, run_result)))
self.assertTrue(result_summary.need_update_from_run_result(run_result))
result_summary.set_from_run_result(run_result, request)
ndb.transaction(lambda: ndb.put_multi([result_summary]))
self.assertFalse(result_summary.need_update_from_run_result(run_result))
def test_set_from_run_result(self):
request = task_request.make_request(_gen_request(), True)
result_summary = task_result.new_result_summary(request)
run_result = task_result.new_run_result(request, 1, 'localhost', 'abc', {})
self.assertTrue(result_summary.need_update_from_run_result(run_result))
result_summary.modified_ts = utils.utcnow()
run_result.modified_ts = utils.utcnow()
ndb.transaction(lambda: ndb.put_multi((result_summary, run_result)))
self.assertTrue(result_summary.need_update_from_run_result(run_result))
result_summary.set_from_run_result(run_result, request)
ndb.transaction(lambda: ndb.put_multi([result_summary]))
self.assertFalse(result_summary.need_update_from_run_result(run_result))
def test_run_result_timeout(self):
request = task_request.make_request(_gen_request(), True)
result_summary = task_result.new_result_summary(request)
result_summary.modified_ts = utils.utcnow()
ndb.transaction(result_summary.put)
run_result = task_result.new_run_result(request, 1, 'localhost', 'abc', {})
run_result.state = task_result.State.TIMED_OUT
run_result.completed_ts = utils.utcnow()
run_result.modified_ts = utils.utcnow()
result_summary.set_from_run_result(run_result, request)
ndb.transaction(lambda: ndb.put_multi((run_result, result_summary)))
run_result = run_result.key.get()
result_summary = result_summary.key.get()
self.assertEqual(True, run_result.failure)
self.assertEqual(True, result_summary.failure)
def test_run_result_timeout(self): request = task_request.make_request(_gen_request_data()) result_summary = task_result.new_result_summary(request) result_summary.modified_ts = utils.utcnow() ndb.transaction(result_summary.put) run_result = task_result.new_run_result(request, 1, 'localhost', 'abc') run_result.state = task_result.State.TIMED_OUT run_result.completed_ts = utils.utcnow() run_result.modified_ts = utils.utcnow() result_summary.set_from_run_result(run_result, request) ndb.transaction(lambda: ndb.put_multi((run_result, result_summary))) run_result = run_result.key.get() result_summary = result_summary.key.get() self.assertEqual(True, run_result.failure) self.assertEqual(True, result_summary.failure)
def test_new_result_summary(self):
request = task_request.make_request(_gen_request(), True)
actual = task_result.new_result_summary(request)
expected = {
'abandoned_ts': None,
'bot_dimensions': None,
'bot_id': None,
'bot_version': None,
'children_task_ids': [],
'completed_ts': None,
'costs_usd': [],
'cost_saved_usd': None,
'created_ts': self.now,
'deduped_from': None,
'duration': None,
'exit_code': None,
'failure': False,
'id': '1d69b9f088008810',
'internal_failure': False,
'modified_ts': None,
'name': u'Request name',
'outputs_ref': None,
'properties_hash': None,
'server_versions': [],
'started_ts': None,
'state': task_result.State.PENDING,
'tags': [
u'pool:default',
u'priority:50',
u'tag:1',
u'user:Jesus',
],
'try_number': None,
'user': u'Jesus',
}
self.assertEqual(expected, actual.to_dict())
self.assertEqual(50, actual.request.priority)
self.assertEqual(True, actual.can_be_canceled)
actual.state = task_result.State.RUNNING
self.assertEqual(False, actual.can_be_canceled)
actual.children_task_ids = [
'1d69ba3ea8008810', '3d69ba3ea8008810', '2d69ba3ea8008810',
]
actual.modified_ts = utils.utcnow()
ndb.transaction(actual.put)
expected = [u'1d69ba3ea8008810', u'2d69ba3ea8008810', u'3d69ba3ea8008810']
self.assertEqual(expected, actual.key.get().children_task_ids)
def test_yield_run_result_keys_with_dead_bot(self):
request = task_request.make_request(_gen_request(), True)
result_summary = task_result.new_result_summary(request)
result_summary.modified_ts = utils.utcnow()
ndb.transaction(result_summary.put)
run_result = task_result.new_run_result(request, 1, "localhost", "abc", {})
run_result.completed_ts = utils.utcnow()
run_result.modified_ts = utils.utcnow()
result_summary.set_from_run_result(run_result, request)
ndb.transaction(lambda: ndb.put_multi((run_result, result_summary)))
self.mock_now(self.now + task_result.BOT_PING_TOLERANCE)
self.assertEqual([], list(task_result.yield_run_result_keys_with_dead_bot()))
self.mock_now(self.now + task_result.BOT_PING_TOLERANCE, 1)
self.assertEqual([run_result.key], list(task_result.yield_run_result_keys_with_dead_bot()))
def setUp(self):
super(TestOutput, self).setUp()
request = _gen_request()
result_summary = task_result.new_result_summary(request)
result_summary.modified_ts = utils.utcnow()
ndb.transaction(result_summary.put)
to_run = task_to_run.new_task_to_run(request, 1, 0)
self.run_result = task_result.new_run_result(request, to_run,
'localhost', 'abc', {})
self.run_result.started_ts = result_summary.modified_ts
self.run_result.modified_ts = utils.utcnow()
ndb.transaction(lambda: result_summary.set_from_run_result(
self.run_result, request))
ndb.transaction(lambda: ndb.put_multi(
(result_summary, self.run_result)))
self.run_result = self.run_result.key.get()
def test_set_from_run_result_two_server_versions(self):
request = task_request.make_request(_gen_request(), True)
result_summary = task_result.new_result_summary(request)
run_result = task_result.new_run_result(request, 1, "localhost", "abc", {})
self.assertTrue(result_summary.need_update_from_run_result(run_result))
result_summary.modified_ts = utils.utcnow()
run_result.modified_ts = utils.utcnow()
ndb.transaction(lambda: ndb.put_multi((result_summary, run_result)))
self.assertTrue(result_summary.need_update_from_run_result(run_result))
result_summary.set_from_run_result(run_result, request)
ndb.transaction(lambda: ndb.put_multi([result_summary]))
run_result.signal_server_version("new-version")
run_result.modified_ts = utils.utcnow()
result_summary.set_from_run_result(run_result, request)
ndb.transaction(lambda: ndb.put_multi((result_summary, run_result)))
self.assertEqual(["v1a", "new-version"], run_result.key.get().server_versions)
self.assertEqual(["v1a", "new-version"], result_summary.key.get().server_versions)
def test_yield_run_result_keys_with_dead_bot(self):
request = mkreq(_gen_request())
result_summary = task_result.new_result_summary(request)
result_summary.modified_ts = utils.utcnow()
ndb.transaction(result_summary.put)
run_result = task_result.new_run_result(request, 1, 'localhost', 'abc', {})
run_result.completed_ts = utils.utcnow()
run_result.modified_ts = utils.utcnow()
result_summary.set_from_run_result(run_result, request)
ndb.transaction(lambda: ndb.put_multi((run_result, result_summary)))
self.mock_now(self.now + task_result.BOT_PING_TOLERANCE)
self.assertEqual(
[], list(task_result.yield_run_result_keys_with_dead_bot()))
self.mock_now(self.now + task_result.BOT_PING_TOLERANCE, 1)
self.assertEqual(
[run_result.key],
list(task_result.yield_run_result_keys_with_dead_bot()))
def test_new_result_summary(self):
request = task_request.make_request(_gen_request(), True)
actual = task_result.new_result_summary(request)
expected = {
"abandoned_ts": None,
"bot_dimensions": None,
"bot_id": None,
"bot_version": None,
"children_task_ids": [],
"completed_ts": None,
"costs_usd": [],
"cost_saved_usd": None,
"created_ts": self.now,
"deduped_from": None,
"durations": [],
"exit_codes": [],
"failure": False,
"id": "1d69b9f088008810",
"internal_failure": False,
"modified_ts": None,
"name": u"Request name",
"outputs_ref": None,
"properties_hash": None,
"server_versions": [],
"started_ts": None,
"state": task_result.State.PENDING,
"tags": [u"priority:50", u"tag:1", u"user:Jesus"],
"try_number": None,
"user": u"Jesus",
}
self.assertEqual(expected, actual.to_dict())
self.assertEqual(50, actual.priority)
self.assertEqual(True, actual.can_be_canceled)
actual.state = task_result.State.RUNNING
self.assertEqual(False, actual.can_be_canceled)
actual.children_task_ids = ["1d69ba3ea8008810", "3d69ba3ea8008810", "2d69ba3ea8008810"]
actual.modified_ts = utils.utcnow()
ndb.transaction(actual.put)
expected = [u"1d69ba3ea8008810", u"2d69ba3ea8008810", u"3d69ba3ea8008810"]
self.assertEqual(expected, actual.key.get().children_task_ids)
def test_run_result_timeout(self):
request = _gen_request()
result_summary = task_result.new_result_summary(request)
result_summary.modified_ts = utils.utcnow()
ndb.transaction(result_summary.put)
to_run = task_to_run.new_task_to_run(request, 1, 0)
run_result = task_result.new_run_result(request, to_run, 'localhost',
'abc', {})
run_result.state = task_result.State.TIMED_OUT
run_result.duration = 0.1
run_result.exit_code = -1
run_result.started_ts = utils.utcnow()
run_result.completed_ts = run_result.started_ts
run_result.modified_ts = run_result.started_ts
ndb.transaction(
lambda: result_summary.set_from_run_result(run_result, request))
ndb.transaction(lambda: ndb.put_multi((run_result, result_summary)))
run_result = run_result.key.get()
result_summary = result_summary.key.get()
self.assertEqual(True, run_result.failure)
self.assertEqual(True, result_summary.failure)
def test_set_from_run_result_two_server_versions(self):
request = task_request.make_request(_gen_request_data())
result_summary = task_result.new_result_summary(request)
run_result = task_result.new_run_result(request, 1, 'localhost', 'abc')
self.assertTrue(result_summary.need_update_from_run_result(run_result))
result_summary.modified_ts = utils.utcnow()
run_result.modified_ts = utils.utcnow()
ndb.transaction(lambda: ndb.put_multi((result_summary, run_result)))
self.assertTrue(result_summary.need_update_from_run_result(run_result))
result_summary.set_from_run_result(run_result, request)
ndb.transaction(lambda: ndb.put_multi([result_summary]))
run_result.signal_server_version('new-version')
run_result.modified_ts = utils.utcnow()
result_summary.set_from_run_result(run_result, request)
ndb.transaction(lambda: ndb.put_multi((result_summary, run_result)))
self.assertEqual(
['v1a', 'new-version'], run_result.key.get().server_versions)
self.assertEqual(
['v1a', 'new-version'], result_summary.key.get().server_versions)
def test_integration(self):
# Creates a TaskRequest, along its TaskResultSummary and TaskToRun. Have a
# bot reap the task, and complete the task. Ensure the resulting
# TaskResultSummary and TaskRunResult are properly updated.
request = task_request.make_request(_gen_request(), True)
result_summary = task_result.new_result_summary(request)
to_run = task_to_run.new_task_to_run(request)
result_summary.modified_ts = utils.utcnow()
ndb.transaction(lambda: ndb.put_multi([result_summary, to_run]))
expected = {
'abandoned_ts': None,
'bot_dimensions': None,
'bot_id': None,
'bot_version': None,
'children_task_ids': [],
'completed_ts': None,
'costs_usd': [],
'cost_saved_usd': None,
'created_ts': self.now,
'deduped_from': None,
'durations': [],
'exit_codes': [],
'failure': False,
'id': '1d69b9f088008810',
'internal_failure': False,
'modified_ts': self.now,
'name': u'Request name',
'outputs_ref': None,
'properties_hash': None,
'server_versions': [],
'started_ts': None,
'state': task_result.State.PENDING,
'try_number': None,
'tags': [u'priority:50', u'tag:1', u'user:Jesus'],
'user': u'Jesus',
}
self.assertEqual(expected, result_summary.to_dict())
# Nothing changed 2 secs later except latency.
self.mock_now(self.now, 2)
self.assertEqual(expected, result_summary.to_dict())
# Task is reaped after 2 seconds (4 secs total).
reap_ts = self.now + datetime.timedelta(seconds=4)
self.mock_now(reap_ts)
to_run.queue_number = None
to_run.put()
run_result = task_result.new_run_result(request, 1, 'localhost', 'abc', {})
run_result.modified_ts = utils.utcnow()
result_summary.set_from_run_result(run_result, request)
ndb.transaction(lambda: ndb.put_multi((result_summary, run_result)))
expected = {
'abandoned_ts': None,
'bot_dimensions': {},
'bot_id': u'localhost',
'bot_version': u'abc',
'children_task_ids': [],
'completed_ts': None,
'costs_usd': [0.],
'cost_saved_usd': None,
'created_ts': self.now,
'deduped_from': None,
'durations': [],
'exit_codes': [],
'failure': False,
'id': '1d69b9f088008810',
'internal_failure': False,
'modified_ts': reap_ts,
'name': u'Request name',
'outputs_ref': None,
'properties_hash': None,
'server_versions': [u'v1a'],
'started_ts': reap_ts,
'state': task_result.State.RUNNING,
'tags': [u'priority:50', u'tag:1', u'user:Jesus'],
'try_number': 1,
'user': u'Jesus',
}
self.assertEqual(expected, result_summary.key.get().to_dict())
# Task completed after 2 seconds (6 secs total), the task has been running
# for 2 seconds.
complete_ts = self.now + datetime.timedelta(seconds=6)
self.mock_now(complete_ts)
run_result.completed_ts = complete_ts
run_result.exit_codes.append(0)
run_result.state = task_result.State.COMPLETED
run_result.modified_ts = utils.utcnow()
ndb.transaction(
lambda: ndb.put_multi(run_result.append_output(0, 'foo', 0)))
result_summary.set_from_run_result(run_result, request)
ndb.transaction(lambda: ndb.put_multi((result_summary, run_result)))
expected = {
'abandoned_ts': None,
'bot_dimensions': {},
'bot_id': u'localhost',
'bot_version': u'abc',
'children_task_ids': [],
'completed_ts': complete_ts,
'costs_usd': [0.],
'cost_saved_usd': None,
'created_ts': self.now,
'deduped_from': None,
'durations': [],
'exit_codes': [0],
'failure': False,
'id': '1d69b9f088008810',
'internal_failure': False,
'modified_ts': complete_ts,
'name': u'Request name',
'outputs_ref': None,
'properties_hash': None,
'server_versions': [u'v1a'],
'started_ts': reap_ts,
'state': task_result.State.COMPLETED,
'tags': [u'priority:50', u'tag:1', u'user:Jesus'],
'try_number': 1,
'user': u'Jesus',
}
self.assertEqual(expected, result_summary.key.get().to_dict())
self.assertEqual(['foo'], list(result_summary.get_outputs()))
self.assertEqual(
datetime.timedelta(seconds=2), result_summary.duration_total)
self.assertEqual(
datetime.timedelta(seconds=2),
result_summary.duration_now(utils.utcnow()))
self.assertEqual(
datetime.timedelta(seconds=4), result_summary.pending)
self.assertEqual(
datetime.timedelta(seconds=4),
result_summary.pending_now(utils.utcnow()))
self.assertEqual(
task_pack.pack_result_summary_key(result_summary.key),
result_summary.task_id)
self.assertEqual(complete_ts, result_summary.ended_ts)
self.assertEqual(
task_pack.pack_run_result_key(run_result.key),
run_result.task_id)
self.assertEqual(complete_ts, run_result.ended_ts)
def schedule_request(request):
"""Creates and stores all the entities to schedule a new task request.
The number of entities created is 3: TaskRequest, TaskResultSummary and
TaskToRun.
The TaskRequest is saved first as a DB transaction, then TaskResultSummary and
TaskToRun are saved as a single DB RPC. The Search index is also updated
in-between.
Arguments:
- request: is in the TaskRequest entity saved in the DB.
Returns:
TaskResultSummary. TaskToRun is not returned.
"""
dupe_future = None
if request.properties.idempotent:
# Find a previously run task that is also idempotent and completed. Start a
# query to fetch items that can be used to dedupe the task. See the comment
# for this property for more details.
#
# Do not use "cls.created_ts > oldest" here because this would require a
# composite index. It's unnecessary because TaskRequest.key is mostly
# equivalent to decreasing TaskRequest.created_ts, ordering by key works as
# well and doesn't require a composite index.
cls = task_result.TaskResultSummary
h = request.properties.properties_hash
dupe_future = cls.query(cls.properties_hash == h).order(cls.key).get_async()
# At this point, the request is now in the DB but not yet in a mode where it
# can be triggered or visible. Index it right away so it is searchable. If any
# of remaining calls in this function fail, the TaskRequest and Search
# Document will simply point to an incomplete task, which will be ignored.
#
# Creates the entities TaskToRun and TaskResultSummary but do not save them
# yet. TaskRunResult will be created once a bot starts it.
task = task_to_run.new_task_to_run(request)
result_summary = task_result.new_result_summary(request)
# Do not specify a doc_id, as they are guaranteed to be monotonically
# increasing and searches are done in reverse order, which fits exactly the
# created_ts ordering. This is useful because DateField is precise to the date
# (!) and NumberField is signed 32 bits so the best it could do with EPOCH is
# second resolution up to year 2038.
index = search.Index(name="requests")
packed = task_pack.pack_result_summary_key(result_summary.key)
doc = search.Document(
fields=[search.TextField(name="name", value=request.name), search.AtomField(name="id", value=packed)]
)
# Even if it fails here, we're still fine, as the task is not "alive" yet.
search_future = index.put_async([doc])
now = utils.utcnow()
if dupe_future:
# Reuse the results!
dupe_summary = dupe_future.get_result()
# Refuse tasks older than X days. This is due to the isolate server dropping
# files. https://code.google.com/p/swarming/issues/detail?id=197
oldest = now - datetime.timedelta(seconds=config.settings().reusable_task_age_secs)
if dupe_summary and dupe_summary.created_ts > oldest:
# If there's a bug, commenting out this block is sufficient to disable the
# functionality.
# Setting task.queue_number to None removes it from the scheduling.
task.queue_number = None
_copy_entity(dupe_summary, result_summary, ("created_ts", "name", "user", "tags"))
result_summary.properties_hash = None
result_summary.try_number = 0
result_summary.cost_saved_usd = result_summary.cost_usd
# Only zap after.
result_summary.costs_usd = []
result_summary.deduped_from = task_pack.pack_run_result_key(dupe_summary.run_result_key)
# Get parent task details if applicable.
parent_task_keys = None
if request.parent_task_id:
parent_run_key = task_pack.unpack_run_result_key(request.parent_task_id)
parent_task_keys = [parent_run_key, task_pack.run_result_key_to_result_summary_key(parent_run_key)]
result_summary.modified_ts = now
# Storing these entities makes this task live. It is important at this point
# that the HTTP handler returns as fast as possible, otherwise the task will
# be run but the client will not know about it.
def run():
ndb.put_multi([result_summary, task])
def run_parent():
# This one is slower.
items = ndb.get_multi(parent_task_keys)
k = result_summary.task_id
for item in items:
item.children_task_ids.append(k)
item.modified_ts = now
ndb.put_multi(items)
# Raising will abort to the caller.
futures = [datastore_utils.transaction_async(run)]
if parent_task_keys:
futures.append(datastore_utils.transaction_async(run_parent))
try:
search_future.get_result()
except search.Error:
# Do not abort the task, for now search is best effort.
logging.exception("Put failed")
for future in futures:
# Check for failures, it would raise in this case, aborting the call.
future.get_result()
stats.add_task_entry(
"task_enqueued", result_summary.key, dimensions=request.properties.dimensions, user=request.user
)
return result_summary
def schedule_request(request, check_acls=True):
"""Creates and stores all the entities to schedule a new task request.
Checks ACLs first. Raises auth.AuthorizationError if caller is not authorized
to post this request.
The number of entities created is 3: TaskRequest, TaskToRun and
TaskResultSummary.
All 3 entities in the same entity group (TaskReqest, TaskToRun,
TaskResultSummary) are saved as a DB transaction.
Arguments:
- request: TaskRequest entity to be saved in the DB. It's key must not be set
and the entity must not be saved in the DB yet.
- check_acls: Whether the request should check ACLs.
Returns:
TaskResultSummary. TaskToRun is not returned.
"""
assert isinstance(request, task_request.TaskRequest), request
assert not request.key, request.key
# Raises AuthorizationError with helpful message if the request.authorized
# can't use some of the requested dimensions.
if check_acls:
_check_dimension_acls(request)
now = utils.utcnow()
request.key = task_request.new_request_key()
task = task_to_run.new_task_to_run(request)
result_summary = task_result.new_result_summary(request)
result_summary.modified_ts = now
def get_new_keys():
# Warning: this assumes knowledge about the hierarchy of each entity.
key = task_request.new_request_key()
task.key.parent = key
old = result_summary.task_id
result_summary.parent = key
logging.info('%s conflicted, using %s', old, result_summary.task_id)
return key
deduped = False
if request.properties.idempotent:
dupe_summary = _find_dupe_task(now, request.properties_hash)
if dupe_summary:
# Setting task.queue_number to None removes it from the scheduling.
task.queue_number = None
_copy_summary(
dupe_summary, result_summary,
('created_ts', 'modified_ts', 'name', 'user', 'tags'))
# Zap irrelevant properties. PerformanceStats is also not copied over,
# since it's not relevant.
result_summary.properties_hash = None
result_summary.try_number = 0
result_summary.cost_saved_usd = result_summary.cost_usd
# Only zap after.
result_summary.costs_usd = []
result_summary.deduped_from = task_pack.pack_run_result_key(
dupe_summary.run_result_key)
# In this code path, there's not much to do as the task will not be run,
# previous results are returned. We still need to store all the entities
# correctly.
datastore_utils.insert(request,
get_new_keys,
extra=[task, result_summary])
logging.debug('New request %s reusing %s', result_summary.task_id,
dupe_summary.task_id)
deduped = True
if not deduped:
# Storing these entities makes this task live. It is important at this point
# that the HTTP handler returns as fast as possible, otherwise the task will
# be run but the client will not know about it.
datastore_utils.insert(request,
get_new_keys,
extra=[task, result_summary])
logging.debug('New request %s', result_summary.task_id)
# Get parent task details if applicable.
if request.parent_task_id:
parent_run_key = task_pack.unpack_run_result_key(
request.parent_task_id)
parent_task_keys = [
parent_run_key,
task_pack.run_result_key_to_result_summary_key(parent_run_key),
]
def run_parent():
# This one is slower.
items = ndb.get_multi(parent_task_keys)
k = result_summary.task_id
for item in items:
item.children_task_ids.append(k)
item.modified_ts = now
ndb.put_multi(items)
# Raising will abort to the caller. There's a risk that for tasks with
# parent tasks, the task will be lost due to this transaction.
# TODO(maruel): An option is to update the parent task as part of a cron
# job, which would remove this code from the critical path.
datastore_utils.transaction(run_parent)
stats.add_task_entry('task_enqueued',
result_summary.key,
dimensions=request.properties.dimensions,
user=request.user)
ts_mon_metrics.update_jobs_requested_metrics(result_summary, deduped)
return result_summary
def test_integration(self):
# Creates a TaskRequest, along its TaskResultSummary and TaskToRun. Have a
# bot reap the task, and complete the task. Ensure the resulting
# TaskResultSummary and TaskRunResult are properly updated.
request = _gen_request()
result_summary = task_result.new_result_summary(request)
to_run = task_to_run.new_task_to_run(request, 1, 0)
result_summary.modified_ts = utils.utcnow()
ndb.transaction(lambda: ndb.put_multi([result_summary, to_run]))
expected = self._gen_summary(modified_ts=self.now)
self.assertEqual(expected, result_summary.to_dict())
# Nothing changed 2 secs later except latency.
self.mock_now(self.now, 2)
self.assertEqual(expected, result_summary.to_dict())
# Task is reaped after 2 seconds (4 secs total).
reap_ts = self.now + datetime.timedelta(seconds=4)
self.mock_now(reap_ts)
to_run.queue_number = None
to_run.put()
run_result = task_result.new_run_result(request, to_run, u'localhost',
u'abc', {})
run_result.started_ts = utils.utcnow()
run_result.modified_ts = run_result.started_ts
ndb.transaction(
lambda: result_summary.set_from_run_result(run_result, request))
ndb.transaction(lambda: ndb.put_multi((result_summary, run_result)))
expected = self._gen_summary(bot_dimensions={},
bot_version=u'abc',
bot_id=u'localhost',
costs_usd=[0.],
modified_ts=reap_ts,
state=task_result.State.RUNNING,
started_ts=reap_ts,
try_number=1)
self.assertEqual(expected, result_summary.key.get().to_dict())
# Task completed after 2 seconds (6 secs total), the task has been running
# for 2 seconds.
complete_ts = self.now + datetime.timedelta(seconds=6)
self.mock_now(complete_ts)
run_result.completed_ts = complete_ts
run_result.duration = 0.1
run_result.exit_code = 0
run_result.state = task_result.State.COMPLETED
run_result.modified_ts = utils.utcnow()
task_result.PerformanceStats(
key=task_pack.run_result_key_to_performance_stats_key(
run_result.key),
bot_overhead=0.1,
isolated_download=task_result.OperationStats(
duration=0.05,
initial_number_items=10,
initial_size=10000,
items_cold=large.pack([1, 2]),
items_hot=large.pack([3, 4, 5])),
isolated_upload=task_result.OperationStats(duration=0.01,
items_cold=large.pack(
[10]))).put()
ndb.transaction(
lambda: ndb.put_multi(run_result.append_output('foo', 0)))
ndb.transaction(
lambda: result_summary.set_from_run_result(run_result, request))
ndb.transaction(lambda: ndb.put_multi((result_summary, run_result)))
expected = self._gen_summary(bot_dimensions={},
bot_version=u'abc',
bot_id=u'localhost',
completed_ts=complete_ts,
costs_usd=[0.],
duration=0.1,
exit_code=0,
modified_ts=complete_ts,
state=task_result.State.COMPLETED,
started_ts=reap_ts,
try_number=1)
self.assertEqual(expected, result_summary.key.get().to_dict())
expected = {
'bot_overhead': 0.1,
'isolated_download': {
'duration': 0.05,
'initial_number_items': 10,
'initial_size': 10000,
'items_cold': large.pack([1, 2]),
'items_hot': large.pack([3, 4, 5]),
'num_items_cold': 2,
'total_bytes_items_cold': 3,
'num_items_hot': 3,
'total_bytes_items_hot': 12,
},
'isolated_upload': {
'duration': 0.01,
'initial_number_items': None,
'initial_size': None,
'items_cold': large.pack([10]),
'items_hot': None,
'num_items_cold': 1,
'total_bytes_items_cold': 10,
'num_items_hot': None,
'total_bytes_items_hot': None,
},
'package_installation': {
'duration': None,
'initial_number_items': None,
'initial_size': None,
'items_cold': None,
'items_hot': None,
'num_items_cold': None,
'total_bytes_items_cold': None,
'num_items_hot': None,
'total_bytes_items_hot': None,
},
}
self.assertEqual(expected, result_summary.performance_stats.to_dict())
self.assertEqual('foo', result_summary.get_output())
self.assertEqual(datetime.timedelta(seconds=2),
result_summary.duration_as_seen_by_server)
self.assertEqual(datetime.timedelta(seconds=0.1),
result_summary.duration_now(utils.utcnow()))
self.assertEqual(datetime.timedelta(seconds=4), result_summary.pending)
self.assertEqual(datetime.timedelta(seconds=4),
result_summary.pending_now(utils.utcnow()))
self.assertEqual(task_pack.pack_result_summary_key(result_summary.key),
result_summary.task_id)
self.assertEqual(complete_ts, result_summary.ended_ts)
self.assertEqual(task_pack.pack_run_result_key(run_result.key),
run_result.task_id)
self.assertEqual(complete_ts, run_result.ended_ts)
def test_integration(self):
# Creates a TaskRequest, along its TaskResultSummary and TaskToRun. Have a
# bot reap the task, and complete the task. Ensure the resulting
# TaskResultSummary and TaskRunResult are properly updated.
request = mkreq(_gen_request())
result_summary = task_result.new_result_summary(request)
to_run = task_to_run.new_task_to_run(request)
result_summary.modified_ts = utils.utcnow()
ndb.transaction(lambda: ndb.put_multi([result_summary, to_run]))
expected = {
'abandoned_ts': None,
'bot_dimensions': None,
'bot_id': None,
'bot_version': None,
'cipd_pins': None,
'children_task_ids': [],
'completed_ts': None,
'costs_usd': [],
'cost_saved_usd': None,
'created_ts': self.now,
'deduped_from': None,
'duration': None,
'exit_code': None,
'failure': False,
'id': '1d69b9f088008810',
'internal_failure': False,
'modified_ts': self.now,
'name': u'Request name',
'outputs_ref': None,
'properties_hash': None,
'server_versions': [],
'started_ts': None,
'state': task_result.State.PENDING,
'try_number': None,
'tags': [
u'pool:default',
u'priority:50',
u'service_account:none',
u'tag:1',
u'user:Jesus',
],
'user': u'Jesus',
}
self.assertEqual(expected, result_summary.to_dict())
# Nothing changed 2 secs later except latency.
self.mock_now(self.now, 2)
self.assertEqual(expected, result_summary.to_dict())
# Task is reaped after 2 seconds (4 secs total).
reap_ts = self.now + datetime.timedelta(seconds=4)
self.mock_now(reap_ts)
to_run.queue_number = None
to_run.put()
run_result = task_result.new_run_result(request, 1, 'localhost', 'abc', {})
run_result.modified_ts = utils.utcnow()
result_summary.set_from_run_result(run_result, request)
ndb.transaction(lambda: ndb.put_multi((result_summary, run_result)))
expected = {
'abandoned_ts': None,
'bot_dimensions': {},
'bot_id': u'localhost',
'bot_version': u'abc',
'cipd_pins': None,
'children_task_ids': [],
'completed_ts': None,
'costs_usd': [0.],
'cost_saved_usd': None,
'created_ts': self.now,
'deduped_from': None,
'duration': None,
'exit_code': None,
'failure': False,
'id': '1d69b9f088008810',
'internal_failure': False,
'modified_ts': reap_ts,
'name': u'Request name',
'outputs_ref': None,
'properties_hash': None,
'server_versions': [u'v1a'],
'started_ts': reap_ts,
'state': task_result.State.RUNNING,
'tags': [
u'pool:default',
u'priority:50',
u'service_account:none',
u'tag:1',
u'user:Jesus',
],
'try_number': 1,
'user': u'Jesus',
}
self.assertEqual(expected, result_summary.key.get().to_dict())
# Task completed after 2 seconds (6 secs total), the task has been running
# for 2 seconds.
complete_ts = self.now + datetime.timedelta(seconds=6)
self.mock_now(complete_ts)
run_result.completed_ts = complete_ts
run_result.duration = 0.1
run_result.exit_code = 0
run_result.state = task_result.State.COMPLETED
run_result.modified_ts = utils.utcnow()
task_result.PerformanceStats(
key=task_pack.run_result_key_to_performance_stats_key(run_result.key),
bot_overhead=0.1,
isolated_download=task_result.OperationStats(
duration=0.05, initial_number_items=10, initial_size=10000,
items_cold='foo', items_hot='bar'),
isolated_upload=task_result.OperationStats(
duration=0.01, items_cold='foo')).put()
ndb.transaction(lambda: ndb.put_multi(run_result.append_output('foo', 0)))
result_summary.set_from_run_result(run_result, request)
ndb.transaction(lambda: ndb.put_multi((result_summary, run_result)))
expected = {
'abandoned_ts': None,
'bot_dimensions': {},
'bot_id': u'localhost',
'bot_version': u'abc',
'cipd_pins': None,
'children_task_ids': [],
'completed_ts': complete_ts,
'costs_usd': [0.],
'cost_saved_usd': None,
'created_ts': self.now,
'deduped_from': None,
'duration': 0.1,
'exit_code': 0,
'failure': False,
'id': '1d69b9f088008810',
'internal_failure': False,
'modified_ts': complete_ts,
'name': u'Request name',
'outputs_ref': None,
'properties_hash': None,
'server_versions': [u'v1a'],
'started_ts': reap_ts,
'state': task_result.State.COMPLETED,
'tags': [
u'pool:default',
u'priority:50',
u'service_account:none',
u'tag:1',
u'user:Jesus',
],
'try_number': 1,
'user': u'Jesus',
}
self.assertEqual(expected, result_summary.key.get().to_dict())
expected = {
'bot_overhead': 0.1,
'isolated_download': {
'duration': 0.05,
'initial_number_items': 10,
'initial_size': 10000,
'items_cold': 'foo',
'items_hot': 'bar',
},
'isolated_upload': {
'duration': 0.01,
'initial_number_items': None,
'initial_size': None,
'items_cold': 'foo',
'items_hot': None,
},
'package_installation': {
'duration': None,
'initial_number_items': None,
'initial_size': None,
'items_cold': None,
'items_hot': None,
},
}
self.assertEqual(expected, result_summary.performance_stats.to_dict())
self.assertEqual('foo', result_summary.get_output())
self.assertEqual(
datetime.timedelta(seconds=2),
result_summary.duration_as_seen_by_server)
self.assertEqual(
datetime.timedelta(seconds=0.1),
result_summary.duration_now(utils.utcnow()))
self.assertEqual(
datetime.timedelta(seconds=4), result_summary.pending)
self.assertEqual(
datetime.timedelta(seconds=4),
result_summary.pending_now(utils.utcnow()))
self.assertEqual(
task_pack.pack_result_summary_key(result_summary.key),
result_summary.task_id)
self.assertEqual(complete_ts, result_summary.ended_ts)
self.assertEqual(
task_pack.pack_run_result_key(run_result.key),
run_result.task_id)
self.assertEqual(complete_ts, run_result.ended_ts)
def schedule_request(request):
"""Creates and stores all the entities to schedule a new task request.
The number of entities created is 3: TaskRequest, TaskResultSummary and
TaskToRun.
The TaskRequest is saved first as a DB transaction, then TaskResultSummary and
TaskToRun are saved as a single DB RPC. The Search index is also updated
in-between.
Arguments:
- request: is in the TaskRequest entity saved in the DB.
Returns:
TaskResultSummary. TaskToRun is not returned.
"""
dupe_future = None
if request.properties.idempotent:
# Find a previously run task that is also idempotent and completed. Start a
# query to fetch items that can be used to dedupe the task. See the comment
# for this property for more details.
#
# Do not use "cls.created_ts > oldest" here because this would require a
# composite index. It's unnecessary because TaskRequest.key is mostly
# equivalent to decreasing TaskRequest.created_ts, ordering by key works as
# well and doesn't require a composite index.
cls = task_result.TaskResultSummary
h = request.properties.properties_hash
dupe_future = cls.query(cls.properties_hash==h).order(cls.key).get_async()
# At this point, the request is now in the DB but not yet in a mode where it
# can be triggered or visible. Index it right away so it is searchable. If any
# of remaining calls in this function fail, the TaskRequest and Search
# Document will simply point to an incomplete task, which will be ignored.
#
# Creates the entities TaskToRun and TaskResultSummary but do not save them
# yet. TaskRunResult will be created once a bot starts it.
task = task_to_run.new_task_to_run(request)
result_summary = task_result.new_result_summary(request)
# Do not specify a doc_id, as they are guaranteed to be monotonically
# increasing and searches are done in reverse order, which fits exactly the
# created_ts ordering. This is useful because DateField is precise to the date
# (!) and NumberField is signed 32 bits so the best it could do with EPOCH is
# second resolution up to year 2038.
index = search.Index(name='requests')
packed = task_pack.pack_result_summary_key(result_summary.key)
doc = search.Document(
fields=[
search.TextField(name='name', value=request.name),
search.AtomField(name='id', value=packed),
])
# Even if it fails here, we're still fine, as the task is not "alive" yet.
search_future = index.put_async([doc])
now = utils.utcnow()
if dupe_future:
# Reuse the results!
dupe_summary = dupe_future.get_result()
# Refuse tasks older than X days. This is due to the isolate server dropping
# files. https://code.google.com/p/swarming/issues/detail?id=197
oldest = now - datetime.timedelta(
seconds=config.settings().reusable_task_age_secs)
if dupe_summary and dupe_summary.created_ts > oldest:
# If there's a bug, commenting out this block is sufficient to disable the
# functionality.
# Setting task.queue_number to None removes it from the scheduling.
task.queue_number = None
_copy_entity(dupe_summary, result_summary, ('created_ts', 'name', 'user'))
result_summary.properties_hash = None
result_summary.try_number = 0
result_summary.cost_saved_usd = result_summary.cost_usd
# Only zap after.
result_summary.costs_usd = []
result_summary.deduped_from = task_pack.pack_run_result_key(
dupe_summary.run_result_key)
# Get parent task details if applicable.
parent_task_keys = None
if request.parent_task_id:
parent_run_key = task_pack.unpack_run_result_key(request.parent_task_id)
parent_task_keys = [
parent_run_key,
task_pack.run_result_key_to_result_summary_key(parent_run_key),
]
result_summary.modified_ts = now
# Storing these entities makes this task live. It is important at this point
# that the HTTP handler returns as fast as possible, otherwise the task will
# be run but the client will not know about it.
def run():
ndb.put_multi([result_summary, task])
def run_parent():
# This one is slower.
items = ndb.get_multi(parent_task_keys)
k = result_summary.task_id
for item in items:
item.children_task_ids.append(k)
item.modified_ts = now
ndb.put_multi(items)
# Raising will abort to the caller.
futures = [datastore_utils.transaction_async(run)]
if parent_task_keys:
futures.append(datastore_utils.transaction_async(run_parent))
try:
search_future.get_result()
except search.Error:
# Do not abort the task, for now search is best effort.
logging.exception('Put failed')
for future in futures:
# Check for failures, it would raise in this case, aborting the call.
future.get_result()
stats.add_task_entry(
'task_enqueued', result_summary.key,
dimensions=request.properties.dimensions,
user=request.user)
return result_summary
def test_integration(self):
# Creates a TaskRequest, along its TaskResultSummary and TaskToRun. Have a
# bot reap the task, and complete the task. Ensure the resulting
# TaskResultSummary and TaskRunResult are properly updated.
request = task_request.make_request(_gen_request(), True)
result_summary = task_result.new_result_summary(request)
to_run = task_to_run.new_task_to_run(request)
result_summary.modified_ts = utils.utcnow()
ndb.transaction(lambda: ndb.put_multi([result_summary, to_run]))
expected = {
'abandoned_ts': None,
'bot_dimensions': None,
'bot_id': None,
'bot_version': None,
'children_task_ids': [],
'completed_ts': None,
'costs_usd': [],
'cost_saved_usd': None,
'created_ts': self.now,
'deduped_from': None,
'durations': [],
'exit_codes': [],
'failure': False,
'id': '1d69b9f088008810',
'internal_failure': False,
'modified_ts': self.now,
'name': u'Request name',
'outputs_ref': None,
'properties_hash': None,
'server_versions': [],
'started_ts': None,
'state': task_result.State.PENDING,
'try_number': None,
'tags': [u'priority:50', u'tag:1', u'user:Jesus'],
'user': u'Jesus',
}
self.assertEqual(expected, result_summary.to_dict())
# Nothing changed 2 secs later except latency.
self.mock_now(self.now, 2)
self.assertEqual(expected, result_summary.to_dict())
# Task is reaped after 2 seconds (4 secs total).
reap_ts = self.now + datetime.timedelta(seconds=4)
self.mock_now(reap_ts)
to_run.queue_number = None
to_run.put()
run_result = task_result.new_run_result(request, 1, 'localhost', 'abc',
{})
run_result.modified_ts = utils.utcnow()
result_summary.set_from_run_result(run_result, request)
ndb.transaction(lambda: ndb.put_multi((result_summary, run_result)))
expected = {
'abandoned_ts': None,
'bot_dimensions': {},
'bot_id': u'localhost',
'bot_version': u'abc',
'children_task_ids': [],
'completed_ts': None,
'costs_usd': [0.],
'cost_saved_usd': None,
'created_ts': self.now,
'deduped_from': None,
'durations': [],
'exit_codes': [],
'failure': False,
'id': '1d69b9f088008810',
'internal_failure': False,
'modified_ts': reap_ts,
'name': u'Request name',
'outputs_ref': None,
'properties_hash': None,
'server_versions': [u'v1a'],
'started_ts': reap_ts,
'state': task_result.State.RUNNING,
'tags': [u'priority:50', u'tag:1', u'user:Jesus'],
'try_number': 1,
'user': u'Jesus',
}
self.assertEqual(expected, result_summary.key.get().to_dict())
# Task completed after 2 seconds (6 secs total), the task has been running
# for 2 seconds.
complete_ts = self.now + datetime.timedelta(seconds=6)
self.mock_now(complete_ts)
run_result.completed_ts = complete_ts
run_result.exit_codes.append(0)
run_result.state = task_result.State.COMPLETED
run_result.modified_ts = utils.utcnow()
ndb.transaction(
lambda: ndb.put_multi(run_result.append_output(0, 'foo', 0)))
result_summary.set_from_run_result(run_result, request)
ndb.transaction(lambda: ndb.put_multi((result_summary, run_result)))
expected = {
'abandoned_ts': None,
'bot_dimensions': {},
'bot_id': u'localhost',
'bot_version': u'abc',
'children_task_ids': [],
'completed_ts': complete_ts,
'costs_usd': [0.],
'cost_saved_usd': None,
'created_ts': self.now,
'deduped_from': None,
'durations': [],
'exit_codes': [0],
'failure': False,
'id': '1d69b9f088008810',
'internal_failure': False,
'modified_ts': complete_ts,
'name': u'Request name',
'outputs_ref': None,
'properties_hash': None,
'server_versions': [u'v1a'],
'started_ts': reap_ts,
'state': task_result.State.COMPLETED,
'tags': [u'priority:50', u'tag:1', u'user:Jesus'],
'try_number': 1,
'user': u'Jesus',
}
self.assertEqual(expected, result_summary.key.get().to_dict())
self.assertEqual(['foo'], list(result_summary.get_outputs()))
self.assertEqual(datetime.timedelta(seconds=2),
result_summary.duration_total)
self.assertEqual(datetime.timedelta(seconds=2),
result_summary.duration_now(utils.utcnow()))
self.assertEqual(datetime.timedelta(seconds=4), result_summary.pending)
self.assertEqual(datetime.timedelta(seconds=4),
result_summary.pending_now(utils.utcnow()))
self.assertEqual(task_pack.pack_result_summary_key(result_summary.key),
result_summary.task_id)
self.assertEqual(complete_ts, result_summary.ended_ts)
self.assertEqual(task_pack.pack_run_result_key(run_result.key),
run_result.task_id)
self.assertEqual(complete_ts, run_result.ended_ts)
def test_integration(self):
# Creates a TaskRequest, along its TaskResultSummary and TaskToRun. Have a
# bot reap the task, and complete the task. Ensure the resulting
# TaskResultSummary and TaskRunResult are properly updated.
request = task_request.make_request(_gen_request(), True)
result_summary = task_result.new_result_summary(request)
to_run = task_to_run.new_task_to_run(request)
result_summary.modified_ts = utils.utcnow()
ndb.transaction(lambda: ndb.put_multi([result_summary, to_run]))
expected = {
"abandoned_ts": None,
"bot_dimensions": None,
"bot_id": None,
"bot_version": None,
"children_task_ids": [],
"completed_ts": None,
"costs_usd": [],
"cost_saved_usd": None,
"created_ts": self.now,
"deduped_from": None,
"durations": [],
"exit_codes": [],
"failure": False,
"id": "1d69b9f088008810",
"internal_failure": False,
"modified_ts": self.now,
"name": u"Request name",
"outputs_ref": None,
"properties_hash": None,
"server_versions": [],
"started_ts": None,
"state": task_result.State.PENDING,
"try_number": None,
"tags": [u"priority:50", u"tag:1", u"user:Jesus"],
"user": u"Jesus",
}
self.assertEqual(expected, result_summary.to_dict())
# Nothing changed 2 secs later except latency.
self.mock_now(self.now, 2)
self.assertEqual(expected, result_summary.to_dict())
# Task is reaped after 2 seconds (4 secs total).
reap_ts = self.now + datetime.timedelta(seconds=4)
self.mock_now(reap_ts)
to_run.queue_number = None
to_run.put()
run_result = task_result.new_run_result(request, 1, "localhost", "abc", {})
run_result.modified_ts = utils.utcnow()
result_summary.set_from_run_result(run_result, request)
ndb.transaction(lambda: ndb.put_multi((result_summary, run_result)))
expected = {
"abandoned_ts": None,
"bot_dimensions": {},
"bot_id": u"localhost",
"bot_version": u"abc",
"children_task_ids": [],
"completed_ts": None,
"costs_usd": [0.0],
"cost_saved_usd": None,
"created_ts": self.now,
"deduped_from": None,
"durations": [],
"exit_codes": [],
"failure": False,
"id": "1d69b9f088008810",
"internal_failure": False,
"modified_ts": reap_ts,
"name": u"Request name",
"outputs_ref": None,
"properties_hash": None,
"server_versions": [u"v1a"],
"started_ts": reap_ts,
"state": task_result.State.RUNNING,
"tags": [u"priority:50", u"tag:1", u"user:Jesus"],
"try_number": 1,
"user": u"Jesus",
}
self.assertEqual(expected, result_summary.key.get().to_dict())
# Task completed after 2 seconds (6 secs total), the task has been running
# for 2 seconds.
complete_ts = self.now + datetime.timedelta(seconds=6)
self.mock_now(complete_ts)
run_result.completed_ts = complete_ts
run_result.exit_codes.append(0)
run_result.state = task_result.State.COMPLETED
run_result.modified_ts = utils.utcnow()
ndb.transaction(lambda: ndb.put_multi(run_result.append_output(0, "foo", 0)))
result_summary.set_from_run_result(run_result, request)
ndb.transaction(lambda: ndb.put_multi((result_summary, run_result)))
expected = {
"abandoned_ts": None,
"bot_dimensions": {},
"bot_id": u"localhost",
"bot_version": u"abc",
"children_task_ids": [],
"completed_ts": complete_ts,
"costs_usd": [0.0],
"cost_saved_usd": None,
"created_ts": self.now,
"deduped_from": None,
"durations": [],
"exit_codes": [0],
"failure": False,
"id": "1d69b9f088008810",
"internal_failure": False,
"modified_ts": complete_ts,
"name": u"Request name",
"outputs_ref": None,
"properties_hash": None,
"server_versions": [u"v1a"],
"started_ts": reap_ts,
"state": task_result.State.COMPLETED,
"tags": [u"priority:50", u"tag:1", u"user:Jesus"],
"try_number": 1,
"user": u"Jesus",
}
self.assertEqual(expected, result_summary.key.get().to_dict())
self.assertEqual(["foo"], list(result_summary.get_outputs()))
self.assertEqual(datetime.timedelta(seconds=2), result_summary.duration)
self.assertEqual(datetime.timedelta(seconds=2), result_summary.duration_now(utils.utcnow()))
self.assertEqual(datetime.timedelta(seconds=4), result_summary.pending)
self.assertEqual(datetime.timedelta(seconds=4), result_summary.pending_now(utils.utcnow()))
self.assertEqual(task_pack.pack_result_summary_key(result_summary.key), result_summary.task_id)
self.assertEqual(complete_ts, result_summary.ended_ts)
self.assertEqual(task_pack.pack_run_result_key(run_result.key), run_result.task_id)
self.assertEqual(complete_ts, run_result.ended_ts)
def schedule_request(request, secret_bytes):
"""Creates and stores all the entities to schedule a new task request.
Assumes ACL check has already happened (see 'check_schedule_request_acl').
The number of entities created is ~4: TaskRequest, TaskToRun and
TaskResultSummary and (optionally) SecretBytes. They are in single entity
group and saved in a single transaction.
Arguments:
- request: TaskRequest entity to be saved in the DB. It's key must not be set
and the entity must not be saved in the DB yet.
- secret_bytes: SecretBytes entity to be saved in the DB. It's key will be set
and the entity will be stored by this function. None is allowed if
there are no SecretBytes for this task.
Returns:
TaskResultSummary. TaskToRun is not returned.
"""
assert isinstance(request, task_request.TaskRequest), request
assert not request.key, request.key
# This does a DB GET, occasionally triggers a task queue. May throw, which is
# surfaced to the user but it is safe as the task request wasn't stored yet.
task_queues.assert_task(request)
now = utils.utcnow()
request.key = task_request.new_request_key()
result_summary = task_result.new_result_summary(request)
result_summary.modified_ts = now
to_run = None
if secret_bytes:
secret_bytes.key = request.secret_bytes_key
dupe_summary = None
for i in xrange(request.num_task_slices):
t = request.task_slice(i)
if t.properties.idempotent:
dupe_summary = _find_dupe_task(now, t.properties_hash())
if dupe_summary:
_dedupe_result_summary(dupe_summary, result_summary, i)
# In this code path, there's not much to do as the task will not be run,
# previous results are returned. We still need to store the TaskRequest
# and TaskResultSummary.
# Since the task is never scheduled, TaskToRun is not stored.
# Since the has_secret_bytes property is already set for UI purposes,
# and the task itself will never be run, we skip storing the
# SecretBytes, as they would never be read and will just consume space
# in the datastore (and the task we deduplicated with will have them
# stored anyway, if we really want to get them again).
secret_bytes = None
break
if not dupe_summary:
# The task has to run. Make sure there's capacity.
index = 0
while index < request.num_task_slices:
# This needs to be extremely fast.
to_run = task_to_run.new_task_to_run(request, 1, index)
if _has_capacity(request.task_slice(index).properties.dimensions):
# It's pending at this index now.
result_summary.current_task_slice = index
break
index += 1
if index == request.num_task_slices:
# Skip to_run since it's not enqueued.
to_run = None
# Same rationale as deduped task.
secret_bytes = None
# Instantaneously denied.
result_summary.abandoned_ts = result_summary.created_ts
result_summary.state = task_result.State.NO_RESOURCE
# Storing these entities makes this task live. It is important at this point
# that the HTTP handler returns as fast as possible, otherwise the task will
# be run but the client will not know about it.
_gen_key = lambda: _gen_new_keys(result_summary, to_run, secret_bytes)
extra = filter(bool, [result_summary, to_run, secret_bytes])
datastore_utils.insert(request, new_key_callback=_gen_key, extra=extra)
if dupe_summary:
logging.debug(
'New request %s reusing %s', result_summary.task_id,
dupe_summary.task_id)
elif result_summary.state == task_result.State.NO_RESOURCE:
logging.warning(
'New request %s denied with NO_RESOURCE', result_summary.task_id)
logging.debug('New request %s', result_summary.task_id)
else:
logging.debug('New request %s', result_summary.task_id)
# Get parent task details if applicable.
if request.parent_task_id:
parent_run_key = task_pack.unpack_run_result_key(request.parent_task_id)
parent_task_keys = [
parent_run_key,
task_pack.run_result_key_to_result_summary_key(parent_run_key),
]
def run_parent():
# This one is slower.
items = ndb.get_multi(parent_task_keys)
k = result_summary.task_id
for item in items:
item.children_task_ids.append(k)
item.modified_ts = now
ndb.put_multi(items)
# Raising will abort to the caller. There's a risk that for tasks with
# parent tasks, the task will be lost due to this transaction.
# TODO(maruel): An option is to update the parent task as part of a cron
# job, which would remove this code from the critical path.
datastore_utils.transaction(run_parent)
ts_mon_metrics.on_task_requested(result_summary, bool(dupe_summary))
return result_summary
