def make_task_request(
self, service_account, service_account_token, try_number=1):
now = utils.utcnow()
args = {
'created_ts': now,
'manual_tags': [u'tag:1'],
'name': 'Request with %s' % service_account,
'priority': 50,
'task_slices': [
task_request.TaskSlice(
expiration_secs=60,
properties=task_request.TaskProperties(
command=[u'command1'],
dimensions_data={u'pool': [u'default']},
execution_timeout_secs=24*60*60)),
],
'user': '******',
}
req = task_request.TaskRequest(**args)
task_request.init_new_request(req, True)
req.key = task_request.new_request_key()
req.service_account = service_account
req.service_account_token = service_account_token
req.put()
summary_key = task_pack.request_key_to_result_summary_key(req.key)
run_result_key = task_pack.result_summary_key_to_run_result_key(
summary_key, try_number)
return task_pack.pack_run_result_key(run_result_key)
def test_new_request_key_end(self):
def getrandbits(i):
self.assertEqual(i, 16)
return 0x7766
self.mock(random, 'getrandbits', getrandbits)
days_until_end_of_the_world = 2**43 / 24. / 60. / 60. / 1000.
num_days = int(days_until_end_of_the_world)
# Remove 1ms to not overflow.
num_seconds = (
(days_until_end_of_the_world - num_days) * 24. * 60. * 60. - 0.001)
self.assertEqual(101806, num_days)
self.assertEqual(278, int(num_days / 365.3))
now = (task_request._BEGINING_OF_THE_WORLD +
datetime.timedelta(days=num_days, seconds=num_seconds))
self.mock_now(now)
key = task_request.new_request_key()
# Remove the XOR.
key_id = key.integer_id() ^ task_pack.TASK_REQUEST_KEY_ID_MASK
# 7ffffffffff 7766 1
# ^ ^ ^
# | | |
# since 2010 | schema version
# |
# rand
self.assertEqual('0x7ffffffffff77661', '0x%016x' % key_id)
def mkreq(req): # This function fits the old style where TaskRequest was stored first, before # TaskToRun and TaskResultSummary. task_request.init_new_request(req, True) req.key = task_request.new_request_key() req.put() return req
def get_new_keys():
# Warning: this assumes knowledge about the hierarchy of each entity.
key = task_request.new_request_key()
task.key.parent = key
old = result_summary.task_id
result_summary.parent = key
logging.info('%s conflicted, using %s', old, result_summary.task_id)
return key
def test_init_new_request_parent(self): parent = _gen_request() # Parent entity must have a valid key id and be stored. parent.key = task_request.new_request_key() parent.put() # The reference is to the TaskRunResult. parent_id = task_pack.pack_request_key(parent.key) + '1' child = _gen_request(parent_task_id=parent_id) self.assertEqual(parent_id, child.parent_task_id)
def mkreq(self, nb_task, req):
"""Stores a new initialized TaskRequest.
nb_task is 1 or 0. It represents the number of GAE task queue
rebuild-task-cache enqueued. It is 1 when the
request.task_slice(0).properties.dimensions is new (unseen before) and a GAE
task queue was enqueued to process it, 0 otherwise.
"""
# It is important that the task queue to be asserted.
task_queues.assert_task(req)
self.assertEqual(nb_task, self.execute_tasks())
req.key = task_request.new_request_key()
req.put()
return req
def test_new_request_key_zero(self):
def getrandbits(i):
self.assertEqual(i, 16)
return 0x7766
self.mock(random, 'getrandbits', getrandbits)
self.mock_now(task_request._BEGINING_OF_THE_WORLD)
key = task_request.new_request_key()
# Remove the XOR.
key_id = key.integer_id() ^ task_pack.TASK_REQUEST_KEY_ID_MASK
# 00000000000 7766 1
# ^ ^ ^
# | | |
# since 2010 | schema version
# |
# rand
self.assertEqual('0x0000000000077661', '0x%016x' % key_id)
def test_new_request_key(self):
for _ in xrange(3):
delta = utils.utcnow() - task_request._BEGINING_OF_THE_WORLD
now = int(round(delta.total_seconds() * 1000.))
key = task_request.new_request_key()
# Remove the XOR.
key_id = key.integer_id() ^ task_pack.TASK_REQUEST_KEY_ID_MASK
timestamp = key_id >> 20
randomness = (key_id >> 4) & 0xFFFF
version = key_id & 0xF
self.assertLess(abs(timestamp - now), 1000)
self.assertEqual(1, version)
if randomness:
break
else:
self.fail('Failed to find randomness')
def _gen_new_keys(result_summary, to_run, secret_bytes):
"""Creates new keys for the entities.
Warning: this assumes knowledge about the hierarchy of each entity.
"""
key = task_request.new_request_key()
if to_run:
to_run.key = ndb.Key(to_run.key.kind(), to_run.key.id(), parent=key)
if secret_bytes:
secret_bytes.key = ndb.Key(
secret_bytes.key.kind(), secret_bytes.key.id(), parent=key)
old = result_summary.task_id
result_summary.key = ndb.Key(
result_summary.key.kind(), result_summary.key.id(), parent=key)
logging.info('%s conflicted, using %s', old, result_summary.task_id)
return key
def _gen_request_slice(**kwargs):
"""Creates a TaskRequest."""
now = utils.utcnow()
args = {
'created_ts':
now,
'manual_tags': [u'tag:1'],
'name':
'Request name',
'priority':
50,
'task_slices': [
task_request.TaskSlice(expiration_secs=60,
properties=_gen_properties()),
],
'user':
'******',
}
args.update(kwargs)
ret = task_request.TaskRequest(**args)
task_request.init_new_request(ret, True)
ret.key = task_request.new_request_key()
ret.put()
return ret
def schedule_request(request, check_acls=True):
"""Creates and stores all the entities to schedule a new task request.
Checks ACLs first. Raises auth.AuthorizationError if caller is not authorized
to post this request.
The number of entities created is 3: TaskRequest, TaskToRun and
TaskResultSummary.
All 3 entities in the same entity group (TaskReqest, TaskToRun,
TaskResultSummary) are saved as a DB transaction.
Arguments:
- request: TaskRequest entity to be saved in the DB. It's key must not be set
and the entity must not be saved in the DB yet.
- check_acls: Whether the request should check ACLs.
Returns:
TaskResultSummary. TaskToRun is not returned.
"""
assert isinstance(request, task_request.TaskRequest), request
assert not request.key, request.key
# Raises AuthorizationError with helpful message if the request.authorized
# can't use some of the requested dimensions.
if check_acls:
_check_dimension_acls(request)
now = utils.utcnow()
request.key = task_request.new_request_key()
task = task_to_run.new_task_to_run(request)
result_summary = task_result.new_result_summary(request)
result_summary.modified_ts = now
def get_new_keys():
# Warning: this assumes knowledge about the hierarchy of each entity.
key = task_request.new_request_key()
task.key.parent = key
old = result_summary.task_id
result_summary.parent = key
logging.info('%s conflicted, using %s', old, result_summary.task_id)
return key
deduped = False
if request.properties.idempotent:
dupe_summary = _find_dupe_task(now, request.properties_hash)
if dupe_summary:
# Setting task.queue_number to None removes it from the scheduling.
task.queue_number = None
_copy_summary(
dupe_summary, result_summary,
('created_ts', 'modified_ts', 'name', 'user', 'tags'))
# Zap irrelevant properties. PerformanceStats is also not copied over,
# since it's not relevant.
result_summary.properties_hash = None
result_summary.try_number = 0
result_summary.cost_saved_usd = result_summary.cost_usd
# Only zap after.
result_summary.costs_usd = []
result_summary.deduped_from = task_pack.pack_run_result_key(
dupe_summary.run_result_key)
# In this code path, there's not much to do as the task will not be run,
# previous results are returned. We still need to store all the entities
# correctly.
datastore_utils.insert(request,
get_new_keys,
extra=[task, result_summary])
logging.debug('New request %s reusing %s', result_summary.task_id,
dupe_summary.task_id)
deduped = True
if not deduped:
# Storing these entities makes this task live. It is important at this point
# that the HTTP handler returns as fast as possible, otherwise the task will
# be run but the client will not know about it.
datastore_utils.insert(request,
get_new_keys,
extra=[task, result_summary])
logging.debug('New request %s', result_summary.task_id)
# Get parent task details if applicable.
if request.parent_task_id:
parent_run_key = task_pack.unpack_run_result_key(
request.parent_task_id)
parent_task_keys = [
parent_run_key,
task_pack.run_result_key_to_result_summary_key(parent_run_key),
]
def run_parent():
# This one is slower.
items = ndb.get_multi(parent_task_keys)
k = result_summary.task_id
for item in items:
item.children_task_ids.append(k)
item.modified_ts = now
ndb.put_multi(items)
# Raising will abort to the caller. There's a risk that for tasks with
# parent tasks, the task will be lost due to this transaction.
# TODO(maruel): An option is to update the parent task as part of a cron
# job, which would remove this code from the critical path.
datastore_utils.transaction(run_parent)
stats.add_task_entry('task_enqueued',
result_summary.key,
dimensions=request.properties.dimensions,
user=request.user)
ts_mon_metrics.update_jobs_requested_metrics(result_summary, deduped)
return result_summary
def test_init_new_request_isolated(self):
parent = _gen_request(
properties=_gen_properties(
command=[],
inputs_ref={
'isolated': '0123456789012345678901234567890123456789',
'isolatedserver': 'http://localhost:1',
'namespace': 'default-gzip',
}))
# Parent entity must have a valid key id and be stored.
parent.key = task_request.new_request_key()
parent.put()
# The reference is to the TaskRunResult.
parent_id = task_pack.pack_request_key(parent.key) + u'1'
req = _gen_request(
properties=_gen_properties(idempotent=True, has_secret_bytes=True),
parent_task_id=parent_id)
# TaskRequest with secret must have a valid key.
req.key = task_request.new_request_key()
# Needed for the get() call below.
req.put()
sb = _gen_secret(req, 'I am not a banana')
# Needed for properties_hash() call.
sb.put()
expected_properties = {
'caches': [],
'cipd_input': {
'client_package': {
'package_name': u'infra/tools/cipd/${platform}',
'path': None,
'version': u'git_revision:deadbeef',
},
'packages': [{
'package_name': u'rm',
'path': u'bin',
'version': u'git_revision:deadbeef',
}],
'server': u'https://chrome-infra-packages.appspot.com'
},
'command': [u'command1', u'arg1'],
'relative_cwd': None,
'dimensions': {
u'OS': [u'Windows-3.1.1'],
u'hostname': [u'localhost'],
u'pool': [u'default'],
},
'env': {u'foo': u'bar', u'joe': u'2'},
'env_prefixes': {u'PATH': [u'local/path']},
'extra_args': [],
'execution_timeout_secs': 30,
'grace_period_secs': 30,
'idempotent': True,
'inputs_ref': {
'isolated': None,
'isolatedserver': u'https://isolateserver.appspot.com',
'namespace': u'default-gzip',
},
'io_timeout_secs': None,
'outputs': [],
'has_secret_bytes': True,
}
expected_request = {
'authenticated': auth_testing.DEFAULT_MOCKED_IDENTITY,
'name': u'Request name',
'parent_task_id': unicode(parent_id),
'priority': 50,
'pubsub_topic': None,
'pubsub_userdata': None,
'service_account': u'none',
'tags': [
u'OS:Windows-3.1.1',
u'hostname:localhost',
u'pool:default',
u'priority:50',
u'service_account:none',
u'tag:1',
u'user:Jesus',
],
'task_slices': [
{
'expiration_secs': 30,
'properties': expected_properties,
'wait_for_capacity': False,
},
],
'user': u'Jesus',
}
actual = req.to_dict()
# expiration_ts - created_ts == scheduling_expiration_secs.
actual.pop('created_ts')
actual.pop('expiration_ts')
self.assertEqual(expected_request, actual)
self.assertEqual(30, req.expiration_secs)
# Intentionally hard code the hash value since it has to be deterministic.
# Other unit tests should use the calculated value.
self.assertEqual(
'121c6bd6216a4cc9c4302a52da6292e5a240807ef13ace6f7f36a0c83aec6f55',
req.task_slice(0).properties_hash().encode('hex'))
def test_new_request_clone(self):
# Compare with test_init_new_request().
parent = mkreq(_gen_request())
# Hack: Would need to know about TaskResultSummary.
parent_id = task_pack.pack_request_key(parent.key) + '1'
data = _gen_request(
properties=dict(idempotent=True), parent_task_id=parent_id)
request = task_request.new_request_clone(mkreq(data), True)
request.key = task_request.new_request_key()
request.put()
# Differences from init_new_request() are:
# - idempotent was reset to False.
# - parent_task_id was reset to None.
expected_properties = {
'caches': [],
'cipd_input': {
'client_package': {
'package_name': 'infra/tools/cipd/${platform}',
'path': None,
'version': 'git_revision:deadbeef',
},
'packages': [{
'package_name': 'rm',
'path': 'bin',
'version': 'git_revision:deadbeef',
}],
'server': 'https://chrome-infra-packages.appspot.com'
},
'command': [u'command1', u'arg1'],
'dimensions': {
u'OS': u'Windows-3.1.1',
u'hostname': u'localhost',
u'pool': u'default',
},
'env': {u'foo': u'bar', u'joe': u'2'},
'execution_timeout_secs': 30,
'extra_args': [],
'grace_period_secs': 30,
'idempotent': False,
'inputs_ref': {
'isolated': None,
'isolatedserver': 'https://isolateserver.appspot.com',
'namespace': 'default-gzip',
},
'io_timeout_secs': None,
}
# Differences from new_request() are:
# - parent_task_id was reset to None.
# - tag 'user:'******'authenticated': auth_testing.DEFAULT_MOCKED_IDENTITY,
'name': u'Request name (Retry #1)',
'parent_task_id': None,
'priority': 49,
'properties': expected_properties,
'properties_hash': None,
'pubsub_topic': None,
'pubsub_userdata': None,
'service_account': u'none',
'tags': [
u'OS:Windows-3.1.1',
u'hostname:localhost',
u'pool:default',
u'priority:49',
u'service_account:none',
u'tag:1',
u'user:[email protected]',
],
'user': u'*****@*****.**',
}
actual = request.to_dict()
# expiration_ts - created_ts == deadline_to_run.
actual.pop('created_ts')
actual.pop('expiration_ts')
self.assertEqual(expected_request, actual)
self.assertEqual(30, request.expiration_secs)
def schedule_request(request, secret_bytes):
"""Creates and stores all the entities to schedule a new task request.
Assumes ACL check has already happened (see 'check_schedule_request_acl').
The number of entities created is ~4: TaskRequest, TaskToRun and
TaskResultSummary and (optionally) SecretBytes. They are in single entity
group and saved in a single transaction.
Arguments:
- request: TaskRequest entity to be saved in the DB. It's key must not be set
and the entity must not be saved in the DB yet.
- secret_bytes: SecretBytes entity to be saved in the DB. It's key will be set
and the entity will be stored by this function. None is allowed if
there are no SecretBytes for this task.
Returns:
TaskResultSummary. TaskToRun is not returned.
"""
assert isinstance(request, task_request.TaskRequest), request
assert not request.key, request.key
# This does a DB GET, occasionally triggers a task queue. May throw, which is
# surfaced to the user but it is safe as the task request wasn't stored yet.
task_queues.assert_task(request)
now = utils.utcnow()
request.key = task_request.new_request_key()
result_summary = task_result.new_result_summary(request)
result_summary.modified_ts = now
to_run = None
if secret_bytes:
secret_bytes.key = request.secret_bytes_key
dupe_summary = None
for i in xrange(request.num_task_slices):
t = request.task_slice(i)
if t.properties.idempotent:
dupe_summary = _find_dupe_task(now, t.properties_hash())
if dupe_summary:
_dedupe_result_summary(dupe_summary, result_summary, i)
# In this code path, there's not much to do as the task will not be run,
# previous results are returned. We still need to store the TaskRequest
# and TaskResultSummary.
# Since the task is never scheduled, TaskToRun is not stored.
# Since the has_secret_bytes property is already set for UI purposes,
# and the task itself will never be run, we skip storing the
# SecretBytes, as they would never be read and will just consume space
# in the datastore (and the task we deduplicated with will have them
# stored anyway, if we really want to get them again).
secret_bytes = None
break
if not dupe_summary:
# The task has to run. Make sure there's capacity.
index = 0
while index < request.num_task_slices:
# This needs to be extremely fast.
to_run = task_to_run.new_task_to_run(request, 1, index)
if _has_capacity(request.task_slice(index).properties.dimensions):
# It's pending at this index now.
result_summary.current_task_slice = index
break
index += 1
if index == request.num_task_slices:
# Skip to_run since it's not enqueued.
to_run = None
# Same rationale as deduped task.
secret_bytes = None
# Instantaneously denied.
result_summary.abandoned_ts = result_summary.created_ts
result_summary.state = task_result.State.NO_RESOURCE
# Storing these entities makes this task live. It is important at this point
# that the HTTP handler returns as fast as possible, otherwise the task will
# be run but the client will not know about it.
_gen_key = lambda: _gen_new_keys(result_summary, to_run, secret_bytes)
extra = filter(bool, [result_summary, to_run, secret_bytes])
datastore_utils.insert(request, new_key_callback=_gen_key, extra=extra)
if dupe_summary:
logging.debug(
'New request %s reusing %s', result_summary.task_id,
dupe_summary.task_id)
elif result_summary.state == task_result.State.NO_RESOURCE:
logging.warning(
'New request %s denied with NO_RESOURCE', result_summary.task_id)
logging.debug('New request %s', result_summary.task_id)
else:
logging.debug('New request %s', result_summary.task_id)
# Get parent task details if applicable.
if request.parent_task_id:
parent_run_key = task_pack.unpack_run_result_key(request.parent_task_id)
parent_task_keys = [
parent_run_key,
task_pack.run_result_key_to_result_summary_key(parent_run_key),
]
def run_parent():
# This one is slower.
items = ndb.get_multi(parent_task_keys)
k = result_summary.task_id
for item in items:
item.children_task_ids.append(k)
item.modified_ts = now
ndb.put_multi(items)
# Raising will abort to the caller. There's a risk that for tasks with
# parent tasks, the task will be lost due to this transaction.
# TODO(maruel): An option is to update the parent task as part of a cron
# job, which would remove this code from the critical path.
datastore_utils.transaction(run_parent)
ts_mon_metrics.on_task_requested(result_summary, bool(dupe_summary))
return result_summary
