def make_task_request( self, service_account, service_account_token, try_number=1): now = utils.utcnow() args = { 'created_ts': now, 'manual_tags': [u'tag:1'], 'name': 'Request with %s' % service_account, 'priority': 50, 'task_slices': [ task_request.TaskSlice( expiration_secs=60, properties=task_request.TaskProperties( command=[u'command1'], dimensions_data={u'pool': [u'default']}, execution_timeout_secs=24*60*60)), ], 'user': '******', } req = task_request.TaskRequest(**args) task_request.init_new_request(req, True) req.key = task_request.new_request_key() req.service_account = service_account req.service_account_token = service_account_token req.put() summary_key = task_pack.request_key_to_result_summary_key(req.key) run_result_key = task_pack.result_summary_key_to_run_result_key( summary_key, try_number) return task_pack.pack_run_result_key(run_result_key)
def test_new_request_key_end(self): def getrandbits(i): self.assertEqual(i, 16) return 0x7766 self.mock(random, 'getrandbits', getrandbits) days_until_end_of_the_world = 2**43 / 24. / 60. / 60. / 1000. num_days = int(days_until_end_of_the_world) # Remove 1ms to not overflow. num_seconds = ( (days_until_end_of_the_world - num_days) * 24. * 60. * 60. - 0.001) self.assertEqual(101806, num_days) self.assertEqual(278, int(num_days / 365.3)) now = (task_request._BEGINING_OF_THE_WORLD + datetime.timedelta(days=num_days, seconds=num_seconds)) self.mock_now(now) key = task_request.new_request_key() # Remove the XOR. key_id = key.integer_id() ^ task_pack.TASK_REQUEST_KEY_ID_MASK # 7ffffffffff 7766 1 # ^ ^ ^ # | | | # since 2010 | schema version # | # rand self.assertEqual('0x7ffffffffff77661', '0x%016x' % key_id)
def mkreq(req): # This function fits the old style where TaskRequest was stored first, before # TaskToRun and TaskResultSummary. task_request.init_new_request(req, True) req.key = task_request.new_request_key() req.put() return req
def get_new_keys(): # Warning: this assumes knowledge about the hierarchy of each entity. key = task_request.new_request_key() task.key.parent = key old = result_summary.task_id result_summary.parent = key logging.info('%s conflicted, using %s', old, result_summary.task_id) return key
def test_init_new_request_parent(self): parent = _gen_request() # Parent entity must have a valid key id and be stored. parent.key = task_request.new_request_key() parent.put() # The reference is to the TaskRunResult. parent_id = task_pack.pack_request_key(parent.key) + '1' child = _gen_request(parent_task_id=parent_id) self.assertEqual(parent_id, child.parent_task_id)
def mkreq(self, nb_task, req): """Stores a new initialized TaskRequest. nb_task is 1 or 0. It represents the number of GAE task queue rebuild-task-cache enqueued. It is 1 when the request.task_slice(0).properties.dimensions is new (unseen before) and a GAE task queue was enqueued to process it, 0 otherwise. """ # It is important that the task queue to be asserted. task_queues.assert_task(req) self.assertEqual(nb_task, self.execute_tasks()) req.key = task_request.new_request_key() req.put() return req
def test_new_request_key_zero(self): def getrandbits(i): self.assertEqual(i, 16) return 0x7766 self.mock(random, 'getrandbits', getrandbits) self.mock_now(task_request._BEGINING_OF_THE_WORLD) key = task_request.new_request_key() # Remove the XOR. key_id = key.integer_id() ^ task_pack.TASK_REQUEST_KEY_ID_MASK # 00000000000 7766 1 # ^ ^ ^ # | | | # since 2010 | schema version # | # rand self.assertEqual('0x0000000000077661', '0x%016x' % key_id)
def test_new_request_key(self): for _ in xrange(3): delta = utils.utcnow() - task_request._BEGINING_OF_THE_WORLD now = int(round(delta.total_seconds() * 1000.)) key = task_request.new_request_key() # Remove the XOR. key_id = key.integer_id() ^ task_pack.TASK_REQUEST_KEY_ID_MASK timestamp = key_id >> 20 randomness = (key_id >> 4) & 0xFFFF version = key_id & 0xF self.assertLess(abs(timestamp - now), 1000) self.assertEqual(1, version) if randomness: break else: self.fail('Failed to find randomness')
def _gen_new_keys(result_summary, to_run, secret_bytes): """Creates new keys for the entities. Warning: this assumes knowledge about the hierarchy of each entity. """ key = task_request.new_request_key() if to_run: to_run.key = ndb.Key(to_run.key.kind(), to_run.key.id(), parent=key) if secret_bytes: secret_bytes.key = ndb.Key( secret_bytes.key.kind(), secret_bytes.key.id(), parent=key) old = result_summary.task_id result_summary.key = ndb.Key( result_summary.key.kind(), result_summary.key.id(), parent=key) logging.info('%s conflicted, using %s', old, result_summary.task_id) return key
def _gen_request_slice(**kwargs): """Creates a TaskRequest.""" now = utils.utcnow() args = { 'created_ts': now, 'manual_tags': [u'tag:1'], 'name': 'Request name', 'priority': 50, 'task_slices': [ task_request.TaskSlice(expiration_secs=60, properties=_gen_properties()), ], 'user': '******', } args.update(kwargs) ret = task_request.TaskRequest(**args) task_request.init_new_request(ret, True) ret.key = task_request.new_request_key() ret.put() return ret
def schedule_request(request, check_acls=True): """Creates and stores all the entities to schedule a new task request. Checks ACLs first. Raises auth.AuthorizationError if caller is not authorized to post this request. The number of entities created is 3: TaskRequest, TaskToRun and TaskResultSummary. All 3 entities in the same entity group (TaskReqest, TaskToRun, TaskResultSummary) are saved as a DB transaction. Arguments: - request: TaskRequest entity to be saved in the DB. It's key must not be set and the entity must not be saved in the DB yet. - check_acls: Whether the request should check ACLs. Returns: TaskResultSummary. TaskToRun is not returned. """ assert isinstance(request, task_request.TaskRequest), request assert not request.key, request.key # Raises AuthorizationError with helpful message if the request.authorized # can't use some of the requested dimensions. if check_acls: _check_dimension_acls(request) now = utils.utcnow() request.key = task_request.new_request_key() task = task_to_run.new_task_to_run(request) result_summary = task_result.new_result_summary(request) result_summary.modified_ts = now def get_new_keys(): # Warning: this assumes knowledge about the hierarchy of each entity. key = task_request.new_request_key() task.key.parent = key old = result_summary.task_id result_summary.parent = key logging.info('%s conflicted, using %s', old, result_summary.task_id) return key deduped = False if request.properties.idempotent: dupe_summary = _find_dupe_task(now, request.properties_hash) if dupe_summary: # Setting task.queue_number to None removes it from the scheduling. task.queue_number = None _copy_summary( dupe_summary, result_summary, ('created_ts', 'modified_ts', 'name', 'user', 'tags')) # Zap irrelevant properties. PerformanceStats is also not copied over, # since it's not relevant. result_summary.properties_hash = None result_summary.try_number = 0 result_summary.cost_saved_usd = result_summary.cost_usd # Only zap after. result_summary.costs_usd = [] result_summary.deduped_from = task_pack.pack_run_result_key( dupe_summary.run_result_key) # In this code path, there's not much to do as the task will not be run, # previous results are returned. We still need to store all the entities # correctly. datastore_utils.insert(request, get_new_keys, extra=[task, result_summary]) logging.debug('New request %s reusing %s', result_summary.task_id, dupe_summary.task_id) deduped = True if not deduped: # Storing these entities makes this task live. It is important at this point # that the HTTP handler returns as fast as possible, otherwise the task will # be run but the client will not know about it. datastore_utils.insert(request, get_new_keys, extra=[task, result_summary]) logging.debug('New request %s', result_summary.task_id) # Get parent task details if applicable. if request.parent_task_id: parent_run_key = task_pack.unpack_run_result_key( request.parent_task_id) parent_task_keys = [ parent_run_key, task_pack.run_result_key_to_result_summary_key(parent_run_key), ] def run_parent(): # This one is slower. items = ndb.get_multi(parent_task_keys) k = result_summary.task_id for item in items: item.children_task_ids.append(k) item.modified_ts = now ndb.put_multi(items) # Raising will abort to the caller. There's a risk that for tasks with # parent tasks, the task will be lost due to this transaction. # TODO(maruel): An option is to update the parent task as part of a cron # job, which would remove this code from the critical path. datastore_utils.transaction(run_parent) stats.add_task_entry('task_enqueued', result_summary.key, dimensions=request.properties.dimensions, user=request.user) ts_mon_metrics.update_jobs_requested_metrics(result_summary, deduped) return result_summary
def test_init_new_request_isolated(self): parent = _gen_request( properties=_gen_properties( command=[], inputs_ref={ 'isolated': '0123456789012345678901234567890123456789', 'isolatedserver': 'http://localhost:1', 'namespace': 'default-gzip', })) # Parent entity must have a valid key id and be stored. parent.key = task_request.new_request_key() parent.put() # The reference is to the TaskRunResult. parent_id = task_pack.pack_request_key(parent.key) + u'1' req = _gen_request( properties=_gen_properties(idempotent=True, has_secret_bytes=True), parent_task_id=parent_id) # TaskRequest with secret must have a valid key. req.key = task_request.new_request_key() # Needed for the get() call below. req.put() sb = _gen_secret(req, 'I am not a banana') # Needed for properties_hash() call. sb.put() expected_properties = { 'caches': [], 'cipd_input': { 'client_package': { 'package_name': u'infra/tools/cipd/${platform}', 'path': None, 'version': u'git_revision:deadbeef', }, 'packages': [{ 'package_name': u'rm', 'path': u'bin', 'version': u'git_revision:deadbeef', }], 'server': u'https://chrome-infra-packages.appspot.com' }, 'command': [u'command1', u'arg1'], 'relative_cwd': None, 'dimensions': { u'OS': [u'Windows-3.1.1'], u'hostname': [u'localhost'], u'pool': [u'default'], }, 'env': {u'foo': u'bar', u'joe': u'2'}, 'env_prefixes': {u'PATH': [u'local/path']}, 'extra_args': [], 'execution_timeout_secs': 30, 'grace_period_secs': 30, 'idempotent': True, 'inputs_ref': { 'isolated': None, 'isolatedserver': u'https://isolateserver.appspot.com', 'namespace': u'default-gzip', }, 'io_timeout_secs': None, 'outputs': [], 'has_secret_bytes': True, } expected_request = { 'authenticated': auth_testing.DEFAULT_MOCKED_IDENTITY, 'name': u'Request name', 'parent_task_id': unicode(parent_id), 'priority': 50, 'pubsub_topic': None, 'pubsub_userdata': None, 'service_account': u'none', 'tags': [ u'OS:Windows-3.1.1', u'hostname:localhost', u'pool:default', u'priority:50', u'service_account:none', u'tag:1', u'user:Jesus', ], 'task_slices': [ { 'expiration_secs': 30, 'properties': expected_properties, 'wait_for_capacity': False, }, ], 'user': u'Jesus', } actual = req.to_dict() # expiration_ts - created_ts == scheduling_expiration_secs. actual.pop('created_ts') actual.pop('expiration_ts') self.assertEqual(expected_request, actual) self.assertEqual(30, req.expiration_secs) # Intentionally hard code the hash value since it has to be deterministic. # Other unit tests should use the calculated value. self.assertEqual( '121c6bd6216a4cc9c4302a52da6292e5a240807ef13ace6f7f36a0c83aec6f55', req.task_slice(0).properties_hash().encode('hex'))
def test_new_request_clone(self): # Compare with test_init_new_request(). parent = mkreq(_gen_request()) # Hack: Would need to know about TaskResultSummary. parent_id = task_pack.pack_request_key(parent.key) + '1' data = _gen_request( properties=dict(idempotent=True), parent_task_id=parent_id) request = task_request.new_request_clone(mkreq(data), True) request.key = task_request.new_request_key() request.put() # Differences from init_new_request() are: # - idempotent was reset to False. # - parent_task_id was reset to None. expected_properties = { 'caches': [], 'cipd_input': { 'client_package': { 'package_name': 'infra/tools/cipd/${platform}', 'path': None, 'version': 'git_revision:deadbeef', }, 'packages': [{ 'package_name': 'rm', 'path': 'bin', 'version': 'git_revision:deadbeef', }], 'server': 'https://chrome-infra-packages.appspot.com' }, 'command': [u'command1', u'arg1'], 'dimensions': { u'OS': u'Windows-3.1.1', u'hostname': u'localhost', u'pool': u'default', }, 'env': {u'foo': u'bar', u'joe': u'2'}, 'execution_timeout_secs': 30, 'extra_args': [], 'grace_period_secs': 30, 'idempotent': False, 'inputs_ref': { 'isolated': None, 'isolatedserver': 'https://isolateserver.appspot.com', 'namespace': 'default-gzip', }, 'io_timeout_secs': None, } # Differences from new_request() are: # - parent_task_id was reset to None. # - tag 'user:'******'authenticated': auth_testing.DEFAULT_MOCKED_IDENTITY, 'name': u'Request name (Retry #1)', 'parent_task_id': None, 'priority': 49, 'properties': expected_properties, 'properties_hash': None, 'pubsub_topic': None, 'pubsub_userdata': None, 'service_account': u'none', 'tags': [ u'OS:Windows-3.1.1', u'hostname:localhost', u'pool:default', u'priority:49', u'service_account:none', u'tag:1', u'user:[email protected]', ], 'user': u'*****@*****.**', } actual = request.to_dict() # expiration_ts - created_ts == deadline_to_run. actual.pop('created_ts') actual.pop('expiration_ts') self.assertEqual(expected_request, actual) self.assertEqual(30, request.expiration_secs)
def schedule_request(request, secret_bytes): """Creates and stores all the entities to schedule a new task request. Assumes ACL check has already happened (see 'check_schedule_request_acl'). The number of entities created is ~4: TaskRequest, TaskToRun and TaskResultSummary and (optionally) SecretBytes. They are in single entity group and saved in a single transaction. Arguments: - request: TaskRequest entity to be saved in the DB. It's key must not be set and the entity must not be saved in the DB yet. - secret_bytes: SecretBytes entity to be saved in the DB. It's key will be set and the entity will be stored by this function. None is allowed if there are no SecretBytes for this task. Returns: TaskResultSummary. TaskToRun is not returned. """ assert isinstance(request, task_request.TaskRequest), request assert not request.key, request.key # This does a DB GET, occasionally triggers a task queue. May throw, which is # surfaced to the user but it is safe as the task request wasn't stored yet. task_queues.assert_task(request) now = utils.utcnow() request.key = task_request.new_request_key() result_summary = task_result.new_result_summary(request) result_summary.modified_ts = now to_run = None if secret_bytes: secret_bytes.key = request.secret_bytes_key dupe_summary = None for i in xrange(request.num_task_slices): t = request.task_slice(i) if t.properties.idempotent: dupe_summary = _find_dupe_task(now, t.properties_hash()) if dupe_summary: _dedupe_result_summary(dupe_summary, result_summary, i) # In this code path, there's not much to do as the task will not be run, # previous results are returned. We still need to store the TaskRequest # and TaskResultSummary. # Since the task is never scheduled, TaskToRun is not stored. # Since the has_secret_bytes property is already set for UI purposes, # and the task itself will never be run, we skip storing the # SecretBytes, as they would never be read and will just consume space # in the datastore (and the task we deduplicated with will have them # stored anyway, if we really want to get them again). secret_bytes = None break if not dupe_summary: # The task has to run. Make sure there's capacity. index = 0 while index < request.num_task_slices: # This needs to be extremely fast. to_run = task_to_run.new_task_to_run(request, 1, index) if _has_capacity(request.task_slice(index).properties.dimensions): # It's pending at this index now. result_summary.current_task_slice = index break index += 1 if index == request.num_task_slices: # Skip to_run since it's not enqueued. to_run = None # Same rationale as deduped task. secret_bytes = None # Instantaneously denied. result_summary.abandoned_ts = result_summary.created_ts result_summary.state = task_result.State.NO_RESOURCE # Storing these entities makes this task live. It is important at this point # that the HTTP handler returns as fast as possible, otherwise the task will # be run but the client will not know about it. _gen_key = lambda: _gen_new_keys(result_summary, to_run, secret_bytes) extra = filter(bool, [result_summary, to_run, secret_bytes]) datastore_utils.insert(request, new_key_callback=_gen_key, extra=extra) if dupe_summary: logging.debug( 'New request %s reusing %s', result_summary.task_id, dupe_summary.task_id) elif result_summary.state == task_result.State.NO_RESOURCE: logging.warning( 'New request %s denied with NO_RESOURCE', result_summary.task_id) logging.debug('New request %s', result_summary.task_id) else: logging.debug('New request %s', result_summary.task_id) # Get parent task details if applicable. if request.parent_task_id: parent_run_key = task_pack.unpack_run_result_key(request.parent_task_id) parent_task_keys = [ parent_run_key, task_pack.run_result_key_to_result_summary_key(parent_run_key), ] def run_parent(): # This one is slower. items = ndb.get_multi(parent_task_keys) k = result_summary.task_id for item in items: item.children_task_ids.append(k) item.modified_ts = now ndb.put_multi(items) # Raising will abort to the caller. There's a risk that for tasks with # parent tasks, the task will be lost due to this transaction. # TODO(maruel): An option is to update the parent task as part of a cron # job, which would remove this code from the critical path. datastore_utils.transaction(run_parent) ts_mon_metrics.on_task_requested(result_summary, bool(dupe_summary)) return result_summary