def test_rebuild_task_cache_fail(self):
# pylint: disable=unused-argument
def _enqueue_task(url, name, payload):
# Enqueueing the task failed.
return False
self.mock(utils, 'enqueue_task', _enqueue_task)
request = _gen_request()
with self.assertRaises(task_queues.Error):
task_queues.assert_task(request)
def test_assert_bot_then_task_with_id(self):
# Assert a task that includes an 'id' dimension. No task queue is triggered
# in this case, rebuild_task_cache() is called inlined.
self.assertEqual(0, _assert_bot())
request = _gen_request(properties=_gen_properties(
dimensions={u'id': [u'bot1']}))
task_queues.assert_task(request)
self.assertEqual(0, self.execute_tasks())
self.assert_count(1, bot_management.BotInfo)
self.assert_count(1, task_queues.BotDimensions)
self.assert_count(1, task_queues.BotTaskDimensions)
self.assert_count(1, task_queues.TaskDimensions)
def mkreq(self, nb_task, req):
"""Stores a new initialized TaskRequest.
nb_task is 1 or 0. It represents the number of GAE task queue
rebuild-task-cache enqueued. It is 1 when the
request.task_slice(0).properties.dimensions is new (unseen before) and a GAE
task queue was enqueued to process it, 0 otherwise.
"""
# It is important that the task queue to be asserted.
task_queues.assert_task(req)
self.assertEqual(nb_task, self.execute_tasks())
req.key = task_request.new_request_key()
req.put()
return req
def test_probably_has_capacity_get_queues(self):
d = {u'pool': [u'default'], u'os': [u'Ubuntu-16.04']}
# Capacity registers there only once there's a request enqueued and
# get_queues() is called.
_assert_bot()
request = _gen_request(properties=_gen_properties(dimensions=d))
task_queues.assert_task(request)
self.assertEqual(1, self.execute_tasks())
self.assertEqual(None, task_queues.probably_has_capacity(d))
# It get sets only once get_queues() is called.
bot_root_key = bot_management.get_root_key(u'bot1')
task_queues.get_queues(bot_root_key)
self.assertEqual(True, task_queues.probably_has_capacity(d))
self.assertEqual(
[1843498234], memcache.get('bot1', namespace='task_queues'))
def _assert_task(self, tasks=1):
request = _gen_request()
task_queues.assert_task(request)
self.assertEqual(tasks, self.execute_tasks())
return request
def schedule_request(request, secret_bytes):
"""Creates and stores all the entities to schedule a new task request.
Assumes ACL check has already happened (see 'check_schedule_request_acl').
The number of entities created is ~4: TaskRequest, TaskToRun and
TaskResultSummary and (optionally) SecretBytes. They are in single entity
group and saved in a single transaction.
Arguments:
- request: TaskRequest entity to be saved in the DB. It's key must not be set
and the entity must not be saved in the DB yet.
- secret_bytes: SecretBytes entity to be saved in the DB. It's key will be set
and the entity will be stored by this function. None is allowed if
there are no SecretBytes for this task.
Returns:
TaskResultSummary. TaskToRun is not returned.
"""
assert isinstance(request, task_request.TaskRequest), request
assert not request.key, request.key
# This does a DB GET, occasionally triggers a task queue. May throw, which is
# surfaced to the user but it is safe as the task request wasn't stored yet.
task_queues.assert_task(request)
now = utils.utcnow()
request.key = task_request.new_request_key()
result_summary = task_result.new_result_summary(request)
result_summary.modified_ts = now
to_run = None
if secret_bytes:
secret_bytes.key = request.secret_bytes_key
dupe_summary = None
for i in xrange(request.num_task_slices):
t = request.task_slice(i)
if t.properties.idempotent:
dupe_summary = _find_dupe_task(now, t.properties_hash())
if dupe_summary:
_dedupe_result_summary(dupe_summary, result_summary, i)
# In this code path, there's not much to do as the task will not be run,
# previous results are returned. We still need to store the TaskRequest
# and TaskResultSummary.
# Since the task is never scheduled, TaskToRun is not stored.
# Since the has_secret_bytes property is already set for UI purposes,
# and the task itself will never be run, we skip storing the
# SecretBytes, as they would never be read and will just consume space
# in the datastore (and the task we deduplicated with will have them
# stored anyway, if we really want to get them again).
secret_bytes = None
break
if not dupe_summary:
# The task has to run. Make sure there's capacity.
index = 0
while index < request.num_task_slices:
# This needs to be extremely fast.
to_run = task_to_run.new_task_to_run(request, 1, index)
if _has_capacity(request.task_slice(index).properties.dimensions):
# It's pending at this index now.
result_summary.current_task_slice = index
break
index += 1
if index == request.num_task_slices:
# Skip to_run since it's not enqueued.
to_run = None
# Same rationale as deduped task.
secret_bytes = None
# Instantaneously denied.
result_summary.abandoned_ts = result_summary.created_ts
result_summary.state = task_result.State.NO_RESOURCE
# Storing these entities makes this task live. It is important at this point
# that the HTTP handler returns as fast as possible, otherwise the task will
# be run but the client will not know about it.
_gen_key = lambda: _gen_new_keys(result_summary, to_run, secret_bytes)
extra = filter(bool, [result_summary, to_run, secret_bytes])
datastore_utils.insert(request, new_key_callback=_gen_key, extra=extra)
if dupe_summary:
logging.debug(
'New request %s reusing %s', result_summary.task_id,
dupe_summary.task_id)
elif result_summary.state == task_result.State.NO_RESOURCE:
logging.warning(
'New request %s denied with NO_RESOURCE', result_summary.task_id)
logging.debug('New request %s', result_summary.task_id)
else:
logging.debug('New request %s', result_summary.task_id)
# Get parent task details if applicable.
if request.parent_task_id:
parent_run_key = task_pack.unpack_run_result_key(request.parent_task_id)
parent_task_keys = [
parent_run_key,
task_pack.run_result_key_to_result_summary_key(parent_run_key),
]
def run_parent():
# This one is slower.
items = ndb.get_multi(parent_task_keys)
k = result_summary.task_id
for item in items:
item.children_task_ids.append(k)
item.modified_ts = now
ndb.put_multi(items)
# Raising will abort to the caller. There's a risk that for tasks with
# parent tasks, the task will be lost due to this transaction.
# TODO(maruel): An option is to update the parent task as part of a cron
# job, which would remove this code from the critical path.
datastore_utils.transaction(run_parent)
ts_mon_metrics.on_task_requested(result_summary, bool(dupe_summary))
return result_summary
def _assert_task(self, tasks=1): """Creates one pending TaskRequest and asserts it in task_queues.""" request = _gen_request() task_queues.assert_task(request) self.assertEqual(tasks, self.execute_tasks()) return request
def test_rebuild_task_cache(self):
# Assert that expiration works.
now = datetime.datetime(2010, 1, 2, 3, 4, 5)
self.mock_now(now)
# We want _yield_BotTaskDimensions_keys() to return multiple
# BotTaskDimensions ndb.Key to confirm that the inner loops work. This
# requires a few bots.
_assert_bot(bot_id=u'bot1')
_assert_bot(bot_id=u'bot2')
_assert_bot(bot_id=u'bot3')
bot_root_key = bot_management.get_root_key(u'bot1')
self.assertEqual(0, task_queues.BotTaskDimensions.query().count())
self.assertEqual(0, task_queues.TaskDimensions.query().count())
# Intentionally force the code to throttle the number of concurrent RPCs,
# otherwise the inner loops wouldn't be reached with less than 50 bots, and
# testing with 50 bots, would make the unit test slow.
self.mock(task_queues, '_CAP_FUTURES_LIMIT', 1)
payloads = []
def _enqueue_task(url, name, payload):
self.assertEqual(
'/internal/taskqueue/important/task_queues/rebuild-cache', url)
self.assertEqual('rebuild-task-cache', name)
payloads.append(payload)
return True
self.mock(utils, 'enqueue_task', _enqueue_task)
# The equivalent of self._assert_task(tasks=1) except that we snapshot the
# payload.
# Trigger multiple task queues to go deeper in the code.
request_1 = _gen_request(
properties=_gen_properties(
dimensions={
u'cpu': [u'x86-64'],
u'pool': [u'default'],
}))
task_queues.assert_task(request_1)
self.assertEqual(1, len(payloads))
self.assertEqual(True, task_queues.rebuild_task_cache(payloads[-1]))
self.assertEqual(3, task_queues.BotTaskDimensions.query().count())
self.assertEqual(1, task_queues.TaskDimensions.query().count())
self.assertEqual(60, request_1.expiration_secs)
expected = now + task_queues._EXTEND_VALIDITY + datetime.timedelta(
seconds=request_1.expiration_secs)
self.assertEqual(
expected, task_queues.TaskDimensions.query().get().valid_until_ts)
request_2 = _gen_request(
properties=_gen_properties(
dimensions={
u'os': [u'Ubuntu-16.04'],
u'pool': [u'default'],
}))
task_queues.assert_task(request_2)
self.assertEqual(2, len(payloads))
self.assertEqual(True, task_queues.rebuild_task_cache(payloads[-1]))
self.assertEqual(6, task_queues.BotTaskDimensions.query().count())
self.assertEqual(2, task_queues.TaskDimensions.query().count())
self.assertEqual(
[227177418, 1843498234], task_queues.get_queues(bot_root_key))
memcache.flush_all()
self.assertEqual(
[227177418, 1843498234], task_queues.get_queues(bot_root_key))
# Now expire the two TaskDimensions, one at a time, and rebuild the task
# queue.
offset = (task_queues._EXTEND_VALIDITY + datetime.timedelta(
seconds=request_1.expiration_secs)).total_seconds() + 1
self.mock_now(now, offset)
self.assertEqual(True, task_queues.rebuild_task_cache(payloads[0]))
self.assertEqual(6, task_queues.BotTaskDimensions.query().count())
self.assertEqual(2, task_queues.TaskDimensions.query().count())
self.assertEqual(
[227177418, 1843498234], task_queues.get_queues(bot_root_key))
# Observe the effect of memcache. See comment in get_queues().
memcache.flush_all()
self.assertEqual([], task_queues.get_queues(bot_root_key))
# Re-running still do not delete TaskDimensions because they are kept until
# _KEEP_DEAD.
self.assertEqual(True, task_queues.rebuild_task_cache(payloads[1]))
self.assertEqual(6, task_queues.BotTaskDimensions.query().count())
self.assertEqual(2, task_queues.TaskDimensions.query().count())
self.assertEqual([], task_queues.get_queues(bot_root_key))
# Get past _KEEP_DEAD.
offset = (
task_queues._EXTEND_VALIDITY +
task_queues._KEEP_DEAD + datetime.timedelta(
seconds=request_1.expiration_secs)
).total_seconds() + 1
self.mock_now(now, offset)
self.assertEqual([], task_queues.get_queues(bot_root_key))
self.assertEqual(True, task_queues.rebuild_task_cache(payloads[0]))
self.assertEqual(6, task_queues.BotTaskDimensions.query().count())
self.assertEqual(1, task_queues.TaskDimensions.query().count())
self.assertEqual([], task_queues.get_queues(bot_root_key))