def run_cron(self):
# It can trigger up to the sum of all the max_taskqueues below.
# It should complete within close to 50 seconds as each function will try to
# limit itself to its allocated chunk.
max_seconds = 50. / 2
bq_state.cron_trigger_tasks(
'task_results_run',
'/internal/taskqueue/monitoring/bq/tasks/results/run/',
'monitoring-bq-tasks-results-run',
max_seconds,
max_taskqueues=30)
bq_state.cron_trigger_tasks(
'task_results_summary',
'/internal/taskqueue/monitoring/bq/tasks/results/summary/',
'monitoring-bq-tasks-results-summary',
max_seconds,
max_taskqueues=30)
bq_state.cron_trigger_tasks(
'bot_events',
'/internal/taskqueue/monitoring/bq/bots/events/',
'monitoring-bq-bots-events',
max_seconds,
max_taskqueues=30)
bq_state.cron_trigger_tasks(
'task_requests',
'/internal/taskqueue/monitoring/bq/tasks/requests/',
'monitoring-bq-tasks-requests',
max_seconds,
max_taskqueues=30)
def test_cron_trigger_tasks_zero(self):
# 2020-01-02T03:04:05.678900
now = datetime.datetime(2020, 1, 2, 3, 4, 5, 678900)
self.mock_now(now)
self.mock(utils, 'enqueue_task', self.fail)
actual = bq_state.cron_trigger_tasks(
'mytable', '/internal/taskqueue/foo/', 'tqname', 0, 10)
self.assertEqual(0, actual)
self.assertEqual(1, bq_state.BqState.query().count())
expected = {
'oldest': datetime.datetime(2020, 1, 2, 3, 1),
'recent': datetime.datetime(2020, 1, 2, 3, 2),
'ts': now,
}
self.assertEqual(expected, bq_state.BqState.get_by_id('mytable').to_dict())
def test_cron_trigger_tasks_follow_up(self):
# Triggers tasks for a cron job, when it happened before.
# 2020-01-02T03:04:05.678900
now = datetime.datetime(2020, 1, 2, 3, 4, 5, 678900)
now_trimmed = datetime.datetime(2020, 1, 2, 3, 4)
self.mock_now(now)
oldest = (now_trimmed - bq_state._OLDEST_BACKFILL +
datetime.timedelta(minutes=3))
bq_state.BqState(id='mytable',
ts=now,
oldest=oldest,
recent=datetime.datetime(2020, 1, 2, 2, 59)).put()
urls = []
def enqueue_task(url, task_name):
urls.append(url)
self.assertEqual('tqname', task_name)
return True
self.mock(utils, 'enqueue_task', enqueue_task)
actual = bq_state.cron_trigger_tasks('mytable',
'/internal/taskqueue/foo/',
'tqname', 100, 10)
self.assertEqual(6, actual)
self.assertEqual(1, bq_state.BqState.query().count())
expected = {
'oldest': datetime.datetime(2019, 1, 3, 3, 4),
'recent': datetime.datetime(2020, 1, 2, 3, 2),
'ts': now,
}
self.assertEqual(expected,
bq_state.BqState.get_by_id('mytable').to_dict())
expected = [
# Front filling.
'/internal/taskqueue/foo/2020-01-02T02:59',
'/internal/taskqueue/foo/2020-01-02T03:00',
'/internal/taskqueue/foo/2020-01-02T03:01',
# Backfilling.
'/internal/taskqueue/foo/2019-01-03T03:07',
'/internal/taskqueue/foo/2019-01-03T03:06',
'/internal/taskqueue/foo/2019-01-03T03:05',
]
self.assertEqual(expected, urls)
def test_cron_trigger_tasks(self):
# Triggers tasks for a cron job, the very first time.
# 2020-01-02T03:04:05.678900
now = datetime.datetime(2020, 1, 2, 3, 4, 5, 678900)
self.mock_now(now)
urls = []
def enqueue_task(url, task_name):
urls.append(url)
self.assertEqual('tqname', task_name)
return True
self.mock(utils, 'enqueue_task', enqueue_task)
actual = bq_state.cron_trigger_tasks('mytable',
'/internal/taskqueue/foo/',
'tqname', 100, 10)
self.assertEqual(10, actual)
self.assertEqual(1, bq_state.BqState.query().count())
expected = {
# Values are exclusive; they are the next values to process.
'oldest': datetime.datetime(2020, 1, 2, 2, 51),
'recent': datetime.datetime(2020, 1, 2, 3, 2),
'ts': now,
}
self.assertEqual(expected,
bq_state.BqState.get_by_id('mytable').to_dict())
expected = [
# Only backfill.
'/internal/taskqueue/foo/2020-01-02T03:01',
'/internal/taskqueue/foo/2020-01-02T03:00',
'/internal/taskqueue/foo/2020-01-02T02:59',
'/internal/taskqueue/foo/2020-01-02T02:58',
'/internal/taskqueue/foo/2020-01-02T02:57',
'/internal/taskqueue/foo/2020-01-02T02:56',
'/internal/taskqueue/foo/2020-01-02T02:55',
'/internal/taskqueue/foo/2020-01-02T02:54',
'/internal/taskqueue/foo/2020-01-02T02:53',
'/internal/taskqueue/foo/2020-01-02T02:52',
]
self.assertEqual(expected, urls)