def test_cron_export_builds_to_bq_insert_errors(self, delete_tasks):
builds = [
test_util.build(id=i + 1, status=common_pb2.SUCCESS)
for i in xrange(3)
]
ndb.put_multi(builds)
tasks = [
taskqueue.Task(method='PULL',
payload=json.dumps({'id': b.key.id()}))
for b in builds
]
self.queue.add(tasks)
net.json_request.return_value = {
'insertErrors': [{
'index': 1,
'errors': [{
'reason': 'bad',
'message': ':('
}],
}]
}
bq._process_pull_task_batch(self.queue.name, 'builds')
self.assertTrue(net.json_request.called)
# assert second task is not deleted
deleted = delete_tasks.call_args[0][1]
self.assertEqual(
[t.payload for t in deleted],
[tasks[0].payload, tasks[2].payload],
)
def test_cron_export_builds_to_bq(self):
builds = [
test_util.build(
id=1,
status=common_pb2.SUCCESS,
infra=dict(swarming=dict(
task_dimensions=[
dict(key='a', value='1', expiration=dict(seconds=1)),
],
caches=[
dict(
path='a',
name='1',
wait_for_warm_cache=dict(seconds=1),
),
],
), ),
),
test_util.build(id=2, status=common_pb2.FAILURE),
test_util.build(id=3, status=common_pb2.SCHEDULED),
test_util.build(id=4, status=common_pb2.STARTED),
]
ndb.put_multi(builds)
build_steps = model.BuildSteps(
key=model.BuildSteps.key_for(builds[0].key))
build_steps.write_steps(
build_pb2.Build(steps=[
dict(
name='bot_update',
status=common_pb2.SUCCESS,
summary_markdown='summary_markdown',
logs=[dict(name='stdout')],
),
], ))
build_steps.put()
self.queue.add([
taskqueue.Task(method='PULL',
payload=json.dumps({'id': b.key.id()}))
for b in builds
])
bq._process_pull_task_batch(self.queue.name, 'builds')
net.json_request.assert_called_once_with(
url=('https://www.googleapis.com/bigquery/v2/'
'projects/testbed-test/datasets/builds/tables/'
'completed_BETA/insertAll'),
method='POST',
payload={
'kind':
'bigquery#tableDataInsertAllRequest',
'skipInvalidRows':
True,
'ignoreUnknownValues':
False,
'rows': [
{
'insertId': '1',
'json': mock.ANY
},
{
'insertId': '2',
'json': mock.ANY
},
],
},
scopes=bqh.INSERT_ROWS_SCOPE,
deadline=5 * 60,
)
actual_payload = net.json_request.call_args[1]['payload']
self.assertEqual(
[r['json']['id'] for r in actual_payload['rows']],
[1, 2],
)
step = actual_payload['rows'][0]['json']['steps'][0]
self.assertEqual(step['name'], 'bot_update')
self.assertEqual(step['summary_markdown'], '')
self.assertNotIn('logs', step)
def test_cron_export_builds_to_bq_no_tasks(self):
bq._process_pull_task_batch(self.queue.name, 'builds')
self.assertFalse(net.json_request.called)
def test_cron_export_builds_to_bq_not_found(self):
self.queue.add(
[taskqueue.Task(method='PULL', payload=json.dumps({'id': 1}))])
bq._process_pull_task_batch(self.queue.name, 'builds')
self.assertFalse(net.json_request.called)