def test_update_tag_indexes_async(self):
builds = [
test_util.build(id=1,
builder=dict(project='chromium'),
tags=[
dict(key='buildset', value='common'),
dict(key='buildset', value='unique'),
]),
test_util.build(id=2,
builder=dict(project='v8'),
tags=[
dict(key='buildset', value='common'),
]),
]
ndb.Future.wait_all(search.update_tag_indexes_async(builds))
common = search.TagIndex.get_by_id('buildset:common')
self.assertEqual(
{e.build_id
for e in common.entries},
{1, 2},
)
unique = search.TagIndex.get_by_id('buildset:unique')
self.assertEqual(
{e.build_id
for e in unique.entries},
{1},
)
def test_get_and_search(self, search_async, get_async):
search_async.return_value = future(
([test_util.build(id=1),
test_util.build(id=2)], ''))
get_async.return_value = future(test_util.build(id=3))
req = rpc_pb2.BatchRequest(requests=[
dict(search_builds=dict(predicate=dict(builder=dict(
project='chromium',
bucket='try',
builder='linux-rel',
), ), ), ),
dict(get_build=dict(id=3)),
], )
res = self.call(self.api.Batch, req)
search_async.assert_called_once_with(
search.Query(
bucket_ids=['chromium/try'],
builder='linux-rel',
status=common_pb2.STATUS_UNSPECIFIED,
include_experimental=False,
tags=[],
start_cursor='',
), )
get_async.assert_called_once_with(3)
self.assertEqual(len(res.responses), 2)
self.assertEqual(len(res.responses[0].search_builds.builds), 2)
self.assertEqual(res.responses[0].search_builds.builds[0].id, 1L)
self.assertEqual(res.responses[0].search_builds.builds[1].id, 2L)
self.assertEqual(res.responses[1].get_build.id, 3L)
def test_basic(self, search_async):
builds = [test_util.build(id=54), test_util.build(id=55)]
search_async.return_value = future((builds, 'next page token'))
req = rpc_pb2.SearchBuildsRequest(
predicate=dict(builder=dict(project='chromium',
bucket='try',
builder='linux-try'), ),
page_size=10,
page_token='page token',
)
res = self.call(self.api.SearchBuilds, req)
search_async.assert_called_once_with(
search.Query(
bucket_ids=['chromium/try'],
builder='linux-try',
include_experimental=False,
tags=[],
status=common_pb2.STATUS_UNSPECIFIED,
max_builds=10,
start_cursor='page token',
))
self.assertEqual(len(res.builds), 2)
self.assertEqual(res.builds[0].id, 54)
self.assertEqual(res.builds[1].id, 55)
self.assertEqual(res.next_page_token, 'next page token')
def test_delete_many_scheduled_builds(self):
scheduled_build = test_util.build(id=1, status=common_pb2.SCHEDULED)
completed_build = test_util.build(id=2, status=common_pb2.SUCCESS)
scheduled_build.put()
completed_build.put()
self.assertIsNotNone(scheduled_build.key.get())
self.assertIsNotNone(completed_build.key.get())
service._task_delete_many_builds(scheduled_build.bucket_id,
model.BuildStatus.SCHEDULED)
self.assertIsNone(scheduled_build.key.get())
self.assertIsNotNone(completed_build.key.get())
def test_cancel(self, cancel_task_async):
test_util.build(id=1).put()
build = service.cancel_async(1, summary_markdown='nope').get_result()
self.assertEqual(build.proto.status, common_pb2.CANCELED)
self.assertEqual(build.proto.end_time.ToDatetime(), utils.utcnow())
self.assertEqual(build.proto.summary_markdown, 'nope')
self.assertEqual(build.proto.canceled_by,
self.current_identity.to_bytes())
cancel_task_async.assert_called_with('swarming.example.com',
'deadbeef')
self.assertEqual(build.status_changed_time, utils.utcnow())
def test_ended_build(self):
test_util.build(id=123, status=common_pb2.SUCCESS).put()
req, ctx = self._mk_update_req(build_pb2.Build(id=123))
self.call(
self.api.UpdateBuild,
req,
ctx=ctx,
expected_code=prpc.StatusCode.FAILED_PRECONDITION,
expected_details='Cannot update an ended build',
)
def mkbuild(bucket='try',
builder='release',
status=common_pb2.SCHEDULED,
experimental=False):
test_util.build(
builder=dict(project='chromium',
bucket=bucket,
builder=builder),
status=status,
input=dict(experimental=experimental),
).put()
def test_delete_many_started_builds(self):
scheduled_build = test_util.build(id=1, status=common_pb2.SCHEDULED)
started_build = test_util.build(id=2, status=common_pb2.STARTED)
completed_build = test_util.build(id=3, status=common_pb2.SUCCESS)
ndb.put_multi([scheduled_build, started_build, completed_build])
service._task_delete_many_builds(scheduled_build.bucket_id,
model.BuildStatus.STARTED)
self.assertIsNotNone(scheduled_build.key.get())
self.assertIsNone(started_build.key.get())
self.assertIsNotNone(completed_build.key.get())
def test_update_steps_of_scheduled_build(self):
test_util.build(id=123, status=common_pb2.SCHEDULED).put()
build_proto = build_pb2.Build(id=123)
req, ctx = self._mk_update_req(build_proto, paths=['build.steps'])
self.call(
self.api.UpdateBuild,
req,
ctx=ctx,
expected_code=prpc.StatusCode.INVALID_ARGUMENT,
)
def test_delete_many_builds_created_by(self):
build1 = test_util.build(id=1, created_by='user:[email protected]')
build2 = test_util.build(id=2, created_by='user:[email protected]')
ndb.put_multi([build1, build2])
service._task_delete_many_builds(
build1.bucket_id,
model.BuildStatus.SCHEDULED,
created_by=build2.created_by,
)
self.assertIsNone(build2.key.get())
self.assertIsNotNone(build1.key.get())
def test_missing_token(self):
test_util.build(id=123).put()
build = build_pb2.Build(
id=123,
status=common_pb2.STARTED,
)
req, ctx = self._mk_update_req(build, token=None)
self.call(
self.api.UpdateBuild,
req,
ctx=ctx,
expected_code=prpc.StatusCode.UNAUTHENTICATED,
expected_details='missing token in build update request',
)
def test_schedule_build_requests(self, add_many_async):
add_many_async.return_value = future([
(test_util.build(id=42), None),
(test_util.build(id=43), None),
(None, errors.InvalidInputError('bad')),
(None, Exception('unexpected')),
(None, auth.AuthorizationError('bad')),
])
user.can_async.side_effect = (
lambda bucket_id, _: future('forbidden' not in bucket_id))
linux_builder = dict(project='chromium', bucket='try', builder='linux')
win_builder = dict(project='chromium', bucket='try', builder='windows')
req = rpc_pb2.BatchRequest(
requests=[
dict(schedule_build=dict(builder=linux_builder)),
dict(schedule_build=dict(builder=linux_builder,
fields=dict(paths=['tags']))),
dict(schedule_build=dict(builder=linux_builder,
fields=dict(paths=['wrong-field']))),
dict(schedule_build=dict(builder=win_builder)),
dict(schedule_build=dict(builder=win_builder)),
dict(schedule_build=dict(builder=win_builder)),
dict(schedule_build=dict(builder=dict(project='chromium',
bucket='forbidden',
builder='nope'), )),
dict(
schedule_build=dict(), # invalid request
),
], )
res = self.call(self.api.Batch, req)
codes = [r.error.code for r in res.responses]
self.assertEqual(codes, [
prpc.StatusCode.OK.value,
prpc.StatusCode.OK.value,
prpc.StatusCode.INVALID_ARGUMENT.value,
prpc.StatusCode.INVALID_ARGUMENT.value,
prpc.StatusCode.INTERNAL.value,
prpc.StatusCode.PERMISSION_DENIED.value,
prpc.StatusCode.PERMISSION_DENIED.value,
prpc.StatusCode.INVALID_ARGUMENT.value,
])
self.assertEqual(res.responses[0].schedule_build.id, 42)
self.assertFalse(len(res.responses[0].schedule_build.tags))
self.assertTrue(len(res.responses[1].schedule_build.tags))
def test_build_to_dict_non_luci(self):
build = test_util.build(builder=dict(bucket='master.chromium'))
build.is_luci = False
actual = api_common.build_to_dict(build, None)
self.assertEqual(actual['project'], 'chromium')
self.assertEqual(actual['bucket'], 'master.chromium')
def test_fields_for(self):
build = test_util.build(
builder=dict(project='chromium', bucket='try', builder='linux'),
status=common_pb2.FAILURE,
tags=[
dict(key='user_agent', value='cq'),
dict(key='something', value='else'),
],
canary=common_pb2.YES,
)
expected = {
'bucket': 'luci.chromium.try',
'builder': 'linux',
'canary': True,
'user_agent': 'cq',
'status': 'COMPLETED',
'result': 'FAILURE',
'failure_reason': 'BUILD_FAILURE',
'cancelation_reason': '',
}
self.assertEqual(set(expected), set(metrics._BUILD_FIELDS))
actual = metrics._fields_for(build, expected.keys())
self.assertEqual(expected, actual)
with self.assertRaises(ValueError):
metrics._fields_for(build, ['wrong field'])
def test_cron_export_builds_to_bq_insert_errors(self, delete_tasks):
builds = [
test_util.build(id=i + 1, status=common_pb2.SUCCESS)
for i in xrange(3)
]
ndb.put_multi(builds)
tasks = [
taskqueue.Task(method='PULL',
payload=json.dumps({'id': b.key.id()}))
for b in builds
]
self.queue.add(tasks)
net.json_request.return_value = {
'insertErrors': [{
'index': 1,
'errors': [{
'reason': 'bad',
'message': ':('
}],
}]
}
bq._process_pull_task_batch(self.queue.name, 'builds')
self.assertTrue(net.json_request.called)
# assert second task is not deleted
deleted = delete_tasks.call_args[0][1]
self.assertEqual(
[t.payload for t in deleted],
[tasks[0].payload, tasks[2].payload],
)
def test_trimming_exclude(self, get_async):
get_async.return_value = future(
test_util.build(input=dict(
properties=bbutil.dict_to_struct({'a': 'b'}))), )
req = rpc_pb2.GetBuildRequest(id=1)
res = self.call(self.api.GetBuild, req)
self.assertFalse(res.input.HasField('properties'))
def test_failed(self, on_build_completed, on_build_completing_async):
steps = model.BuildSteps.make(
build_pb2.Build(
id=123,
steps=[dict(name='step', status=common_pb2.SCHEDULED)],
))
steps.put()
on_build_completing_async.return_value = future(None)
build = test_util.build(id=123)
build.put()
req, ctx = self._mk_update_req(
build_pb2.Build(
id=123,
status=common_pb2.FAILURE,
summary_markdown='bad',
),
paths=['build.status', 'build.summary_markdown'],
)
self.call(self.api.UpdateBuild, req, ctx=ctx)
build = build.key.get()
self.assertEqual(build.proto.status, common_pb2.FAILURE)
self.assertEqual(build.proto.summary_markdown, 'bad')
self.assertEqual(build.proto.end_time.ToDatetime(), self.now)
on_build_completing_async.assert_called_once_with(build)
on_build_completed.assert_called_once_with(build)
steps = steps.key.get()
step_container = build_pb2.Build()
steps.read_steps(step_container)
self.assertEqual(step_container.steps[0].status, common_pb2.CANCELED)
def test_process(self):
builds = [
test_util.build(
id=i,
tags=[
dict(key='t', value='%d' % (i % 3)),
dict(key='a', value='b'),
],
) for i in xrange(50, 60)
]
backfill_tag_index._process_builds(builds, 't', 5)
backfill_tag_index._enqueue_flush_entries.assert_called_with(
't', {
'0': [
['chromium/try', 51],
['chromium/try', 54],
],
'1': [['chromium/try', 52]],
'2': [
['chromium/try', 50],
['chromium/try', 53],
],
})
def test_infra(self):
build = test_util.build(infra=dict(swarming=dict(
hostname='swarming.example.com')))
actual = self.to_proto(build, load_infra=True)
self.assertEqual(actual.infra.swarming.hostname,
'swarming.example.com')
def test_lease_unsuccessful(self, lease):
lease.return_value = (False, test_util.build(id=1))
req = {
'id': '1',
'lease_expiration_ts': self.future_ts,
}
self.expect_error('lease', req, 'CANNOT_LEASE_BUILD')
def setUp(self):
super(CreateTaskTest, self).setUp()
self.patch('components.net.json_request_async', autospec=True)
self.patch('components.auth.delegate_async',
return_value=future('blah'))
self.build_token = 'beeff00d'
self.patch('tokens.generate_build_token',
autospec=True,
return_value='deadbeef')
self.task_def = {
'is_task_def': True,
'task_slices': [{
'properties': {},
}]
}
self.patch('swarming.prepare_task_def',
autospec=True,
return_value=self.task_def)
self.build = test_util.build(id=1, created_by='user:[email protected]')
self.build.swarming_task_key = None
with self.build.mutate_infra() as infra:
infra.swarming.task_id = ''
self.build.put()
def test_put_with_generic_buildset(self, add_async):
tags = [
dict(key='buildset', value='x'),
dict(key='t', value='0'),
]
build = test_util.build(id=1, tags=tags)
add_async.return_value = future(build)
req = {
'client_operation_id': '42',
'bucket': 'luci.chromium.try',
'tags': ['buildset:x', 't:0'],
'parameters_json': json.dumps({api_common.BUILDER_PARAMETER: 'linux'}),
}
resp = self.call_api('put', req).json_body
add_async.assert_called_once_with(
creation.BuildRequest(
schedule_build_request=rpc_pb2.ScheduleBuildRequest(
builder=dict(
project='chromium',
bucket='try',
builder='linux',
),
tags=tags,
request_id='42',
properties=dict(),
),
parameters={},
)
)
self.assertEqual(resp['build']['id'], '1')
self.assertIn('buildset:x', resp['build']['tags'])
self.assertIn('t:0', resp['build']['tags'])
def test_cancel(self, cancel_async):
cancel_async.return_value = future(
test_util.build(id=54, status=common_pb2.CANCELED), )
req = rpc_pb2.CancelBuildRequest(id=54, summary_markdown='unnecesary')
res = self.call(self.api.CancelBuild, req)
self.assertEqual(res.id, 54)
self.assertEqual(res.status, common_pb2.CANCELED)
cancel_async.assert_called_once_with(54, summary_markdown='unnecesary')
def test_expired_build_to_message(self):
yesterday = utils.utcnow() - datetime.timedelta(days=1)
yesterday_timestamp = utils.datetime_to_timestamp(yesterday)
build = test_util.build()
build.lease_key = 1
build.lease_expiration_date = yesterday
msg = api_common.build_to_message(build, None)
self.assertEqual(msg.lease_expiration_ts, yesterday_timestamp)
def test_trimming_include(self, get_async):
get_async.return_value = future(
test_util.build(input=dict(
properties=bbutil.dict_to_struct({'a': 'b'}))), )
req = rpc_pb2.GetBuildRequest(id=1,
fields=dict(paths=['input.properties']))
res = self.call(self.api.GetBuild, req)
self.assertEqual(res.input.properties.items(), [('a', 'b')])
def test_invalid_token(self):
test_util.build(id=123).put()
self.validate_build_token.side_effect = auth.InvalidTokenError
build = build_pb2.Build(
id=123,
status=common_pb2.STARTED,
)
req, ctx = self._mk_update_req(build)
self.call(
self.api.UpdateBuild,
req,
ctx=ctx,
expected_code=prpc.StatusCode.UNAUTHENTICATED,
)
def test_retry_forbidden(self):
config.put_bucket(
'chromium',
'a' * 40,
test_util.parse_bucket_cfg('''
name: "readonly"
acls {
role: READER
identity: "anonymous:anonymous"
}
'''),
)
test_util.build(id=1,
builder=dict(project='chromium',
bucket='readonly')).put()
self.call_api('retry', {'id': '1'}, status=403)
def test_sync_build_async_no_task(self, load_task_result_async):
load_task_result_async.return_value = future(None)
build = test_util.build()
build.put()
swarming.CronUpdateBuilds().update_build_async(build).get_result()
build = build.key.get()
self.assertEqual(build.proto.status, common_pb2.INFRA_FAILURE)
self.assertTrue(build.proto.summary_markdown)
def test_post(self, load_task_result_async):
build = test_util.build(id=1)
build.put()
self.mock_request({
'build_id': 1L,
'created_ts': 1448841600000000,
'swarming_hostname': 'swarming.example.com',
})
def test_put_with_gerrit_change(self, add_async):
buildset = 'patch/gerrit/gerrit.example.com/1234/5'
buildset_tag = 'buildset:' + buildset
props = {'patch_project': 'repo'}
expected_sbr = rpc_pb2.ScheduleBuildRequest(
builder=dict(
project='chromium',
bucket='try',
builder='linux',
),
gerrit_changes=[
dict(
host='gerrit.example.com',
project='repo',
change=1234,
patchset=5,
)
],
tags=[dict(key='t', value='0')],
request_id='42',
properties=bbutil.dict_to_struct(props),
)
expected_request = creation.BuildRequest(
schedule_build_request=expected_sbr,
parameters={},
)
build = test_util.build(
id=1,
input=dict(
gerrit_changes=expected_sbr.gerrit_changes,
properties=expected_sbr.properties,
),
tags=expected_sbr.tags,
)
build.tags.append(buildset_tag)
build.tags.sort()
add_async.return_value = future(build)
req = {
'client_operation_id':
'42',
'bucket':
'luci.chromium.try',
'tags': [buildset_tag, 't:0'],
'parameters_json':
json.dumps({
api_common.BUILDER_PARAMETER: 'linux',
api_common.PROPERTIES_PARAMETER: props,
}),
}
resp = self.call_api('put', req).json_body
add_async.assert_called_once_with(expected_request)
self.assertEqual(resp['build']['id'], '1')
self.assertIn(buildset_tag, resp['build']['tags'])
self.assertIn('t:0', resp['build']['tags'])
