def test_without_buckets(self):
self.mock_cannot(user.Action.SEARCH_BUILDS, 'chromium/ci')
build1 = self.put_build(builder=dict(bucket='try'))
build2 = self.put_build(builder=dict(bucket='ci'))
builds, _ = self.search()
self.assertEqual(builds, [build1])
builds, _ = self.search(tags=[self.INDEXED_TAG])
self.assertEqual(builds, [build1])
# All buckets are available.
user.get_accessible_buckets_async.return_value = future(None)
user.can_async.side_effect = None
builds, _ = self.search()
self.assertEqual(builds, [build2, build1])
builds, _ = self.search(tags=[self.INDEXED_TAG])
self.assertEqual(builds, [build2, build1])
# No buckets are available.
user.get_accessible_buckets_async.return_value = future(set())
self.mock_cannot(user.Action.SEARCH_BUILDS)
builds, _ = self.search()
self.assertEqual(builds, [])
builds, _ = self.search(tags=[self.INDEXED_TAG])
self.assertEqual(builds, [])
def setUp(self):
super(BaseTest, self).setUp()
user.clear_request_cache()
self.patch('tq.enqueue_async', autospec=True, return_value=future(None))
self.now = NOW
self.patch(
'components.utils.utcnow', autospec=True, side_effect=lambda: self.now
)
self.settings = service_config_pb2.SettingsCfg(
swarming=dict(
milo_hostname='milo.example.com',
bbagent_package=dict(
package_name='infra/tools/bbagent',
version='luci-runner-version',
version_canary='luci-runner-version-canary',
builders=service_config_pb2.BuilderPredicate(
regex=['chromium/try/linux'],
),
),
kitchen_package=dict(
package_name='infra/tools/kitchen',
version='kitchen-version',
version_canary='kitchen-version-canary',
),
user_packages=[
dict(
package_name='infra/tools/git',
version='git-version',
version_canary='git-version-canary',
),
dict(
package_name='infra/cpython/python',
version='py-version',
subdir='python',
),
dict(
package_name='infra/excluded',
version='excluded-version',
builders=service_config_pb2.BuilderPredicate(
regex_exclude=['.*'],
),
),
],
),
known_public_gerrit_hosts=['chromium-review.googlesource.com'],
)
self.patch(
'config.get_settings_async',
autospec=True,
return_value=future(self.settings)
)
def setUp(self):
super(ExpireBuildTests, self).setUp()
self.now = datetime.datetime(2015, 1, 1)
self.patch('components.utils.utcnow', side_effect=lambda: self.now)
self.patch('tq.enqueue_async',
autospec=True,
return_value=future(None))
def setUp(self):
super(SyncBuildTest, self).setUp()
self.patch('components.net.json_request_async', autospec=True)
self.patch('components.auth.delegate_async', return_value=future('blah'))
self.build_token = 'beeff00d'
self.patch(
'tokens.generate_build_token', autospec=True, return_value='deadbeef'
)
self.task_def = {'is_task_def': True, 'task_slices': [{
'properties': {},
}]}
self.patch(
'swarming.compute_task_def', autospec=True, return_value=self.task_def
)
self.patch(
'google.appengine.api.app_identity.get_default_version_hostname',
return_value='cr-buildbucket.appspot.com'
)
self.build_bundle = test_util.build_bundle(
id=1, created_by='user:[email protected]'
)
self.build_bundle.build.swarming_task_key = None
with self.build_bundle.infra.mutate() as infra:
infra.swarming.task_id = ''
self.build_bundle.put()
def test_put_with_generic_buildset(self, add_async):
tags = [
dict(key='buildset', value='x'),
dict(key='t', value='0'),
]
build = test_util.build(id=1, tags=tags)
add_async.return_value = future(build)
req = {
'client_operation_id': '42',
'bucket': 'luci.chromium.try',
'tags': ['buildset:x', 't:0'],
'parameters_json': json.dumps({api_common.BUILDER_PARAMETER: 'linux'}),
}
resp = self.call_api('put', req).json_body
add_async.assert_called_once_with(
creation.BuildRequest(
schedule_build_request=rpc_pb2.ScheduleBuildRequest(
builder=dict(
project='chromium',
bucket='try',
builder='linux',
),
tags=tags,
request_id='42',
properties=dict(),
),
parameters={},
)
)
self.assertEqual(resp['build']['id'], '1')
self.assertIn('buildset:x', resp['build']['tags'])
self.assertIn('t:0', resp['build']['tags'])
def setUp(self):
super(CreateTaskTest, self).setUp()
self.patch('components.net.json_request_async', autospec=True)
self.patch('components.auth.delegate_async',
return_value=future('blah'))
self.build_token = 'beeff00d'
self.patch('tokens.generate_build_token',
autospec=True,
return_value='deadbeef')
self.task_def = {
'is_task_def': True,
'task_slices': [{
'properties': {},
}]
}
self.patch('swarming.prepare_task_def',
autospec=True,
return_value=self.task_def)
self.build = test_util.build(id=1, created_by='user:[email protected]')
self.build.swarming_task_key = None
with self.build.mutate_infra() as infra:
infra.swarming.task_id = ''
self.build.put()
def test_success(self):
expected_task_def = self.task_def.copy()
expected_secrets = launcher_pb2.BuildSecrets(
build_token=self.build_token)
expected_task_def[u'task_slices'][0][u'properties'][
u'secret_bytes'] = (base64.b64encode(
expected_secrets.SerializeToString()))
net.json_request_async.return_value = future({'task_id': 'x'})
swarming._create_swarming_task(1)
actual_task_def = net.json_request_async.call_args[1]['payload']
self.assertEqual(actual_task_def, expected_task_def)
self.assertEqual(
net.json_request_async.call_args[0][0],
'https://swarming.example.com/_ah/api/swarming/v1/tasks/new')
# Test delegation token params.
self.assertEqual(auth.delegate_async.mock_calls, [
mock.call(
services=[u'https://swarming.example.com'],
audience=[auth.Identity('user', 'test@localhost')],
impersonate=auth.Identity('user', '*****@*****.**'),
tags=['buildbucket:bucket:chromium/try'],
)
])
def test_filter_by_project_admin(self):
user.get_accessible_buckets_async.return_value = future(None)
build = self.put_build(builder=dict(project='chromium'))
self.put_build(builder=dict(project='v8'))
builds, _ = self.search(project='chromium')
self.assertEqual(builds, [build])
def test_put_batch_with_exception(self, add_many_async):
add_many_async.return_value = future([(None, Exception())])
req = {
'builds': [{
'bucket': 'luci.chromium.try'
}],
}
self.call_api('put_batch', req, status=500)
def test_get_build_url_milo(self, get_settings_async):
get_settings_async.return_value = test_util.future(
service_config_pb2.SettingsCfg(swarming=dict(
milo_hostname='milo.example.com')), )
build = test_util.build(id=1)
build.url = None
self.assertEqual(api_common.get_build_url(build),
'https://milo.example.com/b/1')
def test_put_with_gerrit_change(self, add_async):
buildset = 'patch/gerrit/gerrit.example.com/1234/5'
buildset_tag = 'buildset:' + buildset
props = {'patch_project': 'repo'}
expected_sbr = rpc_pb2.ScheduleBuildRequest(
builder=dict(
project='chromium',
bucket='try',
builder='linux',
),
gerrit_changes=[
dict(
host='gerrit.example.com',
project='repo',
change=1234,
patchset=5,
)
],
tags=[dict(key='t', value='0')],
request_id='42',
properties=bbutil.dict_to_struct(props),
)
expected_request = creation.BuildRequest(
schedule_build_request=expected_sbr,
parameters={},
)
build = test_util.build(
id=1,
input=dict(
gerrit_changes=expected_sbr.gerrit_changes,
properties=expected_sbr.properties,
),
tags=expected_sbr.tags,
)
build.tags.append(buildset_tag)
build.tags.sort()
add_async.return_value = future(build)
req = {
'client_operation_id':
'42',
'bucket':
'luci.chromium.try',
'tags': [buildset_tag, 't:0'],
'parameters_json':
json.dumps({
api_common.BUILDER_PARAMETER: 'linux',
api_common.PROPERTIES_PARAMETER: props,
}),
}
resp = self.call_api('put', req).json_body
add_async.assert_called_once_with(expected_request)
self.assertEqual(resp['build']['id'], '1')
self.assertIn(buildset_tag, resp['build']['tags'])
self.assertIn('t:0', resp['build']['tags'])
def test_sync_build_async_no_task(self, load_task_result_async):
load_task_result_async.return_value = future(None)
build = test_util.build()
build.put()
swarming.CronUpdateBuilds().update_build_async(build).get_result()
build = build.key.get()
self.assertEqual(build.proto.status, common_pb2.INFRA_FAILURE)
self.assertTrue(build.proto.summary_markdown)
def setUp(self):
super(UserTest, self).setUp()
self.current_identity = auth.Identity.from_bytes('user:[email protected]')
self.patch('components.auth.get_current_identity',
autospec=True,
side_effect=lambda: self.current_identity)
user.clear_request_cache()
self.patch('components.auth.is_admin',
autospec=True,
return_value=False)
bucket_a = Bucket(name='a',
acls=[
Acl(role=Acl.WRITER, group='a-writers'),
Acl(role=Acl.READER, group='a-readers'),
])
bucket_b = Bucket(name='b',
acls=[
Acl(role=Acl.WRITER, group='b-writers'),
Acl(role=Acl.READER, group='b-readers'),
])
bucket_c = Bucket(name='c',
acls=[
Acl(role=Acl.READER, group='c-readers'),
Acl(role=Acl.READER,
identity='user:[email protected]'),
Acl(role=Acl.WRITER, group='c-writers'),
Acl(role=Acl.READER, identity='project:p1'),
])
all_buckets = [('p1', bucket_a), ('p2', bucket_b), ('p3', bucket_c)]
self.patch('config.get_buckets_async',
autospec=True,
return_value=future(all_buckets))
bucket_map = {
config.format_bucket_id(pid, b.name): b
for pid, b in all_buckets
}
self.patch('config.get_bucket_async',
autospec=True,
side_effect=lambda bid: future(
('deadbeef', bucket_map.get(bid))))
def test_update_global_metrics(self, set_build_count_metric_async,
set_build_latency):
set_build_count_metric_async.return_value = future(None)
set_build_latency.return_value = future(None)
model.Builder(id='chromium:luci.chromium.try:release').put()
model.Builder(id='chromium:luci.chromium.try:debug').put()
model.Builder(id='chromium:try:debug').put()
config.put_bucket(
'chromium',
'a' * 40,
test_util.parse_bucket_cfg('''
name: "luci.chromium.try"
swarming {
builders {}
}
'''),
)
metrics.update_global_metrics()
set_build_latency.assert_any_call('chromium/try', 'luci.chromium.try',
'release', True)
set_build_latency.assert_any_call('chromium/try', 'luci.chromium.try',
'release', False)
set_build_latency.assert_any_call('chromium/try', 'luci.chromium.try',
'debug', True)
set_build_latency.assert_any_call('chromium/try', 'luci.chromium.try',
'debug', False)
set_build_count_metric_async.assert_any_call(
'chromium/try', 'luci.chromium.try', 'release',
model.BuildStatus.SCHEDULED, False)
set_build_count_metric_async.assert_any_call(
'chromium/try', 'luci.chromium.try', 'release',
model.BuildStatus.SCHEDULED, True)
set_build_count_metric_async.assert_any_call(
'chromium/try', 'luci.chromium.try', 'debug',
model.BuildStatus.SCHEDULED, False)
set_build_count_metric_async.assert_any_call(
'chromium/try', 'luci.chromium.try', 'debug',
model.BuildStatus.SCHEDULED, True)
def test_update_global_metrics(self, set_build_status_metric,
get_buckets_async):
get_buckets_async.return_value = future(
[project_config_pb2.Bucket(name='x')])
metrics.update_global_metrics()
set_build_status_metric.assert_any_call(metrics.CURRENTLY_PENDING, 'x',
model.BuildStatus.SCHEDULED)
set_build_status_metric.assert_any_call(metrics.CURRENTLY_RUNNING, 'x',
model.BuildStatus.STARTED)
def test_delegate_async(self, delegate_async):
delegate_async.return_value = future('token')
token = user.delegate_async('swarming.example.com',
tag='buildbucket:bucket:x').get_result()
self.assertEqual(token, 'token')
delegate_async.assert_called_with(
audience=[user.self_identity()],
services=['https://swarming.example.com'],
impersonate=auth.get_current_identity(),
tags=['buildbucket:bucket:x'],
)
def setUp(self):
super(SearchTest, self).setUp()
self.current_identity = auth.Identity('service', 'unittest')
self.patch('components.auth.get_current_identity',
side_effect=lambda: self.current_identity)
self.patch('user.can_async', return_value=future(True))
self.now = datetime.datetime(2015, 1, 1)
self.patch('components.utils.utcnow', side_effect=lambda: self.now)
self.chromium_try = project_config_pb2.Bucket(name='try')
self.patch('config.get_bucket_async',
return_value=future({'deadbeef': self.chromium_try}))
self.patch(
'user.get_accessible_buckets_async',
autospec=True,
return_value=future({'chromium/try'}),
)
self.patch('search.TagIndex.random_shard_index', return_value=0)
def test_sync_build_async(self, load_task_result_async):
load_task_result_async.return_value = future({
'state': 'RUNNING',
})
build = test_util.build()
build.put()
swarming.CronUpdateBuilds().update_build_async(build).get_result()
build = build.key.get()
self.assertEqual(build.proto.status, common_pb2.STARTED)
self.assertFalse(build.proto.HasField('end_time'))
load_task_result_async.return_value = future({
'state': 'COMPLETED',
})
swarming.CronUpdateBuilds().update_build_async(build).get_result()
build = build.key.get()
self.assertEqual(build.proto.status, common_pb2.SUCCESS)
def test_create_task(self):
expected_task_def = self.task_def.copy()
expected_secrets = launcher_pb2.BuildSecrets(build_token=self.build_token)
expected_task_def[u'task_slices'][0][u'properties'][u'secret_bytes'] = (
base64.b64encode(expected_secrets.SerializeToString())
)
net.json_request_async.return_value = future({'task_id': 'x'})
swarming._sync_build_and_swarming(1, 0)
actual_task_def = net.json_request_async.call_args[1]['payload']
self.assertEqual(actual_task_def, expected_task_def)
self.assertEqual(
net.json_request_async.call_args[0][0],
'https://swarming.example.com/_ah/api/swarming/v1/tasks/new'
)
# Test delegation token params.
self.assertEqual(
auth.delegate_async.mock_calls, [
mock.call(
services=[u'https://swarming.example.com'],
audience=[auth.Identity('user', 'test@localhost')],
impersonate=auth.Identity('user', '*****@*****.**'),
tags=['buildbucket:bucket:chromium/try'],
)
]
)
# Assert that we've persisted information about the new task.
bundle = model.BuildBundle.get(1, infra=True)
self.assertIsNotNone(bundle)
self.assertTrue(bundle.build.swarming_task_key)
self.assertTrue(bundle.infra.parse().swarming.task_id)
expected_continuation_payload = {
'id': 1,
'generation': 1,
}
expected_continuation = {
'name': 'sync-task-1-1',
'url': '/internal/task/swarming/sync-build/1',
'payload': json.dumps(expected_continuation_payload, sort_keys=True),
'retry_options': {
'task_age_limit': model.BUILD_TIMEOUT.total_seconds()
},
'countdown': 60,
}
tq.enqueue_async.assert_called_with(
swarming.SYNC_QUEUE_NAME, [expected_continuation], transactional=False
)
def test_update_build_async(self, load_task_result_async):
load_task_result_async.return_value = future({
'state': 'RUNNING',
})
build = self.build
swarming.CronUpdateBuilds().update_build_async(build).get_result()
build = build.key.get()
self.assertEqual(build.status, model.BuildStatus.STARTED)
self.assertIsNotNone(build.lease_key)
self.assertIsNone(build.complete_time)
load_task_result_async.return_value = future({
'state': 'COMPLETED',
})
swarming.CronUpdateBuilds().update_build_async(build).get_result()
build = build.key.get()
self.assertEqual(build.status, model.BuildStatus.COMPLETED)
self.assertEqual(build.result, model.BuildResult.SUCCESS)
self.assertIsNone(build.lease_key)
self.assertIsNotNone(build.complete_time)
def test_update_build_async_no_task(self, load_task_result_async):
load_task_result_async.return_value = future(None)
build = self.build
swarming.CronUpdateBuilds().update_build_async(build).get_result()
build = build.key.get()
self.assertEqual(build.status, model.BuildStatus.COMPLETED)
self.assertEqual(build.result, model.BuildResult.FAILURE)
self.assertEqual(build.failure_reason,
model.FailureReason.INFRA_FAILURE)
self.assertIsNotNone(build.result_details)
self.assertIsNone(build.lease_key)
self.assertIsNotNone(build.complete_time)
def get_self_config_async(path, *_args, **_kwargs):
if path not in (
'swarming_task_template.json',
'swarming_task_template_canary.json'): # pragma: no cover
self.fail()
if path == 'swarming_task_template.json':
template = self.task_template
else:
template = self.task_template_canary
return future(
('template_rev',
json.dumps(template) if template is not None else None))
def setUp(self):
super(AclTest, self).setUp()
self.current_identity = auth.Identity.from_bytes('user:[email protected]')
self.patch(
'components.auth.get_current_identity',
autospec=True,
return_value=self.current_identity)
self.patch('components.auth.is_admin', autospec=True, return_value=False)
bucket_a = Bucket(
name='a',
acls=[
Acl(role=Acl.WRITER, group='a-writers'),
Acl(role=Acl.READER, group='a-readers'),
])
bucket_b = Bucket(
name='b',
acls=[
Acl(role=Acl.WRITER, group='b-writers'),
Acl(role=Acl.READER, group='b-readers'),
])
bucket_c = Bucket(
name='c',
acls=[
Acl(role=Acl.READER, group='c-readers'),
Acl(role=Acl.READER, identity='user:[email protected]'),
Acl(role=Acl.WRITER, group='c-writers'),
])
all_buckets = [bucket_a, bucket_b, bucket_c]
self.patch(
'config.get_buckets_async', autospec=True,
return_value=future(all_buckets))
bucket_map = {b.name: b for b in all_buckets}
self.patch(
'config.get_bucket_async',
autospec=True,
side_effect=lambda name: future(('chromium', bucket_map.get(name))))
def test_put_with_commit(self, add_async):
buildset = (
'commit/gitiles/gitiles.example.com/chromium/src/+/'
'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'
)
buildset_tag = 'buildset:' + buildset
gitiles_ref_tag = 'gitiles_ref:refs/heads/master'
gitiles_commit = common_pb2.GitilesCommit(
host='gitiles.example.com',
project='chromium/src',
ref='refs/heads/master',
id='aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
)
build = test_util.build(
id=1,
input=dict(gitiles_commit=gitiles_commit),
tags=[dict(key='t', value='0')],
)
build.tags.append(buildset_tag)
build.tags.append(gitiles_ref_tag)
build.tags.sort()
add_async.return_value = future(build)
req = {
'client_operation_id': '42',
'bucket': 'luci.chromium.try',
'tags': [buildset_tag, gitiles_ref_tag, 't:0'],
'parameters_json': json.dumps({api_common.BUILDER_PARAMETER: 'linux'}),
}
resp = self.call_api('put', req).json_body
add_async.assert_called_once_with(
creation.BuildRequest(
schedule_build_request=rpc_pb2.ScheduleBuildRequest(
builder=dict(
project='chromium',
bucket='try',
builder='linux',
),
gitiles_commit=gitiles_commit,
tags=[dict(key='t', value='0')],
request_id='42',
properties=dict(),
),
parameters={},
)
)
self.assertEqual(resp['build']['id'], '1')
self.assertIn(buildset_tag, resp['build']['tags'])
self.assertIn(gitiles_ref_tag, resp['build']['tags'])
self.assertIn('t:0', resp['build']['tags'])
def callback_test(self, build):
with mock.patch('notifications.enqueue_notifications_async',
autospec=True):
notifications.enqueue_notifications_async.return_value = future(
None)
build.pubsub_callback = model.PubSubCallback(
topic='projects/example/topics/buildbucket',
user_data='hello',
auth_token='secret',
)
build.put()
yield
build = build.key.get()
notifications.enqueue_notifications_async.assert_called_with(build)
def test_get_accessible_buckets_async_admin(self, is_admin):
is_admin.return_value = True
config.get_buckets_async.return_value = future([
('p1',
Bucket(
name='available_bucket1',
acls=[
Acl(role=Acl.READER, group='xxx'),
Acl(role=Acl.WRITER, group='yyy')
],
)),
])
self.assertIsNone(user.get_accessible_buckets_async().get_result())
def test_enqueue_bq_export_async(self, enqueue_async):
enqueue_async.return_value = test_util.future(None)
build = test_util.build(id=1, status=common_pb2.SUCCESS)
ndb.transactional( # pylint: disable=no-value-for-parameter
lambda: bq.enqueue_bq_export_async(build).get_result())()
task_def = {
'method': 'PULL',
'payload': {
'id': 1
},
}
enqueue_async.assert_any_call('bq-export', [task_def])
def test_put_with_leasing(self, add_async):
expiration = utils.utcnow() + datetime.timedelta(hours=1)
build = test_util.build(id=1)
build.lease_expiration_date = expiration
add_async.return_value = future(build)
req = {
'bucket': 'luci.chromium.try',
'lease_expiration_ts': str(utils.datetime_to_timestamp(expiration)),
}
resp = self.call_api('put', req).json_body
build_req = add_async.call_args[0][0]
self.assertEqual(build_req.lease_expiration_date, expiration)
self.assertEqual(
resp['build']['lease_expiration_ts'], req['lease_expiration_ts']
)
def test_get_accessible_buckets_async(self, is_group_member):
is_group_member.side_effect = lambda g, _=None: g in ('xxx', 'yyy')
config.get_buckets_async.return_value = future({
'p1/available_bucket1':
Bucket(
name='available_bucket1',
acls=[
Acl(role=Acl.READER, group='xxx'),
Acl(role=Acl.WRITER, group='yyy')
],
),
'p2/available_bucket2':
Bucket(
name='available_bucket2',
acls=[
Acl(role=Acl.READER, group='xxx'),
Acl(role=Acl.WRITER, group='zzz')
],
),
'p3/available_bucket3':
Bucket(
name='available_bucket3',
acls=[
Acl(role=Acl.READER, identity='user:[email protected]'),
],
),
'p4/not_available_bucket':
Bucket(
name='not_available_bucket',
acls=[Acl(role=Acl.WRITER, group='zzz')],
),
})
# call twice for per-request caching of futures.
user.get_accessible_buckets_async()
availble_buckets = user.get_accessible_buckets_async().get_result()
expected = {
'p1/available_bucket1',
'p2/available_bucket2',
'p3/available_bucket3',
}
self.assertEqual(availble_buckets, expected)
# call again for memcache coverage.
user.clear_request_cache()
availble_buckets = user.get_accessible_buckets_async().get_result()
self.assertEqual(availble_buckets, expected)
def test_cancel_with_details(self, cancel):
build = test_util.build(id=1)
cancel.return_value = future(build)
props = {'a': 'b'}
model.BuildOutputProperties(
key=model.BuildOutputProperties.key_for(build.key),
properties=bbutil.dict_to_struct(props).SerializeToString(),
).put()
result_details = {'properties': props}
req = {'id': '1', 'result_details_json': json.dumps(result_details)}
res = self.call_api('cancel', req).json_body
cancel.assert_called_once_with(1, result_details=result_details)
self.assertEqual(res['build']['result_details_json'],
req['result_details_json'])
