def test_valid(self):
msg = rpc_pb2.ScheduleBuildRequest(
builder=dict(project='chromium', bucket='try',
builder='linux-rel'),
gitiles_commit=dict(
host='gerrit.example.com',
project='project',
id='a' * 40,
ref='refs/heads/master',
position=1,
),
gerrit_changes=[
dict(host='gerrit.example.com', change=1, patchset=1),
dict(host='gerrit.example.com', change=2, patchset=2),
],
tags=[
dict(key='a', value='a1'),
dict(key='a', value='a2'),
dict(key='b', value='b1'),
],
dimensions=[
dict(key='d1', value='dv1'),
dict(key='d2', value='dv2'),
],
priority=100,
notify=dict(
pubsub_topic='projects/project_id/topics/topic_id',
user_data='blob',
),
)
msg.properties.update({'a': 1, '$recipe_engine/runtime': {'b': 1}})
self.assert_valid(msg)
def test_put_with_generic_buildset(self, add_async):
tags = [
dict(key='buildset', value='x'),
dict(key='t', value='0'),
]
build = test_util.build(id=1, tags=tags)
add_async.return_value = future(build)
req = {
'client_operation_id': '42',
'bucket': 'luci.chromium.try',
'tags': ['buildset:x', 't:0'],
'parameters_json': json.dumps({api_common.BUILDER_PARAMETER: 'linux'}),
}
resp = self.call_api('put', req).json_body
add_async.assert_called_once_with(
creation.BuildRequest(
schedule_build_request=rpc_pb2.ScheduleBuildRequest(
builder=dict(
project='chromium',
bucket='try',
builder='linux',
),
tags=tags,
request_id='42',
properties=dict(),
),
parameters={},
)
)
self.assertEqual(resp['build']['id'], '1')
self.assertIn('buildset:x', resp['build']['tags'])
self.assertIn('t:0', resp['build']['tags'])
def build_request(self, schedule_build_request_fields=None, **kwargs):
schedule_build_request_fields = schedule_build_request_fields or {}
sbr = rpc_pb2.ScheduleBuildRequest(**schedule_build_request_fields)
sbr.builder.project = sbr.builder.project or 'chromium'
sbr.builder.bucket = sbr.builder.bucket or 'try'
sbr.builder.builder = sbr.builder.builder or 'linux'
return creation.BuildRequest(schedule_build_request=sbr, **kwargs)
def test_forbidden(self):
user.can_async.return_value = future(False)
req = rpc_pb2.ScheduleBuildRequest(builder=dict(project='chromium',
bucket='try',
builder='linux'), )
self.call(self.api.ScheduleBuild,
req,
expected_code=prpc.StatusCode.PERMISSION_DENIED)
def test_priority(self):
msg = rpc_pb2.ScheduleBuildRequest(
builder=build_pb2.BuilderID(project='chromium',
bucket='try',
builder='linux-rel'),
priority=256,
)
self.assert_invalid(msg, r'priority: must be in \[0, 255\]')
def test_notify_pubsub_topic(self):
msg = rpc_pb2.ScheduleBuildRequest(
builder=build_pb2.BuilderID(project='chromium',
bucket='try',
builder='linux-rel'),
notify=notification_pb2.NotificationConfig(),
)
self.assert_invalid(msg, r'notify.pubsub_topic: required')
def test_cipd_version_latest(self):
msg = rpc_pb2.ScheduleBuildRequest(
builder=build_pb2.BuilderID(project='chromium',
bucket='try',
builder='linux-rel'),
exe=dict(cipd_version='latest'),
)
self.assert_valid(msg)
def test_tags(self):
msg = rpc_pb2.ScheduleBuildRequest(
builder=build_pb2.BuilderID(
project='chromium', bucket='try', builder='linux-rel'
),
tags=[dict()]
)
self.assert_invalid(msg, r'tags: Invalid tag ":": starts with ":"')
def test_dimensions(self):
msg = rpc_pb2.ScheduleBuildRequest(
builder=build_pb2.BuilderID(
project='chromium', bucket='try', builder='linux-rel'
),
dimensions=[dict()]
)
self.assert_invalid(msg, r'dimensions\[0\]\.key: required')
def test_cipd_version_invalid(self):
msg = rpc_pb2.ScheduleBuildRequest(
builder=build_pb2.BuilderID(project='chromium',
bucket='try',
builder='linux-rel'),
exe=dict(cipd_version=':'),
)
self.assert_invalid(msg, r'exe.cipd_version: invalid version ":"')
def test_expiration_not_suported(self):
msg = rpc_pb2.ScheduleBuildRequest(
builder=dict(project='chromium', bucket='try', builder='linux-rel'),
dimensions=[
dict(key='a', value='b', expiration=dict(seconds=10)),
],
)
self.assert_invalid(msg, r'dimensions\[0\]\.expiration: not supported')
def test_empty_property_value(self):
msg = rpc_pb2.ScheduleBuildRequest(
builder=dict(project='chromium', bucket='try', builder='linux-rel'),
properties=dict(fields=dict(a=struct_pb2.Value())),
)
self.assert_invalid(
msg, r'properties\.a: value is not set; for null, initialize null_value'
)
def test_cipd_package(self):
msg = rpc_pb2.ScheduleBuildRequest(
builder=build_pb2.BuilderID(project='chromium',
bucket='try',
builder='linux-rel'),
exe=dict(cipd_package='something'),
)
self.assert_invalid(msg, r'exe.cipd_package: disallowed')
def test_put_with_gerrit_change(self, add_async):
buildset = 'patch/gerrit/gerrit.example.com/1234/5'
buildset_tag = 'buildset:' + buildset
props = {'patch_project': 'repo'}
expected_sbr = rpc_pb2.ScheduleBuildRequest(
builder=dict(
project='chromium',
bucket='try',
builder='linux',
),
gerrit_changes=[
dict(
host='gerrit.example.com',
project='repo',
change=1234,
patchset=5,
)
],
tags=[dict(key='t', value='0')],
request_id='42',
properties=bbutil.dict_to_struct(props),
)
expected_request = creation.BuildRequest(
schedule_build_request=expected_sbr,
parameters={},
)
build = test_util.build(
id=1,
input=dict(
gerrit_changes=expected_sbr.gerrit_changes,
properties=expected_sbr.properties,
),
tags=expected_sbr.tags,
)
build.tags.append(buildset_tag)
build.tags.sort()
add_async.return_value = future(build)
req = {
'client_operation_id':
'42',
'bucket':
'luci.chromium.try',
'tags': [buildset_tag, 't:0'],
'parameters_json':
json.dumps({
api_common.BUILDER_PARAMETER: 'linux',
api_common.PROPERTIES_PARAMETER: props,
}),
}
resp = self.call_api('put', req).json_body
add_async.assert_called_once_with(expected_request)
self.assertEqual(resp['build']['id'], '1')
self.assertIn(buildset_tag, resp['build']['tags'])
self.assertIn('t:0', resp['build']['tags'])
def test_gitiles_commit_incomplete(self):
msg = rpc_pb2.ScheduleBuildRequest(
builder=build_pb2.BuilderID(project='chromium',
bucket='try',
builder='linux-rel'),
gitiles_commit=common_pb2.GitilesCommit(host='gerrit.example.com',
project='project'),
)
self.assert_invalid(msg, r'gitiles_commit\.ref: required')
def test_gerrit_change(self):
msg = rpc_pb2.ScheduleBuildRequest(
builder=build_pb2.BuilderID(project='chromium',
bucket='try',
builder='linux-rel'),
gerrit_changes=[
common_pb2.GerritChange(host='gerrit.example.com', change=2),
],
)
self.assert_invalid(msg, r'gerrit_changes\[0\]\.patchset: required')
def test_repeating_dimension_key_and_expiration(self):
msg = rpc_pb2.ScheduleBuildRequest(
builder=dict(project='chromium', bucket='try',
builder='linux-rel'),
dimensions=[
dict(key='a', value='b'),
dict(key='a', value='b'),
],
)
self.assert_invalid(
msg, r'dimensions: key "a" and expiration 0s are not unique')
def test_notify_user_data(self):
msg = rpc_pb2.ScheduleBuildRequest(
builder=build_pb2.BuilderID(project='chromium',
bucket='try',
builder='linux-rel'),
notify=notification_pb2.NotificationConfig(
pubsub_topic='x',
user_data='a' * 5000,
),
)
self.assert_invalid(msg, r'notify.user_data: must be <= 4096 bytes')
def test_put_with_commit(self, add_async):
buildset = (
'commit/gitiles/gitiles.example.com/chromium/src/+/'
'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'
)
buildset_tag = 'buildset:' + buildset
gitiles_ref_tag = 'gitiles_ref:refs/heads/master'
gitiles_commit = common_pb2.GitilesCommit(
host='gitiles.example.com',
project='chromium/src',
ref='refs/heads/master',
id='aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
)
build = test_util.build(
id=1,
input=dict(gitiles_commit=gitiles_commit),
tags=[dict(key='t', value='0')],
)
build.tags.append(buildset_tag)
build.tags.append(gitiles_ref_tag)
build.tags.sort()
add_async.return_value = future(build)
req = {
'client_operation_id': '42',
'bucket': 'luci.chromium.try',
'tags': [buildset_tag, gitiles_ref_tag, 't:0'],
'parameters_json': json.dumps({api_common.BUILDER_PARAMETER: 'linux'}),
}
resp = self.call_api('put', req).json_body
add_async.assert_called_once_with(
creation.BuildRequest(
schedule_build_request=rpc_pb2.ScheduleBuildRequest(
builder=dict(
project='chromium',
bucket='try',
builder='linux',
),
gitiles_commit=gitiles_commit,
tags=[dict(key='t', value='0')],
request_id='42',
properties=dict(),
),
parameters={},
)
)
self.assertEqual(resp['build']['id'], '1')
self.assertIn(buildset_tag, resp['build']['tags'])
self.assertIn(gitiles_ref_tag, resp['build']['tags'])
self.assertIn('t:0', resp['build']['tags'])
def test_schedule(self, add_async):
add_async.return_value = future(
test_util.build(
id=54,
builder=dict(project='chromium', bucket='try',
builder='linux'),
), )
req = rpc_pb2.ScheduleBuildRequest(builder=dict(project='chromium',
bucket='try',
builder='linux'), )
res = self.call(self.api.ScheduleBuild, req)
self.assertEqual(res.id, 54)
add_async.assert_called_once_with(
creation.BuildRequest(schedule_build_request=req))
def test_reserved_properties(self):
properties = [
{'buildbucket': 1},
{'buildername': 1},
{'$recipe_engine/runtime': {'is_luci': 1}},
{'$recipe_engine/runtime': {'is_experimental': 1}},
]
for p in properties:
msg = rpc_pb2.ScheduleBuildRequest(
builder=build_pb2.BuilderID(
project='chromium', bucket='try', builder='linux-rel'
),
)
msg.properties.update(p)
self.assert_invalid(msg, r'property .+ is reserved')
def retry(self, request):
"""Retries an existing build."""
lease_expiration_date = parse_datetime(request.lease_expiration_ts)
errors.validate_lease_expiration_date(lease_expiration_date)
build = model.Build.get_by_id(request.id)
if not build:
raise errors.BuildNotFoundError('Build %s not found' % request.id)
check_scheduling_permissions([build.bucket_id])
# Prepare v2 request.
sbr = rpc_pb2.ScheduleBuildRequest(
builder=build.proto.builder,
request_id=request.client_operation_id,
canary=common_pb2.YES if build.canary else common_pb2.NO,
properties=build.proto.input.properties,
gerrit_changes=build.proto.input.gerrit_changes[:],
)
build.tags_to_protos(sbr.tags)
if build.input_properties_bytes: # pragma: no branch
sbr.properties.ParseFromString(build.input_properties_bytes)
if build.proto.input.HasField('gitiles_commit'): # pragma: no branch
sbr.gitiles_commit.CopyFrom(build.proto.input.gitiles_commit)
# Read PubSub callback.
pubsub_callback_auth_token = None
if request.pubsub_callback: # pragma: no branch
pubsub_callback_auth_token = request.pubsub_callback.auth_token
pubsub_callback_to_notification_config(request.pubsub_callback,
sbr.notify)
with _wrap_validation_error():
validation.validate_notification_config(sbr.notify)
# Remove properties from parameters.
params = build.parameters.copy()
params.pop(model.PROPERTIES_PARAMETER, None)
# Create the build.
build_req = creation.BuildRequest(
schedule_build_request=sbr,
parameters=params,
lease_expiration_date=lease_expiration_date,
retry_of=request.id,
pubsub_callback_auth_token=pubsub_callback_auth_token,
)
build = creation.add_async(build_req).get_result()
return build_to_response_message(build, include_lease_key=True)
def test_put(self, add_async):
build = test_util.build(id=1, tags=[dict(key='a', value='b')])
add_async.return_value = future(build)
props = {'foo': 'bar'}
parameters_json = json.dumps({
api_common.BUILDER_PARAMETER: 'linux',
api_common.PROPERTIES_PARAMETER: props,
})
req = {
'client_operation_id': '42',
'bucket': 'luci.chromium.try',
'tags': ['a:b'],
'parameters_json': parameters_json,
'pubsub_callback': {
'topic': 'projects/foo/topic/bar',
'user_data': 'hello',
'auth_token': 'secret',
},
}
resp = self.call_api('put', req).json_body
add_async.assert_called_once_with(
creation.BuildRequest(
schedule_build_request=rpc_pb2.ScheduleBuildRequest(
builder=dict(
project='chromium',
bucket='try',
builder='linux',
),
tags=[dict(key='a', value='b')],
request_id='42',
notify=dict(
pubsub_topic='projects/foo/topic/bar',
user_data='hello',
),
properties=bbutil.dict_to_struct(props),
),
parameters={},
pubsub_callback_auth_token='secret',
)
)
self.assertEqual(resp['build']['id'], '1')
self.assertEqual(resp['build']['bucket'], req['bucket'])
self.assertIn('a:b', resp['build']['tags'])
def test_put_with_v2_gerrit_changes(self, add_async):
changes = [
common_pb2.GerritChange(
host='chromium.googlesource.com',
project='project',
change=1,
patchset=1,
),
common_pb2.GerritChange(
host='chromium.googlesource.com',
project='project',
change=2,
patchset=1,
),
]
expected_sbr = rpc_pb2.ScheduleBuildRequest(
builder=dict(
project='chromium',
bucket='try',
builder='linux',
),
properties=dict(),
gerrit_changes=changes,
)
expected_request = creation.BuildRequest(
schedule_build_request=expected_sbr,
parameters={},
)
add_async.return_value = future(test_util.build(id=1))
params = {
api_common.BUILDER_PARAMETER: 'linux',
'gerrit_changes': [json_format.MessageToDict(c) for c in changes],
}
req = {
'bucket': 'luci.chromium.try',
'parameters_json': json.dumps(params),
}
self.call_api('put', req)
add_async.assert_called_once_with(expected_request)
def test_retry(self, add_async):
props = bbutil.dict_to_struct({
'foo': 'bar',
'recipe': 'recipe',
})
orig_bundle = test_util.build_bundle(
id=1,
input=dict(
properties=props,
gitiles_commit=dict(
host='gitiles.example.com',
project='chromium/src',
id='a' * 40,
),
),
)
orig_build = orig_bundle.build
orig_build.parameters.pop('changes')
orig_build.tags = ['a:b']
ndb.put_multi([orig_build, orig_bundle.input_properties])
retried_build_bundle = test_util.build_bundle(
id=2,
input=dict(
properties=orig_build.proto.input.properties,
gitiles_commit=orig_build.proto.input.gitiles_commit,
),
)
retried_build_bundle.infra.put()
retried_build = retried_build_bundle.build
retried_build.retry_of = 1
add_async.return_value = future(retried_build)
req = {
'id': '1',
'client_operation_id': '42',
'pubsub_callback': {
'topic': 'projects/foo/topic/bar',
'user_data': 'hello',
'auth_token': 'secret',
},
}
resp = self.call_api('retry', req).json_body
add_async.assert_called_once_with(
creation.BuildRequest(
schedule_build_request=rpc_pb2.ScheduleBuildRequest(
builder=orig_build.proto.builder,
request_id='42',
notify=dict(
pubsub_topic='projects/foo/topic/bar',
user_data='hello',
),
properties=props,
tags=[dict(key='a', value='b')],
canary=common_pb2.NO,
gitiles_commit=orig_build.proto.input.gitiles_commit,
),
parameters={},
lease_expiration_date=None,
retry_of=1,
pubsub_callback_auth_token='secret',
)
)
self.assertEqual(resp['build']['id'], '2')
self.assertEqual(resp['build']['bucket'], 'luci.chromium.try')
self.assertEqual(resp['build']['retry_of'], '1')
def put_request_message_to_build_request(put_request):
"""Converts PutRequest to BuildRequest.
Raises errors.InvalidInputError if the put_request is invalid.
"""
lease_expiration_date = parse_datetime(put_request.lease_expiration_ts)
errors.validate_lease_expiration_date(lease_expiration_date)
# Read parameters.
parameters = parse_json_object(put_request.parameters_json,
'parameters_json')
parameters = parameters or {}
validate_known_build_parameters(parameters)
builder = parameters.get(model.BUILDER_PARAMETER) or ''
# Validate tags.
buildtags.validate_tags(put_request.tags, 'new', builder=builder)
# Read properties. Remove them from parameters.
props = parameters.pop(model.PROPERTIES_PARAMETER, None)
if props is not None and not isinstance(props, dict):
raise errors.InvalidInputError(
'"properties" parameter must be a JSON object or null')
props = props or {}
changes = parameters.get(_PARAM_CHANGES)
if changes: # pragma: no branch
# Buildbucket-Buildbot integration passes repo_url of the first change in
# build parameter "changes" as "repository" attribute of SourceStamp.
# https://chromium.googlesource.com/chromium/tools/build/+/2c6023d
# /scripts/master/buildbucket/changestore.py#140
# Buildbot passes repository of the build source stamp as "repository"
# build property. Recipes, in partiular bot_update recipe module, rely on
# "repository" property and it is an almost sane property to support in
# swarmbucket.
repo_url = changes[0].get('repo_url')
if repo_url: # pragma: no branch
props['repository'] = repo_url
# Buildbot-Buildbucket integration converts emails in changes to blamelist
# property.
emails = [c.get('author', {}).get('email') for c in changes]
props['blamelist'] = filter(None, emails)
# Create a v2 request.
sbr = rpc_pb2.ScheduleBuildRequest(
builder=build_pb2.BuilderID(builder=builder),
properties=bbutil.dict_to_struct(props),
request_id=put_request.client_operation_id,
experimental=bbutil.BOOLISH_TO_TRINARY[put_request.experimental],
canary=api_common.CANARY_PREFERENCE_TO_TRINARY.get(
put_request.canary_preference, common_pb2.UNSET),
)
sbr.builder.project, sbr.builder.bucket = config.parse_bucket_id(
put_request.bucket)
# Parse tags. Extract gitiles commit and gerrit changes.
tags, gitiles_commit, gerrit_changes = parse_v1_tags(put_request.tags)
sbr.tags.extend(tags)
if gitiles_commit:
sbr.gitiles_commit.CopyFrom(gitiles_commit)
# Gerrit changes explicitly passed via "gerrit_changes" parameter win.
gerrit_change_list = parameters.pop('gerrit_changes', None)
if gerrit_change_list is not None:
if not isinstance(gerrit_change_list, list): # pragma: no cover
raise errors.InvalidInputError('gerrit_changes must be a list')
try:
gerrit_changes = [
json_format.ParseDict(c, common_pb2.GerritChange())
for c in gerrit_change_list
]
except json_format.ParseError as ex: # pragma: no cover
raise errors.InvalidInputError('Invalid gerrit_changes: %s' % ex)
sbr.gerrit_changes.extend(gerrit_changes)
if (not gerrit_changes and
not sbr.builder.bucket.startswith('master.')): # pragma: no cover
changes = parameters.get('changes')
if isinstance(changes, list) and changes and not gitiles_commit:
legacy_revision = changes[0].get('revision')
if legacy_revision:
raise errors.InvalidInputError(
'legacy revision without gitiles buildset tag')
# Populate Gerrit project from patch_project property.
# V2 API users will have to provide this.
patch_project = props.get('patch_project')
if len(sbr.gerrit_changes) == 1 and isinstance(patch_project, basestring):
sbr.gerrit_changes[0].project = patch_project
# Read PubSub callback.
pubsub_callback_auth_token = None
if put_request.pubsub_callback:
pubsub_callback_auth_token = put_request.pubsub_callback.auth_token
pubsub_callback_to_notification_config(put_request.pubsub_callback,
sbr.notify)
# Validate the resulting v2 request before continuing.
with _wrap_validation_error():
validation.validate_schedule_build_request(sbr, legacy=True)
return creation.BuildRequest(
schedule_build_request=sbr,
parameters=parameters,
lease_expiration_date=lease_expiration_date,
pubsub_callback_auth_token=pubsub_callback_auth_token,
override_builder_cfg=_override_builder_cfg_func(parameters),
)
def test_put_batch(self, add_many_async):
bundle1 = test_util.build_bundle(id=1, tags=[dict(key='a', value='b')])
bundle2 = test_util.build_bundle(id=2)
bundle1.infra.put()
bundle2.infra.put()
config.put_bucket(
'chromium',
'a' * 40,
test_util.parse_bucket_cfg(
'''
name: "luci.chromium.try"
acls {
role: SCHEDULER
identity: "anonymous:anonymous"
}
'''
),
)
add_many_async.return_value = future([
(bundle1.build, None),
(bundle2.build, None),
(None, errors.InvalidInputError('bad')),
])
req = {
'builds': [
{
'bucket': 'luci.chromium.try',
'tags': ['a:b'],
'client_operation_id': '0',
},
{
'bucket': 'luci.chromium.try',
'client_operation_id': '1',
},
{
'bucket': 'luci.chromium.try',
'tags': ['bad tag'],
'client_operation_id': '2',
},
{
'bucket': 'luci.chromium.try',
'client_operation_id': '3',
},
],
}
resp = self.call_api('put_batch', req).json_body
add_many_async.assert_called_once_with([
creation.BuildRequest(
schedule_build_request=rpc_pb2.ScheduleBuildRequest(
builder=dict(project='chromium', bucket='try'),
tags=[dict(key='a', value='b')],
request_id='0',
properties=dict(),
),
parameters={},
),
creation.BuildRequest(
schedule_build_request=rpc_pb2.ScheduleBuildRequest(
builder=dict(project='chromium', bucket='try'),
request_id='1',
properties=dict(),
),
parameters={},
),
creation.BuildRequest(
schedule_build_request=rpc_pb2.ScheduleBuildRequest(
builder=dict(project='chromium', bucket='try'),
request_id='3',
properties=dict(),
),
parameters={},
),
])
res0 = resp['results'][0]
self.assertEqual(res0['client_operation_id'], '0')
self.assertEqual(res0['build']['id'], '1')
self.assertEqual(res0['build']['bucket'], 'luci.chromium.try')
res1 = resp['results'][1]
self.assertEqual(res1['client_operation_id'], '1')
self.assertEqual(res1['build']['id'], '2')
self.assertEqual(res1['build']['bucket'], 'luci.chromium.try')
res2 = resp['results'][2]
self.assertEqual(
res2, {
'client_operation_id': '2',
'error': {
'reason': 'INVALID_INPUT',
'message': u'Invalid tag "bad tag": does not contain ":"',
},
}
)
res3 = resp['results'][3]
self.assertEqual(
res3, {
'client_operation_id': '3',
'error': {
'reason': 'INVALID_INPUT',
'message': 'bad',
},
}
)
def test_no_builder_and_template_build_id(self):
msg = rpc_pb2.ScheduleBuildRequest()
self.assert_invalid(msg, 'builder or template_build_id is required')
def test_incomplete_builder(self):
msg = rpc_pb2.ScheduleBuildRequest(builder=build_pb2.BuilderID(
project='chromium', bucket='try'), )
self.assert_invalid(msg, 'builder.builder: required')
def test_no_builder_but_template_build_id(self):
msg = rpc_pb2.ScheduleBuildRequest(template_build_id=1)
self.assert_valid(msg)
