def build_request(self, schedule_build_request_fields=None, **kwargs):
schedule_build_request_fields = schedule_build_request_fields or {}
sbr = rpc_pb2.ScheduleBuildRequest(**schedule_build_request_fields)
sbr.builder.project = sbr.builder.project or 'chromium'
sbr.builder.bucket = sbr.builder.bucket or 'try'
sbr.builder.builder = sbr.builder.builder or 'linux'
return creation.BuildRequest(schedule_build_request=sbr, **kwargs)
def test_put_with_generic_buildset(self, add_async):
tags = [
dict(key='buildset', value='x'),
dict(key='t', value='0'),
]
build = test_util.build(id=1, tags=tags)
add_async.return_value = future(build)
req = {
'client_operation_id': '42',
'bucket': 'luci.chromium.try',
'tags': ['buildset:x', 't:0'],
'parameters_json': json.dumps({api_common.BUILDER_PARAMETER: 'linux'}),
}
resp = self.call_api('put', req).json_body
add_async.assert_called_once_with(
creation.BuildRequest(
schedule_build_request=rpc_pb2.ScheduleBuildRequest(
builder=dict(
project='chromium',
bucket='try',
builder='linux',
),
tags=tags,
request_id='42',
properties=dict(),
),
parameters={},
)
)
self.assertEqual(resp['build']['id'], '1')
self.assertIn('buildset:x', resp['build']['tags'])
self.assertIn('t:0', resp['build']['tags'])
def test_put_with_gerrit_change(self, add_async):
buildset = 'patch/gerrit/gerrit.example.com/1234/5'
buildset_tag = 'buildset:' + buildset
props = {'patch_project': 'repo'}
expected_sbr = rpc_pb2.ScheduleBuildRequest(
builder=dict(
project='chromium',
bucket='try',
builder='linux',
),
gerrit_changes=[
dict(
host='gerrit.example.com',
project='repo',
change=1234,
patchset=5,
)
],
tags=[dict(key='t', value='0')],
request_id='42',
properties=bbutil.dict_to_struct(props),
)
expected_request = creation.BuildRequest(
schedule_build_request=expected_sbr,
parameters={},
)
build = test_util.build(
id=1,
input=dict(
gerrit_changes=expected_sbr.gerrit_changes,
properties=expected_sbr.properties,
),
tags=expected_sbr.tags,
)
build.tags.append(buildset_tag)
build.tags.sort()
add_async.return_value = future(build)
req = {
'client_operation_id':
'42',
'bucket':
'luci.chromium.try',
'tags': [buildset_tag, 't:0'],
'parameters_json':
json.dumps({
api_common.BUILDER_PARAMETER: 'linux',
api_common.PROPERTIES_PARAMETER: props,
}),
}
resp = self.call_api('put', req).json_body
add_async.assert_called_once_with(expected_request)
self.assertEqual(resp['build']['id'], '1')
self.assertIn(buildset_tag, resp['build']['tags'])
self.assertIn('t:0', resp['build']['tags'])
def schedule_build_async(req, res, _ctx, mask):
"""Schedules one build."""
validation.validate_schedule_build_request(req)
bucket_id = config.format_bucket_id(req.builder.project,
req.builder.bucket)
if not (yield user.can_add_build_async(bucket_id)):
raise current_identity_cannot('schedule builds to bucket %s',
bucket_id)
build_req = creation.BuildRequest(schedule_build_request=req)
build = yield creation.add_async(build_req)
yield build_to_proto_async(build, res, mask)
def test_put_with_commit(self, add_async):
buildset = (
'commit/gitiles/gitiles.example.com/chromium/src/+/'
'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'
)
buildset_tag = 'buildset:' + buildset
gitiles_ref_tag = 'gitiles_ref:refs/heads/master'
gitiles_commit = common_pb2.GitilesCommit(
host='gitiles.example.com',
project='chromium/src',
ref='refs/heads/master',
id='aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
)
build = test_util.build(
id=1,
input=dict(gitiles_commit=gitiles_commit),
tags=[dict(key='t', value='0')],
)
build.tags.append(buildset_tag)
build.tags.append(gitiles_ref_tag)
build.tags.sort()
add_async.return_value = future(build)
req = {
'client_operation_id': '42',
'bucket': 'luci.chromium.try',
'tags': [buildset_tag, gitiles_ref_tag, 't:0'],
'parameters_json': json.dumps({api_common.BUILDER_PARAMETER: 'linux'}),
}
resp = self.call_api('put', req).json_body
add_async.assert_called_once_with(
creation.BuildRequest(
schedule_build_request=rpc_pb2.ScheduleBuildRequest(
builder=dict(
project='chromium',
bucket='try',
builder='linux',
),
gitiles_commit=gitiles_commit,
tags=[dict(key='t', value='0')],
request_id='42',
properties=dict(),
),
parameters={},
)
)
self.assertEqual(resp['build']['id'], '1')
self.assertIn(buildset_tag, resp['build']['tags'])
self.assertIn(gitiles_ref_tag, resp['build']['tags'])
self.assertIn('t:0', resp['build']['tags'])
def test_schedule(self, add_async):
add_async.return_value = future(
test_util.build(
id=54,
builder=dict(project='chromium', bucket='try',
builder='linux'),
), )
req = rpc_pb2.ScheduleBuildRequest(builder=dict(project='chromium',
bucket='try',
builder='linux'), )
res = self.call(self.api.ScheduleBuild, req)
self.assertEqual(res.id, 54)
add_async.assert_called_once_with(
creation.BuildRequest(schedule_build_request=req))
def retry(self, request):
"""Retries an existing build."""
lease_expiration_date = parse_datetime(request.lease_expiration_ts)
errors.validate_lease_expiration_date(lease_expiration_date)
build = model.Build.get_by_id(request.id)
if not build:
raise errors.BuildNotFoundError('Build %s not found' % request.id)
check_scheduling_permissions([build.bucket_id])
# Prepare v2 request.
sbr = rpc_pb2.ScheduleBuildRequest(
builder=build.proto.builder,
request_id=request.client_operation_id,
canary=common_pb2.YES if build.canary else common_pb2.NO,
properties=build.proto.input.properties,
gerrit_changes=build.proto.input.gerrit_changes[:],
)
build.tags_to_protos(sbr.tags)
if build.input_properties_bytes: # pragma: no branch
sbr.properties.ParseFromString(build.input_properties_bytes)
if build.proto.input.HasField('gitiles_commit'): # pragma: no branch
sbr.gitiles_commit.CopyFrom(build.proto.input.gitiles_commit)
# Read PubSub callback.
pubsub_callback_auth_token = None
if request.pubsub_callback: # pragma: no branch
pubsub_callback_auth_token = request.pubsub_callback.auth_token
pubsub_callback_to_notification_config(request.pubsub_callback,
sbr.notify)
with _wrap_validation_error():
validation.validate_notification_config(sbr.notify)
# Remove properties from parameters.
params = build.parameters.copy()
params.pop(model.PROPERTIES_PARAMETER, None)
# Create the build.
build_req = creation.BuildRequest(
schedule_build_request=sbr,
parameters=params,
lease_expiration_date=lease_expiration_date,
retry_of=request.id,
pubsub_callback_auth_token=pubsub_callback_auth_token,
)
build = creation.add_async(build_req).get_result()
return build_to_response_message(build, include_lease_key=True)
def test_put(self, add_async):
build = test_util.build(id=1, tags=[dict(key='a', value='b')])
add_async.return_value = future(build)
props = {'foo': 'bar'}
parameters_json = json.dumps({
api_common.BUILDER_PARAMETER: 'linux',
api_common.PROPERTIES_PARAMETER: props,
})
req = {
'client_operation_id': '42',
'bucket': 'luci.chromium.try',
'tags': ['a:b'],
'parameters_json': parameters_json,
'pubsub_callback': {
'topic': 'projects/foo/topic/bar',
'user_data': 'hello',
'auth_token': 'secret',
},
}
resp = self.call_api('put', req).json_body
add_async.assert_called_once_with(
creation.BuildRequest(
schedule_build_request=rpc_pb2.ScheduleBuildRequest(
builder=dict(
project='chromium',
bucket='try',
builder='linux',
),
tags=[dict(key='a', value='b')],
request_id='42',
notify=dict(
pubsub_topic='projects/foo/topic/bar',
user_data='hello',
),
properties=bbutil.dict_to_struct(props),
),
parameters={},
pubsub_callback_auth_token='secret',
)
)
self.assertEqual(resp['build']['id'], '1')
self.assertEqual(resp['build']['bucket'], req['bucket'])
self.assertIn('a:b', resp['build']['tags'])
def test_put_with_v2_gerrit_changes(self, add_async):
changes = [
common_pb2.GerritChange(
host='chromium.googlesource.com',
project='project',
change=1,
patchset=1,
),
common_pb2.GerritChange(
host='chromium.googlesource.com',
project='project',
change=2,
patchset=1,
),
]
expected_sbr = rpc_pb2.ScheduleBuildRequest(
builder=dict(
project='chromium',
bucket='try',
builder='linux',
),
properties=dict(),
gerrit_changes=changes,
)
expected_request = creation.BuildRequest(
schedule_build_request=expected_sbr,
parameters={},
)
add_async.return_value = future(test_util.build(id=1))
params = {
api_common.BUILDER_PARAMETER: 'linux',
'gerrit_changes': [json_format.MessageToDict(c) for c in changes],
}
req = {
'bucket': 'luci.chromium.try',
'parameters_json': json.dumps(params),
}
self.call_api('put', req)
add_async.assert_called_once_with(expected_request)
def schedule_build_multi(batch):
"""Schedules multiple builds.
Args:
batch: list of _ReqRes where
request is rpc_pb2.ScheduleBuildRequest and
response is rpc_pb2.BatchResponse.Response.
Response objects will be mutated.
"""
# Validate requests.
valid_items = []
for rr in batch:
try:
validation.validate_schedule_build_request(rr.request)
except validation.Error as ex:
rr.response.error.code = prpc.StatusCode.INVALID_ARGUMENT.value
rr.response.error.message = ex.message
continue
# Parse the field mask.
# Normally it is done by rpc_impl_async.
mask = None
if rr.request.HasField('fields'):
try:
mask = protoutil.Mask.from_field_mask(
rr.request.fields, build_pb2.Build.DESCRIPTOR)
except ValueError as ex:
rr.response.error.code = prpc.StatusCode.INVALID_ARGUMENT.value
rr.response.error.message = 'invalid fields: %s' % ex.message
continue
valid_items.append(_ScheduleItem(rr.request, rr.response, mask))
# Check permissions.
def get_bucket_id(req):
return config.format_bucket_id(req.builder.project, req.builder.bucket)
bucket_ids = {get_bucket_id(x.request) for x in valid_items}
can_add = dict(utils.async_apply(bucket_ids, user.can_add_build_async))
identity_str = auth.get_current_identity().to_bytes()
to_schedule = []
for x in valid_items:
bid = get_bucket_id(x.request)
if can_add[bid]:
to_schedule.append(x)
else:
x.response.error.code = prpc.StatusCode.PERMISSION_DENIED.value
x.response.error.message = (
'%s cannot schedule builds in bucket %s' % (identity_str, bid))
# Schedule builds.
if not to_schedule: # pragma: no cover
return
build_requests = [
creation.BuildRequest(schedule_build_request=x.request)
for x in to_schedule
]
results = creation.add_many_async(build_requests).get_result()
futs = []
for x, (build, ex) in zip(to_schedule, results):
res = x.response
err = res.error
if isinstance(ex, errors.Error):
err.code = ex.code.value
err.message = ex.message
elif isinstance(ex, auth.AuthorizationError):
err.code = prpc.StatusCode.PERMISSION_DENIED.value
err.message = ex.message
elif ex:
err.code = prpc.StatusCode.INTERNAL.value
err.message = ex.message
else:
futs.append(build_to_proto_async(build, res.schedule_build,
x.mask))
for f in futs:
f.get_result()
def test_put_batch(self, add_many_async):
bundle1 = test_util.build_bundle(id=1, tags=[dict(key='a', value='b')])
bundle2 = test_util.build_bundle(id=2)
bundle1.infra.put()
bundle2.infra.put()
config.put_bucket(
'chromium',
'a' * 40,
test_util.parse_bucket_cfg(
'''
name: "luci.chromium.try"
acls {
role: SCHEDULER
identity: "anonymous:anonymous"
}
'''
),
)
add_many_async.return_value = future([
(bundle1.build, None),
(bundle2.build, None),
(None, errors.InvalidInputError('bad')),
])
req = {
'builds': [
{
'bucket': 'luci.chromium.try',
'tags': ['a:b'],
'client_operation_id': '0',
},
{
'bucket': 'luci.chromium.try',
'client_operation_id': '1',
},
{
'bucket': 'luci.chromium.try',
'tags': ['bad tag'],
'client_operation_id': '2',
},
{
'bucket': 'luci.chromium.try',
'client_operation_id': '3',
},
],
}
resp = self.call_api('put_batch', req).json_body
add_many_async.assert_called_once_with([
creation.BuildRequest(
schedule_build_request=rpc_pb2.ScheduleBuildRequest(
builder=dict(project='chromium', bucket='try'),
tags=[dict(key='a', value='b')],
request_id='0',
properties=dict(),
),
parameters={},
),
creation.BuildRequest(
schedule_build_request=rpc_pb2.ScheduleBuildRequest(
builder=dict(project='chromium', bucket='try'),
request_id='1',
properties=dict(),
),
parameters={},
),
creation.BuildRequest(
schedule_build_request=rpc_pb2.ScheduleBuildRequest(
builder=dict(project='chromium', bucket='try'),
request_id='3',
properties=dict(),
),
parameters={},
),
])
res0 = resp['results'][0]
self.assertEqual(res0['client_operation_id'], '0')
self.assertEqual(res0['build']['id'], '1')
self.assertEqual(res0['build']['bucket'], 'luci.chromium.try')
res1 = resp['results'][1]
self.assertEqual(res1['client_operation_id'], '1')
self.assertEqual(res1['build']['id'], '2')
self.assertEqual(res1['build']['bucket'], 'luci.chromium.try')
res2 = resp['results'][2]
self.assertEqual(
res2, {
'client_operation_id': '2',
'error': {
'reason': 'INVALID_INPUT',
'message': u'Invalid tag "bad tag": does not contain ":"',
},
}
)
res3 = resp['results'][3]
self.assertEqual(
res3, {
'client_operation_id': '3',
'error': {
'reason': 'INVALID_INPUT',
'message': 'bad',
},
}
)
def test_retry(self, add_async):
props = bbutil.dict_to_struct({
'foo': 'bar',
'recipe': 'recipe',
})
orig_bundle = test_util.build_bundle(
id=1,
input=dict(
properties=props,
gitiles_commit=dict(
host='gitiles.example.com',
project='chromium/src',
id='a' * 40,
),
),
)
orig_build = orig_bundle.build
orig_build.parameters.pop('changes')
orig_build.tags = ['a:b']
ndb.put_multi([orig_build, orig_bundle.input_properties])
retried_build_bundle = test_util.build_bundle(
id=2,
input=dict(
properties=orig_build.proto.input.properties,
gitiles_commit=orig_build.proto.input.gitiles_commit,
),
)
retried_build_bundle.infra.put()
retried_build = retried_build_bundle.build
retried_build.retry_of = 1
add_async.return_value = future(retried_build)
req = {
'id': '1',
'client_operation_id': '42',
'pubsub_callback': {
'topic': 'projects/foo/topic/bar',
'user_data': 'hello',
'auth_token': 'secret',
},
}
resp = self.call_api('retry', req).json_body
add_async.assert_called_once_with(
creation.BuildRequest(
schedule_build_request=rpc_pb2.ScheduleBuildRequest(
builder=orig_build.proto.builder,
request_id='42',
notify=dict(
pubsub_topic='projects/foo/topic/bar',
user_data='hello',
),
properties=props,
tags=[dict(key='a', value='b')],
canary=common_pb2.NO,
gitiles_commit=orig_build.proto.input.gitiles_commit,
),
parameters={},
lease_expiration_date=None,
retry_of=1,
pubsub_callback_auth_token='secret',
)
)
self.assertEqual(resp['build']['id'], '2')
self.assertEqual(resp['build']['bucket'], 'luci.chromium.try')
self.assertEqual(resp['build']['retry_of'], '1')
def put_request_message_to_build_request(put_request):
"""Converts PutRequest to BuildRequest.
Raises errors.InvalidInputError if the put_request is invalid.
"""
lease_expiration_date = parse_datetime(put_request.lease_expiration_ts)
errors.validate_lease_expiration_date(lease_expiration_date)
# Read parameters.
parameters = parse_json_object(put_request.parameters_json,
'parameters_json')
parameters = parameters or {}
validate_known_build_parameters(parameters)
builder = parameters.get(model.BUILDER_PARAMETER) or ''
# Validate tags.
buildtags.validate_tags(put_request.tags, 'new', builder=builder)
# Read properties. Remove them from parameters.
props = parameters.pop(model.PROPERTIES_PARAMETER, None)
if props is not None and not isinstance(props, dict):
raise errors.InvalidInputError(
'"properties" parameter must be a JSON object or null')
props = props or {}
changes = parameters.get(_PARAM_CHANGES)
if changes: # pragma: no branch
# Buildbucket-Buildbot integration passes repo_url of the first change in
# build parameter "changes" as "repository" attribute of SourceStamp.
# https://chromium.googlesource.com/chromium/tools/build/+/2c6023d
# /scripts/master/buildbucket/changestore.py#140
# Buildbot passes repository of the build source stamp as "repository"
# build property. Recipes, in partiular bot_update recipe module, rely on
# "repository" property and it is an almost sane property to support in
# swarmbucket.
repo_url = changes[0].get('repo_url')
if repo_url: # pragma: no branch
props['repository'] = repo_url
# Buildbot-Buildbucket integration converts emails in changes to blamelist
# property.
emails = [c.get('author', {}).get('email') for c in changes]
props['blamelist'] = filter(None, emails)
# Create a v2 request.
sbr = rpc_pb2.ScheduleBuildRequest(
builder=build_pb2.BuilderID(builder=builder),
properties=bbutil.dict_to_struct(props),
request_id=put_request.client_operation_id,
experimental=bbutil.BOOLISH_TO_TRINARY[put_request.experimental],
canary=api_common.CANARY_PREFERENCE_TO_TRINARY.get(
put_request.canary_preference, common_pb2.UNSET),
)
sbr.builder.project, sbr.builder.bucket = config.parse_bucket_id(
put_request.bucket)
# Parse tags. Extract gitiles commit and gerrit changes.
tags, gitiles_commit, gerrit_changes = parse_v1_tags(put_request.tags)
sbr.tags.extend(tags)
if gitiles_commit:
sbr.gitiles_commit.CopyFrom(gitiles_commit)
# Gerrit changes explicitly passed via "gerrit_changes" parameter win.
gerrit_change_list = parameters.pop('gerrit_changes', None)
if gerrit_change_list is not None:
if not isinstance(gerrit_change_list, list): # pragma: no cover
raise errors.InvalidInputError('gerrit_changes must be a list')
try:
gerrit_changes = [
json_format.ParseDict(c, common_pb2.GerritChange())
for c in gerrit_change_list
]
except json_format.ParseError as ex: # pragma: no cover
raise errors.InvalidInputError('Invalid gerrit_changes: %s' % ex)
sbr.gerrit_changes.extend(gerrit_changes)
if (not gerrit_changes and
not sbr.builder.bucket.startswith('master.')): # pragma: no cover
changes = parameters.get('changes')
if isinstance(changes, list) and changes and not gitiles_commit:
legacy_revision = changes[0].get('revision')
if legacy_revision:
raise errors.InvalidInputError(
'legacy revision without gitiles buildset tag')
# Populate Gerrit project from patch_project property.
# V2 API users will have to provide this.
patch_project = props.get('patch_project')
if len(sbr.gerrit_changes) == 1 and isinstance(patch_project, basestring):
sbr.gerrit_changes[0].project = patch_project
# Read PubSub callback.
pubsub_callback_auth_token = None
if put_request.pubsub_callback:
pubsub_callback_auth_token = put_request.pubsub_callback.auth_token
pubsub_callback_to_notification_config(put_request.pubsub_callback,
sbr.notify)
# Validate the resulting v2 request before continuing.
with _wrap_validation_error():
validation.validate_schedule_build_request(sbr, legacy=True)
return creation.BuildRequest(
schedule_build_request=sbr,
parameters=parameters,
lease_expiration_date=lease_expiration_date,
pubsub_callback_auth_token=pubsub_callback_auth_token,
override_builder_cfg=_override_builder_cfg_func(parameters),
)
