def test_properties(self):
props = {'foo': 'bar', 'qux': 1}
prop_struct = bbutil.dict_to_struct(props)
build = self.add(dict(properties=prop_struct))
in_props = model.BuildInputProperties.key_for(build.key).get()
actual = in_props.parse()
expected = bbutil.dict_to_struct(props)
expected['recipe'] = 'recipe'
self.assertEqual(actual, expected)
infra = model.BuildInfra.key_for(build.key).get().parse()
self.assertEqual(infra.buildbucket.requested_properties, prop_struct)
def test_properties(self):
props = {'foo': 'bar', 'qux': 1}
prop_struct = bbutil.dict_to_struct(props)
build = self.add(dict(properties=prop_struct))
actual = struct_pb2.Struct()
actual.ParseFromString(build.input_properties_bytes)
expected = bbutil.dict_to_struct(props)
expected['recipe'] = 'recipe'
self.assertEqual(actual, expected)
self.assertEqual(build.parse_infra().buildbucket.requested_properties,
prop_struct)
self.assertEqual(build.parameters.get(model.PROPERTIES_PARAMETER),
props)
def test_trimming_exclude(self, get_async):
get_async.return_value = future(
test_util.build(input=dict(
properties=bbutil.dict_to_struct({'a': 'b'}))), )
req = rpc_pb2.GetBuildRequest(id=1)
res = self.call(self.api.GetBuild, req)
self.assertFalse(res.input.HasField('properties'))
def test_out_props(self):
props = bbutil.dict_to_struct({'a': 'b'})
bundle = test_util.build_bundle(output=dict(properties=props))
bundle.put()
actual = self.to_proto(bundle.build, load_output_properties=True)
self.assertEqual(actual.output.properties, props)
def test_trimming_include(self, get_async):
get_async.return_value = future(
test_util.build(input=dict(
properties=bbutil.dict_to_struct({'a': 'b'}))), )
req = rpc_pb2.GetBuildRequest(id=1,
fields=dict(paths=['input.properties']))
res = self.call(self.api.GetBuild, req)
self.assertEqual(res.input.properties.items(), [('a', 'b')])
def test_put_with_gerrit_change(self, add_async):
buildset = 'patch/gerrit/gerrit.example.com/1234/5'
buildset_tag = 'buildset:' + buildset
props = {'patch_project': 'repo'}
expected_sbr = rpc_pb2.ScheduleBuildRequest(
builder=dict(
project='chromium',
bucket='try',
builder='linux',
),
gerrit_changes=[
dict(
host='gerrit.example.com',
project='repo',
change=1234,
patchset=5,
)
],
tags=[dict(key='t', value='0')],
request_id='42',
properties=bbutil.dict_to_struct(props),
)
expected_request = creation.BuildRequest(
schedule_build_request=expected_sbr,
parameters={},
)
build = test_util.build(
id=1,
input=dict(
gerrit_changes=expected_sbr.gerrit_changes,
properties=expected_sbr.properties,
),
tags=expected_sbr.tags,
)
build.tags.append(buildset_tag)
build.tags.sort()
add_async.return_value = future(build)
req = {
'client_operation_id':
'42',
'bucket':
'luci.chromium.try',
'tags': [buildset_tag, 't:0'],
'parameters_json':
json.dumps({
api_common.BUILDER_PARAMETER: 'linux',
api_common.PROPERTIES_PARAMETER: props,
}),
}
resp = self.call_api('put', req).json_body
add_async.assert_called_once_with(expected_request)
self.assertEqual(resp['build']['id'], '1')
self.assertIn(buildset_tag, resp['build']['tags'])
self.assertIn('t:0', resp['build']['tags'])
def test_out_props(self):
props = bbutil.dict_to_struct({'a': 'b'})
build = test_util.build()
model.BuildOutputProperties(
key=model.BuildOutputProperties.key_for(build.key),
properties=props.SerializeToString(),
).put()
actual = self.to_proto(build, load_output_properties=True)
self.assertEqual(actual.output.properties, props)
def _properties_to_dict(properties):
"""Implements properties_to_json."""
assert isinstance(properties, (dict, struct_pb2.Struct)), properties
if isinstance(properties, dict): # pragma: no branch
properties = bbutil.dict_to_struct(properties)
# Note: this dict does not necessarily equal the original one.
# In particular, an int may turn into a float.
as_dict = json_format.MessageToDict(properties)
for p in INTEGER_PROPERTIES:
if isinstance(as_dict.get(p), float):
as_dict[p] = int(as_dict[p])
return as_dict
def test_cancel_with_details(self, cancel):
build = test_util.build(id=1)
cancel.return_value = future(build)
props = {'a': 'b'}
model.BuildOutputProperties(
key=model.BuildOutputProperties.key_for(build.key),
properties=bbutil.dict_to_struct(props).SerializeToString(),
).put()
result_details = {'properties': props}
req = {'id': '1', 'result_details_json': json.dumps(result_details)}
res = self.call_api('cancel', req).json_body
cancel.assert_called_once_with(1, result_details=result_details)
self.assertEqual(res['build']['result_details_json'],
req['result_details_json'])
def _is_migrating_builder_prod_async(
builder_cfg,
build_proto): # pragma: no cover | TODO(nodir): delete this code
"""Returns True if the builder is prod according to the migration app.
See also 'luci_migration_host' in the project config.
If unknown, returns None.
On failures, logs them and returns None.
TODO(nodir): remove this function when Buildbot is turned down.
"""
ret = None
master = None
props_list = (
build_proto.input.properties,
bbutil.dict_to_struct(
flatten_swarmingcfg.read_properties(builder_cfg.recipe)),
)
for prop_name in ('luci_migration_master_name', 'mastername'):
for props in props_list:
if prop_name in props:
master = props[prop_name]
break
if master: # pragma: no branch
break
host = _clear_dash(builder_cfg.luci_migration_host)
if master and host:
try:
url = 'https://%s/masters/%s/builders/%s/' % (host, master,
builder_cfg.name)
res = yield net.json_request_async(url,
params={'format': 'json'},
scopes=net.EMAIL_SCOPE)
ret = res.get('luci_is_prod')
except net.NotFoundError:
logging.warning('missing migration status for %r/%r', master,
builder_cfg.name)
except net.Error:
logging.exception('failed to get migration status for %r/%r',
master, builder_cfg.name)
raise ndb.Return(ret)
def test_put(self, add_async):
build = test_util.build(id=1, tags=[dict(key='a', value='b')])
add_async.return_value = future(build)
props = {'foo': 'bar'}
parameters_json = json.dumps({
api_common.BUILDER_PARAMETER: 'linux',
api_common.PROPERTIES_PARAMETER: props,
})
req = {
'client_operation_id': '42',
'bucket': 'luci.chromium.try',
'tags': ['a:b'],
'parameters_json': parameters_json,
'pubsub_callback': {
'topic': 'projects/foo/topic/bar',
'user_data': 'hello',
'auth_token': 'secret',
},
}
resp = self.call_api('put', req).json_body
add_async.assert_called_once_with(
creation.BuildRequest(
schedule_build_request=rpc_pb2.ScheduleBuildRequest(
builder=dict(
project='chromium',
bucket='try',
builder='linux',
),
tags=[dict(key='a', value='b')],
request_id='42',
notify=dict(
pubsub_topic='projects/foo/topic/bar',
user_data='hello',
),
properties=bbutil.dict_to_struct(props),
),
parameters={},
pubsub_callback_auth_token='secret',
)
)
self.assertEqual(resp['build']['id'], '1')
self.assertEqual(resp['build']['bucket'], req['bucket'])
self.assertIn('a:b', resp['build']['tags'])
def test_succeed_with_result_details(self, succeed):
build = test_util.build(id=1, tags=[dict(key='t', value='0')])
succeed.return_value = build
props = {'p': '0'}
model.BuildOutputProperties(
key=model.BuildOutputProperties.key_for(build.key),
properties=bbutil.dict_to_struct(props).SerializeToString(),
).put()
result_details = {'properties': props}
req = {
'id': '1',
'lease_key': 42,
'result_details_json': json.dumps(result_details),
}
res = self.call_api('succeed', req).json_body
_, kwargs = service.succeed.call_args
self.assertEqual(kwargs['result_details'], result_details)
self.assertEqual(res['build']['result_details_json'],
req['result_details_json'])
self.assertIn('t:0', res['build']['tags'])
def test_succeed_with_result_details(self, succeed):
props = {'p': '0'}
bundle = test_util.build_bundle(
id=1,
tags=[dict(key='t', value='0')],
output=dict(properties=bbutil.dict_to_struct(props)),
)
succeed.return_value = bundle.build
bundle.output_properties.put()
result_details = {'properties': props}
req = {
'id': '1',
'lease_key': 42,
'result_details_json': json.dumps(result_details),
}
res = self.call_api('succeed', req).json_body
_, kwargs = service.succeed.call_args
self.assertEqual(kwargs['result_details'], result_details)
self.assertEqual(
res['build']['result_details_json'], req['result_details_json']
)
self.assertIn('t:0', res['build']['tags'])
def test_cancel_batch(self, cancel):
props = {'p': 0}
build = test_util.build(
id=1, output=dict(properties=bbutil.dict_to_struct(props)))
cancel.side_effect = [
future(build),
future_exception(errors.BuildIsCompletedError()),
]
req = {
'build_ids': ['1', '2'],
'result_details_json': json.dumps(build.result_details),
}
res = self.call_api('cancel_batch', req).json_body
res0 = res['results'][0]
self.assertEqual(res0['build_id'], '1')
self.assertEqual(res0['build']['id'], '1')
cancel.assert_any_call(1, result_details=build.result_details)
res1 = res['results'][1]
self.assertEqual(res1['build_id'], '2')
self.assertEqual(res1['error']['reason'], 'BUILD_IS_COMPLETED')
cancel.assert_any_call(2, result_details=build.result_details)
def test_overall(self):
self.patch('components.auth.get_current_identity',
autospec=True,
return_value=auth.Identity('user', '*****@*****.**'))
build = self._test_build(
id=1,
number=1,
scheduling_timeout=dict(seconds=3600),
execution_timeout=dict(seconds=3600),
builder=build_pb2.BuilderID(project='chromium',
bucket='try',
builder='linux'),
exe=dict(
cipd_package='infra/recipe_bundle',
cipd_version='refs/heads/master',
),
input=dict(
properties=bbutil.dict_to_struct({
'a': 'b',
'recipe': 'recipe',
}),
gerrit_changes=[
dict(
host='chromium-review.googlesource.com',
project='chromium/src',
change=1234,
patchset=5,
),
],
),
infra=dict(swarming=dict(
task_service_account='*****@*****.**',
priority=108,
task_dimensions=[
dict(key='cores', value='8'),
dict(key='os', value='Ubuntu'),
dict(key='pool', value='Chrome'),
],
caches=[
dict(path='a', name='1'),
],
), ),
)
actual = self.prepare_task_def(build)
expected_swarming_props_def = {
'env': [{
'key': 'BUILDBUCKET_EXPERIMENTAL',
'value': 'FALSE',
}],
'execution_timeout_secs':
'3600',
'extra_args': [
'cook',
'-recipe',
'recipe',
'-properties',
# Properties are tested by test_properties() above.
swarming._setup_recipes(build)[1]['properties_json'],
'-logdog-project',
'chromium',
],
'dimensions': [
{
'key': 'cores',
'value': '8'
},
{
'key': 'os',
'value': 'Ubuntu'
},
{
'key': 'pool',
'value': 'Chrome'
},
],
'caches': [{
'path': 'cache/a',
'name': '1'
}],
'cipd_input': {
'packages': [
{
'package_name': 'infra/tools/luci_runner',
'path': '.',
'version': 'luci-runner-version',
},
{
'package_name': 'infra/tools/kitchen',
'path': '.',
'version': 'kitchen-version',
},
{
'package_name': 'infra/recipe_bundle',
'path': 'kitchen-checkout',
'version': 'refs/heads/master',
},
{
'package_name': 'infra/tools/git',
'path': swarming.USER_PACKAGE_DIR,
'version': 'git-version',
},
{
'package_name': 'infra/cpython/python',
'path': '%s/python' % swarming.USER_PACKAGE_DIR,
'version': 'py-version',
},
],
},
}
expected = {
'name':
'bb-1-chromium-linux',
'priority':
'108',
'tags': [
'build_address:luci.chromium.try/linux/1',
'buildbucket_bucket:chromium/try',
'buildbucket_build_id:1',
'buildbucket_hostname:cr-buildbucket.appspot.com',
'buildbucket_template_canary:0',
'buildbucket_template_revision:template_rev',
'builder:linux',
'buildset:1',
('log_location:logdog://luci-logdog-dev.appspot.com/chromium/'
'buildbucket/cr-buildbucket.appspot.com/1/+/annotations'),
'luci_project:chromium',
'recipe_name:recipe',
'recipe_package:infra/recipe_bundle',
],
'pool_task_template':
'CANARY_NEVER',
'task_slices': [{
'expiration_secs': '3600',
'properties': expected_swarming_props_def,
'wait_for_capacity': False,
}],
'pubsub_topic':
'projects/testbed-test/topics/swarming',
'pubsub_userdata':
json.dumps(
{
'created_ts': 1448841600000000,
'swarming_hostname': 'swarming.example.com',
'build_id': 1L,
},
sort_keys=True),
'service_account':
'*****@*****.**',
}
def test_properties(self):
self.patch('components.auth.get_current_identity',
autospec=True,
return_value=auth.Identity('user', '*****@*****.**'))
build = self._test_build(
id=1,
number=1,
builder=build_pb2.BuilderID(project='chromium',
bucket='try',
builder='linux'),
exe=dict(
cipd_package='infra/recipe_bundle',
cipd_version='refs/heads/master',
),
input=dict(
properties=bbutil.dict_to_struct({
'a': 'b',
'recipe': 'recipe',
}),
gerrit_changes=[
dict(
host='chromium-review.googlesource.com',
project='chromium/src',
change=1234,
patchset=5,
),
],
),
infra=dict(swarming=dict(
task_service_account='*****@*****.**',
priority=108,
task_dimensions=[
dict(key='cores', value='8'),
dict(key='os', value='Ubuntu'),
dict(key='pool', value='Chrome'),
],
), ),
)
_, extra_task_template_params = swarming._setup_recipes(build)
actual = json.loads(extra_task_template_params['properties_json'])
expected = {
'a': 'b',
'buildbucket': {
'hostname': 'cr-buildbucket.appspot.com',
'build': {
'project':
'chromium',
'bucket':
'luci.chromium.try',
'created_by':
'anonymous:anonymous',
'created_ts':
1448841600000000,
'id':
'1',
'tags': [
'build_address:luci.chromium.try/linux/1',
'builder:linux',
'buildset:1',
],
},
},
'buildername': 'linux',
'buildnumber': 1,
'recipe': 'recipe',
'repository': 'https://chromium.googlesource.com/chromium/src',
'$recipe_engine/buildbucket': {
'hostname': 'cr-buildbucket.appspot.com',
'build': {
'id': '1',
'builder': {
'project': 'chromium',
'bucket': 'try',
'builder': 'linux',
},
'number': 1,
'tags': [{
'value': '1',
'key': 'buildset'
}],
'exe': {
'cipdPackage': 'infra/recipe_bundle',
'cipdVersion': 'refs/heads/master',
},
'input': {
'gerritChanges': [{
'host': 'chromium-review.googlesource.com',
'project': 'chromium/src',
'change': '1234',
'patchset': '5',
}],
},
'infra': {
'buildbucket': {},
'swarming': {
'hostname':
'swarming.example.com',
'taskId':
'deadbeef',
'taskServiceAccount':
'*****@*****.**',
'priority':
108,
'taskDimensions': [
{
'key': 'cores',
'value': '8'
},
{
'key': 'os',
'value': 'Ubuntu'
},
{
'key': 'pool',
'value': 'Chrome'
},
],
},
'logdog': {
'hostname': 'logdog.example.com',
'project': 'chromium',
'prefix': 'bb',
},
},
'createdBy': 'anonymous:anonymous',
'createTime': '2015-11-30T00:00:00Z',
},
},
'$recipe_engine/runtime': {
'is_experimental': False,
'is_luci': True,
},
}
self.assertEqual(test_util.ununicode(actual), expected)
def test_retry(self, add_async):
props = bbutil.dict_to_struct({
'foo': 'bar',
'recipe': 'recipe',
})
orig_bundle = test_util.build_bundle(
id=1,
input=dict(
properties=props,
gitiles_commit=dict(
host='gitiles.example.com',
project='chromium/src',
id='a' * 40,
),
),
)
orig_build = orig_bundle.build
orig_build.parameters.pop('changes')
orig_build.tags = ['a:b']
ndb.put_multi([orig_build, orig_bundle.input_properties])
retried_build_bundle = test_util.build_bundle(
id=2,
input=dict(
properties=orig_build.proto.input.properties,
gitiles_commit=orig_build.proto.input.gitiles_commit,
),
)
retried_build_bundle.infra.put()
retried_build = retried_build_bundle.build
retried_build.retry_of = 1
add_async.return_value = future(retried_build)
req = {
'id': '1',
'client_operation_id': '42',
'pubsub_callback': {
'topic': 'projects/foo/topic/bar',
'user_data': 'hello',
'auth_token': 'secret',
},
}
resp = self.call_api('retry', req).json_body
add_async.assert_called_once_with(
creation.BuildRequest(
schedule_build_request=rpc_pb2.ScheduleBuildRequest(
builder=orig_build.proto.builder,
request_id='42',
notify=dict(
pubsub_topic='projects/foo/topic/bar',
user_data='hello',
),
properties=props,
tags=[dict(key='a', value='b')],
canary=common_pb2.NO,
gitiles_commit=orig_build.proto.input.gitiles_commit,
),
parameters={},
lease_expiration_date=None,
retry_of=1,
pubsub_callback_auth_token='secret',
)
)
self.assertEqual(resp['build']['id'], '2')
self.assertEqual(resp['build']['bucket'], 'luci.chromium.try')
self.assertEqual(resp['build']['retry_of'], '1')
def put_request_message_to_build_request(put_request):
"""Converts PutRequest to BuildRequest.
Raises errors.InvalidInputError if the put_request is invalid.
"""
lease_expiration_date = parse_datetime(put_request.lease_expiration_ts)
errors.validate_lease_expiration_date(lease_expiration_date)
# Read parameters.
parameters = parse_json_object(put_request.parameters_json,
'parameters_json')
parameters = parameters or {}
validate_known_build_parameters(parameters)
builder = parameters.get(model.BUILDER_PARAMETER) or ''
# Validate tags.
buildtags.validate_tags(put_request.tags, 'new', builder=builder)
# Read properties. Remove them from parameters.
props = parameters.pop(model.PROPERTIES_PARAMETER, None)
if props is not None and not isinstance(props, dict):
raise errors.InvalidInputError(
'"properties" parameter must be a JSON object or null')
props = props or {}
changes = parameters.get(_PARAM_CHANGES)
if changes: # pragma: no branch
# Buildbucket-Buildbot integration passes repo_url of the first change in
# build parameter "changes" as "repository" attribute of SourceStamp.
# https://chromium.googlesource.com/chromium/tools/build/+/2c6023d
# /scripts/master/buildbucket/changestore.py#140
# Buildbot passes repository of the build source stamp as "repository"
# build property. Recipes, in partiular bot_update recipe module, rely on
# "repository" property and it is an almost sane property to support in
# swarmbucket.
repo_url = changes[0].get('repo_url')
if repo_url: # pragma: no branch
props['repository'] = repo_url
# Buildbot-Buildbucket integration converts emails in changes to blamelist
# property.
emails = [c.get('author', {}).get('email') for c in changes]
props['blamelist'] = filter(None, emails)
# Create a v2 request.
sbr = rpc_pb2.ScheduleBuildRequest(
builder=build_pb2.BuilderID(builder=builder),
properties=bbutil.dict_to_struct(props),
request_id=put_request.client_operation_id,
experimental=bbutil.BOOLISH_TO_TRINARY[put_request.experimental],
canary=api_common.CANARY_PREFERENCE_TO_TRINARY.get(
put_request.canary_preference, common_pb2.UNSET),
)
sbr.builder.project, sbr.builder.bucket = config.parse_bucket_id(
put_request.bucket)
# Parse tags. Extract gitiles commit and gerrit changes.
tags, gitiles_commit, gerrit_changes = parse_v1_tags(put_request.tags)
sbr.tags.extend(tags)
if gitiles_commit:
sbr.gitiles_commit.CopyFrom(gitiles_commit)
# Gerrit changes explicitly passed via "gerrit_changes" parameter win.
gerrit_change_list = parameters.pop('gerrit_changes', None)
if gerrit_change_list is not None:
if not isinstance(gerrit_change_list, list): # pragma: no cover
raise errors.InvalidInputError('gerrit_changes must be a list')
try:
gerrit_changes = [
json_format.ParseDict(c, common_pb2.GerritChange())
for c in gerrit_change_list
]
except json_format.ParseError as ex: # pragma: no cover
raise errors.InvalidInputError('Invalid gerrit_changes: %s' % ex)
sbr.gerrit_changes.extend(gerrit_changes)
if (not gerrit_changes and
not sbr.builder.bucket.startswith('master.')): # pragma: no cover
changes = parameters.get('changes')
if isinstance(changes, list) and changes and not gitiles_commit:
legacy_revision = changes[0].get('revision')
if legacy_revision:
raise errors.InvalidInputError(
'legacy revision without gitiles buildset tag')
# Populate Gerrit project from patch_project property.
# V2 API users will have to provide this.
patch_project = props.get('patch_project')
if len(sbr.gerrit_changes) == 1 and isinstance(patch_project, basestring):
sbr.gerrit_changes[0].project = patch_project
# Read PubSub callback.
pubsub_callback_auth_token = None
if put_request.pubsub_callback:
pubsub_callback_auth_token = put_request.pubsub_callback.auth_token
pubsub_callback_to_notification_config(put_request.pubsub_callback,
sbr.notify)
# Validate the resulting v2 request before continuing.
with _wrap_validation_error():
validation.validate_schedule_build_request(sbr, legacy=True)
return creation.BuildRequest(
schedule_build_request=sbr,
parameters=parameters,
lease_expiration_date=lease_expiration_date,
pubsub_callback_auth_token=pubsub_callback_auth_token,
override_builder_cfg=_override_builder_cfg_func(parameters),
)
def test_properties(self):
self.patch(
'components.auth.get_current_identity',
autospec=True,
return_value=auth.Identity('user', '*****@*****.**')
)
now_ts = timestamp_pb2.Timestamp()
now_ts.FromDatetime(utils.utcnow())
build = model.Build(
tags=['t:1'],
created_by=auth.Anonymous,
proto=build_pb2.Build(
id=1,
builder=dict(project='chromium', bucket='try', builder='linux-rel'),
number=1,
status=common_pb2.SCHEDULED,
created_by='anonymous:anonymous',
create_time=now_ts,
update_time=now_ts,
input=dict(
properties=bbutil.dict_to_struct({
'recipe': 'recipe',
'a': 'b',
}),
gerrit_changes=[
dict(
host='chromium-review.googlesource.com',
project='chromium/src',
change=1234,
patchset=5,
)
],
),
output=dict(),
infra=dict(
buildbucket=dict(
requested_properties=bbutil.dict_to_struct({'a': 'b'}),
),
recipe=dict(),
),
),
)
actual = bbutil.struct_to_dict(swarming._compute_legacy_properties(build))
expected = {
'a': 'b',
'buildbucket': {
'hostname': 'cr-buildbucket.appspot.com',
'build': {
'project': 'chromium',
'bucket': 'luci.chromium.try',
'created_by': 'anonymous:anonymous',
'created_ts': 1448841600000000,
'id': '1',
'tags': ['t:1'],
},
},
'buildername': 'linux-rel',
'buildnumber': 1,
'recipe': 'recipe',
'repository': 'https://chromium.googlesource.com/chromium/src',
'$recipe_engine/buildbucket': {
'hostname': 'cr-buildbucket.appspot.com',
'build': {
'id': '1',
'builder': {
'project': 'chromium',
'bucket': 'try',
'builder': 'linux-rel',
},
'number': 1,
'tags': [{'key': 't', 'value': '1'}],
'input': {
'gerritChanges': [{
'host': 'chromium-review.googlesource.com',
'project': 'chromium/src',
'change': '1234',
'patchset': '5',
}],
},
'infra': {'buildbucket': {},},
'createdBy': 'anonymous:anonymous',
'createTime': '2015-11-30T00:00:00Z',
},
},
'$recipe_engine/runtime': {
'is_experimental': False,
'is_luci': True,
},
}
self.assertEqual(test_util.ununicode(actual), expected)
def test_overall(self):
self.patch(
'components.auth.get_current_identity',
autospec=True,
return_value=auth.Identity('user', '*****@*****.**')
)
build = self._test_build(
id=1,
number=1,
scheduling_timeout=dict(seconds=3600),
execution_timeout=dict(seconds=3600),
builder=build_pb2.BuilderID(
project='chromium', bucket='try', builder='linux'
),
exe=dict(
cipd_package='infra/recipe_bundle',
cipd_version='refs/heads/master',
),
input=dict(
properties=bbutil.dict_to_struct({
'a': 'b',
'recipe': 'recipe',
}),
gerrit_changes=[
dict(
host='chromium-review.googlesource.com',
project='chromium/src',
change=1234,
patchset=5,
),
],
),
infra=dict(
logdog=dict(
hostname='logs.example.com',
project='chromium',
prefix='bb',
),
swarming=dict(
task_service_account='*****@*****.**',
priority=108,
task_dimensions=[
dict(key='cores', value='8'),
dict(key='os', value='Ubuntu'),
dict(key='pool', value='Chrome'),
],
caches=[
dict(
path='vpython',
name='vpython',
env_var='VPYTHON_VIRTUALENV_ROOT'
),
],
),
),
)
actual = self.compute_task_def(build)
expected_args = launcher_pb2.BBAgentArgs(
executable_path=swarming._KITCHEN_CHECKOUT + '/luciexe',
cache_dir=swarming._CACHE_DIR,
known_public_gerrit_hosts=['chromium-review.googlesource.com'],
build=build.proto,
)
expected_swarming_props_def = {
'env': [{
'key': 'BUILDBUCKET_EXPERIMENTAL',
'value': 'FALSE',
}],
'env_prefixes': [
{
'key':
'PATH',
'value': [
'cipd_bin_packages',
'cipd_bin_packages/bin',
'cipd_bin_packages/python',
'cipd_bin_packages/python/bin',
],
},
{
'key': 'VPYTHON_VIRTUALENV_ROOT',
'value': ['cache/vpython'],
},
],
'execution_timeout_secs':
'3600',
'command': [
'bbagent${EXECUTABLE_SUFFIX}',
swarming._cli_encode_proto(expected_args),
],
'dimensions': [
{'key': 'cores', 'value': '8'},
{'key': 'os', 'value': 'Ubuntu'},
{'key': 'pool', 'value': 'Chrome'},
],
'caches': [{'path': 'cache/vpython', 'name': 'vpython'},],
'cipd_input': {
'packages': [
{
'package_name': 'infra/tools/bbagent',
'path': '.',
'version': 'luci-runner-version',
},
{
'package_name': 'infra/tools/kitchen',
'path': '.',
'version': 'kitchen-version',
},
{
'package_name': 'infra/recipe_bundle',
'path': 'kitchen-checkout',
'version': 'refs/heads/master',
},
{
'package_name': 'infra/tools/git',
'path': swarming.USER_PACKAGE_DIR,
'version': 'git-version',
},
{
'package_name': 'infra/cpython/python',
'path': '%s/python' % swarming.USER_PACKAGE_DIR,
'version': 'py-version',
},
],
},
}
expected = {
'name':
'bb-1-chromium/try/linux-1',
'priority':
'108',
'tags': [
'build_address:luci.chromium.try/linux/1',
'buildbucket_bucket:chromium/try',
'buildbucket_build_id:1',
'buildbucket_hostname:cr-buildbucket.appspot.com',
'buildbucket_template_canary:0',
'builder:linux',
'buildset:1',
'luci_project:chromium',
],
'task_slices': [{
'expiration_secs': '3600',
'properties': expected_swarming_props_def,
'wait_for_capacity': False,
}],
'pubsub_topic':
'projects/testbed-test/topics/swarming',
'pubsub_userdata':
json.dumps({
'created_ts': 1448841600000000,
'swarming_hostname': 'swarming.example.com',
'build_id': 1L,
},
sort_keys=True),
'service_account':
'*****@*****.**',
}
def test_build_to_dict(self):
params_json = json.dumps(
{
api_common.BUILDER_PARAMETER: 'linux',
api_common.PROPERTIES_PARAMETER: {
'build-defined-property': 1.0,
},
},
sort_keys=True,
)
tags = [
'build_address:luci.chromium.try/linux/1',
'builder:linux',
'buildset:1',
'swarming_hostname:swarming.example.com',
('swarming_tag:log_location:'
'logdog://logdog.example.com/chromium/bb/+/annotations'),
'swarming_tag:luci_project:chromium',
'swarming_tag:os:Ubuntu',
'swarming_tag:recipe_name:recipe',
'swarming_tag:recipe_package:infra/recipe_bundle',
'swarming_task_id:deadbeef',
]
result_details = {
'properties': {
'a': 'b'
},
'swarming': {
'bot_dimensions': {
'dim1': ['v1', 'v2'],
'os': ['Ubuntu'],
},
},
'error': {
'message': 'bad'
},
'ui': {
'info': 'bad'
},
}
expected = {
'project': 'chromium',
'bucket': 'luci.chromium.try',
'created_by': 'anonymous:anonymous',
'created_ts': '1483228800000000',
'experimental': False,
'completed_ts': '1483228800000000',
'id': '8991715593768927232',
'parameters_json': params_json,
'result_details_json': json.dumps(result_details, sort_keys=True),
'status': 'COMPLETED',
'result': 'FAILURE',
'failure_reason': 'INFRA_FAILURE',
'status_changed_ts': '1483228800000000',
'tags': tags,
'utcnow_ts': '1483228800000000',
'updated_ts': '1483228800000000',
'canary_preference': 'PROD',
'canary': False,
'service_account': '*****@*****.**',
'url': 'https://ci.example.com/8991715593768927232',
}
bundle = test_util.build_bundle(
status=common_pb2.INFRA_FAILURE,
summary_markdown='bad',
input=dict(properties=bbutil.dict_to_struct({
'recipe':
'recipe',
'build-defined-property':
1,
'builder-defined-property':
2,
}), ),
output=dict(properties=bbutil.dict_to_struct({'a': 'b'}), ),
infra=dict(
swarming=dict(bot_dimensions=[
dict(key='dim1', value='v1'),
dict(key='dim1', value='v2'),
dict(key='os', value='Ubuntu'),
], ),
buildbucket=dict(requested_properties=bbutil.dict_to_struct({
'build-defined-property':
1,
}), ),
))
self.assertEqual(expected,
test_util.ununicode(api_common.build_to_dict(bundle)))
def test_in_props(self):
props = bbutil.dict_to_struct({'a': 'b'})
build = test_util.build(input=dict(properties=props))
actual = self.to_proto(build, load_input_properties=True)
self.assertEqual(actual.input.properties, props)
def test_pubsub_callback(self):
build = test_util.build(id=1)
build.pubsub_callback = model.PubSubCallback(
topic='projects/example/topics/buildbucket',
user_data='hello',
auth_token='secret',
)
out_props = model.BuildOutputProperties(
key=model.BuildOutputProperties.key_for(build.key),
)
out_props.serialize(bbutil.dict_to_struct({'a': 'b'}))
@ndb.transactional
def txn():
build.put()
out_props.put()
notifications.enqueue_notifications_async(build).get_result()
txn()
build = build.key.get()
global_task_payload = {
'id': 1,
'mode': 'global',
}
callback_task_payload = {
'id': 1,
'mode': 'callback',
}
tq.enqueue_async.assert_called_with(
'backend-default', [
{
'url': '/internal/task/buildbucket/notify/1',
'payload': global_task_payload,
'retry_options': {
'task_age_limit': model.BUILD_TIMEOUT.total_seconds(),
},
},
{
'url': '/internal/task/buildbucket/notify/1',
'payload': callback_task_payload,
'retry_options': {
'task_age_limit': model.BUILD_TIMEOUT.total_seconds(),
},
},
]
)
self.app.post_json(
'/internal/task/buildbucket/notify/1',
params=global_task_payload,
headers={'X-AppEngine-QueueName': 'backend-default'}
)
pubsub.publish.assert_called_with(
'projects/testbed-test/topics/builds',
json.dumps({
'build': api_common.build_to_dict(build, out_props),
'hostname': 'buildbucket.example.com',
},
sort_keys=True),
{'build_id': '1'},
)
self.app.post_json(
'/internal/task/buildbucket/notify/1',
params=callback_task_payload,
headers={'X-AppEngine-QueueName': 'backend-default'}
)
pubsub.publish.assert_called_with(
'projects/example/topics/buildbucket',
json.dumps({
'build': api_common.build_to_dict(build, out_props),
'hostname': 'buildbucket.example.com',
'user_data': 'hello',
},
sort_keys=True),
{
'build_id': '1',
'auth_token': 'secret',
},
)
