def _put_output_properties_async(build_key, legacy_result_details):
prop_dict = (legacy_result_details or {}).get('properties')
if isinstance(prop_dict, dict):
props = struct_pb2.Struct()
props.update(prop_dict)
yield model.BuildOutputProperties(
key=model.BuildOutputProperties.key_for(build_key),
properties=props.SerializeToString(),
).put_async()
def test_out_props(self):
props = bbutil.dict_to_struct({'a': 'b'})
build = test_util.build()
model.BuildOutputProperties(
key=model.BuildOutputProperties.key_for(build.key),
properties=props.SerializeToString(),
).put()
actual = self.to_proto(build, load_output_properties=True)
self.assertEqual(actual.output.properties, props)
def txn_async():
build = yield get_async()
orig_status = build.status
futures = []
if 'build.output.properties' in update_paths:
futures.append(
model.BuildOutputProperties(
key=model.BuildOutputProperties.key_for(build.key),
properties=out_prop_bytes,
).put_async())
if model_build_proto_mask: # pragma: no branch
# Merge the rest into build.proto using model_build_proto_mask.
model_build_proto_mask.merge(req.build, build.proto)
# If we are updating build status, update some other dependent fields
# and schedule notifications.
status_changed = orig_status != build.proto.status
if status_changed:
if build.proto.status == common_pb2.STARTED:
if not build.proto.HasField('start_time'): # pragma: no branch
build.proto.start_time.FromDatetime(now)
futures.append(events.on_build_starting_async(build))
else:
assert model.is_terminal_status(
build.proto.status), build.proto.status
build.clear_lease()
if not build.proto.HasField('end_time'): # pragma: no branch
build.proto.end_time.FromDatetime(now)
futures.append(events.on_build_completing_async(build))
if 'build.steps' in update_paths:
# TODO(crbug.com/936892): reject requests with a terminal build status
# and incomplete steps, when
# https://chromium-review.googlesource.com/c/infra/infra/+/1553291
# is deployed.
futures.append(build_steps.put_async())
elif build.is_ended:
futures.append(
model.BuildSteps.cancel_incomplete_steps_async(
req.build.id, build.proto.end_time))
futures.append(build.put_async())
yield futures
raise ndb.Return(build, status_changed)
def test_cancel_with_details(self, cancel):
build = test_util.build(id=1)
cancel.return_value = future(build)
props = {'a': 'b'}
model.BuildOutputProperties(
key=model.BuildOutputProperties.key_for(build.key),
properties=bbutil.dict_to_struct(props).SerializeToString(),
).put()
result_details = {'properties': props}
req = {'id': '1', 'result_details_json': json.dumps(result_details)}
res = self.call_api('cancel', req).json_body
cancel.assert_called_once_with(1, result_details=result_details)
self.assertEqual(res['build']['result_details_json'],
req['result_details_json'])
def test_succeed_with_result_details(self, succeed):
build = test_util.build(id=1, tags=[dict(key='t', value='0')])
succeed.return_value = build
props = {'p': '0'}
model.BuildOutputProperties(
key=model.BuildOutputProperties.key_for(build.key),
properties=bbutil.dict_to_struct(props).SerializeToString(),
).put()
result_details = {'properties': props}
req = {
'id': '1',
'lease_key': 42,
'result_details_json': json.dumps(result_details),
}
res = self.call_api('succeed', req).json_body
_, kwargs = service.succeed.call_args
self.assertEqual(kwargs['result_details'], result_details)
self.assertEqual(res['build']['result_details_json'],
req['result_details_json'])
self.assertIn('t:0', res['build']['tags'])
def test_build_to_dict(self):
props_json = json.dumps(
{
model.BUILDER_PARAMETER: 'linux',
model.PROPERTIES_PARAMETER: {}
},
sort_keys=True,
)
tags = [
'build_address:luci.chromium.try/linux/1',
'builder:linux',
'buildset:1',
'swarming_hostname:swarming.example.com',
('swarming_tag:log_location:'
'logdog://logdog.example.com/chromium/bb/+/annotations'),
'swarming_tag:luci_project:chromium',
'swarming_tag:os:Ubuntu',
'swarming_tag:recipe_name:recipe',
'swarming_tag:recipe_package:infra/recipe_bundle',
'swarming_task_id:deadbeef',
]
result_details = {
'properties': {
'a': 'b'
},
'swarming': {
'bot_dimensions': {
'dim1': ['v1', 'v2'],
'os': ['Ubuntu'],
},
},
'error': {
'message': 'bad'
},
'ui': {
'info': 'bad'
},
}
expected = {
'project': 'chromium',
'bucket': 'luci.chromium.try',
'created_by': 'anonymous:anonymous',
'created_ts': '1483228800000000',
'experimental': False,
'completed_ts': '1483228800000000',
'id': '8991715593768927232',
'parameters_json': props_json,
'result_details_json': json.dumps(result_details, sort_keys=True),
'status': 'COMPLETED',
'result': 'FAILURE',
'failure_reason': 'INFRA_FAILURE',
'status_changed_ts': '1483228800000000',
'tags': tags,
'utcnow_ts': '1483228800000000',
'updated_ts': '1483228800000000',
'canary_preference': 'PROD',
'canary': False,
'service_account': '*****@*****.**',
'url': 'https://ci.example.com/8991715593768927232',
}
out_props = model.BuildOutputProperties()
out_props.serialize(bbutil.dict_to_struct({'a': 'b'}))
build = test_util.build(
status=common_pb2.INFRA_FAILURE,
summary_markdown='bad',
input=dict(properties=bbutil.dict_to_struct({
'recipe': 'recipe',
})),
infra=dict(swarming=dict(bot_dimensions=[
dict(key='dim1', value='v1'),
dict(key='dim1', value='v2'),
dict(key='os', value='Ubuntu'),
], ), ),
)
self.assertEqual(
expected,
test_util.ununicode(api_common.build_to_dict(build, out_props)))
def test_pubsub_callback(self):
build = test_util.build(id=1)
build.pubsub_callback = model.PubSubCallback(
topic='projects/example/topics/buildbucket',
user_data='hello',
auth_token='secret',
)
out_props = model.BuildOutputProperties(
key=model.BuildOutputProperties.key_for(build.key),
)
out_props.serialize(bbutil.dict_to_struct({'a': 'b'}))
@ndb.transactional
def txn():
build.put()
out_props.put()
notifications.enqueue_notifications_async(build).get_result()
txn()
build = build.key.get()
global_task_payload = {
'id': 1,
'mode': 'global',
}
callback_task_payload = {
'id': 1,
'mode': 'callback',
}
tq.enqueue_async.assert_called_with(
'backend-default', [
{
'url': '/internal/task/buildbucket/notify/1',
'payload': global_task_payload,
'retry_options': {
'task_age_limit': model.BUILD_TIMEOUT.total_seconds(),
},
},
{
'url': '/internal/task/buildbucket/notify/1',
'payload': callback_task_payload,
'retry_options': {
'task_age_limit': model.BUILD_TIMEOUT.total_seconds(),
},
},
]
)
self.app.post_json(
'/internal/task/buildbucket/notify/1',
params=global_task_payload,
headers={'X-AppEngine-QueueName': 'backend-default'}
)
pubsub.publish.assert_called_with(
'projects/testbed-test/topics/builds',
json.dumps({
'build': api_common.build_to_dict(build, out_props),
'hostname': 'buildbucket.example.com',
},
sort_keys=True),
{'build_id': '1'},
)
self.app.post_json(
'/internal/task/buildbucket/notify/1',
params=callback_task_payload,
headers={'X-AppEngine-QueueName': 'backend-default'}
)
pubsub.publish.assert_called_with(
'projects/example/topics/buildbucket',
json.dumps({
'build': api_common.build_to_dict(build, out_props),
'hostname': 'buildbucket.example.com',
'user_data': 'hello',
},
sort_keys=True),
{
'build_id': '1',
'auth_token': 'secret',
},
)
def build_bundle(for_creation=False, **build_proto_fields): # pragma: no cover
"""Creates a model.BuildBundle from proto fields, with reasonable defaults.
If for_creation is True, returned Build.proto.{infra, input.properties} will
be set.
"""
now = utils.utcnow()
# Compute defaults.
proto = copy.deepcopy(BUILD_DEFAULTS)
if not proto.HasField('create_time'):
proto.create_time.FromDatetime(now)
proto.MergeFrom(build_pb2.Build(**build_proto_fields))
proto.id = proto.id or model.create_build_ids(
proto.create_time.ToDatetime(), 1, randomness=False)[0]
with_start_time = (common_pb2.STARTED, common_pb2.SUCCESS)
if not proto.HasField('start_time') and proto.status in with_start_time:
proto.start_time.FromDatetime(now)
completed = proto.status not in (common_pb2.SCHEDULED, common_pb2.STARTED)
if not proto.HasField('end_time') and completed:
proto.end_time.FromDatetime(now)
proto.update_time.FromDatetime(now)
if (proto.input.properties
and not proto.infra.buildbucket.HasField('requested_properties')):
proto.infra.buildbucket.requested_properties.CopyFrom(
proto.input.properties)
tags = {buildtags.unparse(t.key, t.value) for t in proto.tags}
tags.add('builder:%s' % proto.builder.builder)
if proto.number:
tags.add(buildtags.build_address_tag(proto.builder, proto.number))
proto.ClearField('tags')
b = model.Build(
id=proto.id,
proto=proto,
created_by=auth.Identity.from_bytes(proto.created_by),
create_time=proto.create_time.ToDatetime(),
status_changed_time=now,
tags=sorted(tags),
parameters={},
url='https://ci.example.com/%d' % proto.id,
is_luci=True,
swarming_task_key='swarming_task_key',
)
b.update_v1_status_fields()
if proto.input.HasField('gitiles_commit'):
b.parameters['changes'] = [{
'author': {
'email': '*****@*****.**'
},
'repo_url':
'https://chromium.googlesource.com/chromium/src',
}]
ret = model.BuildBundle(
b,
infra=model.BuildInfra(key=model.BuildInfra.key_for(b.key),
infra=proto.infra.SerializeToString()),
input_properties=model.BuildInputProperties(
key=model.BuildInputProperties.key_for(b.key),
properties=proto.input.properties.SerializeToString(),
),
output_properties=model.BuildOutputProperties(
key=model.BuildOutputProperties.key_for(b.key),
properties=proto.output.properties.SerializeToString(),
),
steps=model.BuildSteps(
key=model.BuildSteps.key_for(b.key),
step_container_bytes=(build_pb2.Build(
steps=proto.steps).SerializeToString()),
),
)
if not for_creation:
proto.ClearField('infra')
proto.input.ClearField('properties')
proto.output.ClearField('properties')
proto.ClearField('steps')
return ret