def test_properties(self): props = {'foo': 'bar', 'qux': 1} prop_struct = bbutil.dict_to_struct(props) build = self.add(dict(properties=prop_struct)) in_props = model.BuildInputProperties.key_for(build.key).get() actual = in_props.parse() expected = bbutil.dict_to_struct(props) expected['recipe'] = 'recipe' self.assertEqual(actual, expected) infra = model.BuildInfra.key_for(build.key).get().parse() self.assertEqual(infra.buildbucket.requested_properties, prop_struct)
def test_properties(self): props = {'foo': 'bar', 'qux': 1} prop_struct = bbutil.dict_to_struct(props) build = self.add(dict(properties=prop_struct)) actual = struct_pb2.Struct() actual.ParseFromString(build.input_properties_bytes) expected = bbutil.dict_to_struct(props) expected['recipe'] = 'recipe' self.assertEqual(actual, expected) self.assertEqual(build.parse_infra().buildbucket.requested_properties, prop_struct) self.assertEqual(build.parameters.get(model.PROPERTIES_PARAMETER), props)
def test_trimming_exclude(self, get_async): get_async.return_value = future( test_util.build(input=dict( properties=bbutil.dict_to_struct({'a': 'b'}))), ) req = rpc_pb2.GetBuildRequest(id=1) res = self.call(self.api.GetBuild, req) self.assertFalse(res.input.HasField('properties'))
def test_out_props(self): props = bbutil.dict_to_struct({'a': 'b'}) bundle = test_util.build_bundle(output=dict(properties=props)) bundle.put() actual = self.to_proto(bundle.build, load_output_properties=True) self.assertEqual(actual.output.properties, props)
def test_trimming_include(self, get_async): get_async.return_value = future( test_util.build(input=dict( properties=bbutil.dict_to_struct({'a': 'b'}))), ) req = rpc_pb2.GetBuildRequest(id=1, fields=dict(paths=['input.properties'])) res = self.call(self.api.GetBuild, req) self.assertEqual(res.input.properties.items(), [('a', 'b')])
def test_put_with_gerrit_change(self, add_async): buildset = 'patch/gerrit/gerrit.example.com/1234/5' buildset_tag = 'buildset:' + buildset props = {'patch_project': 'repo'} expected_sbr = rpc_pb2.ScheduleBuildRequest( builder=dict( project='chromium', bucket='try', builder='linux', ), gerrit_changes=[ dict( host='gerrit.example.com', project='repo', change=1234, patchset=5, ) ], tags=[dict(key='t', value='0')], request_id='42', properties=bbutil.dict_to_struct(props), ) expected_request = creation.BuildRequest( schedule_build_request=expected_sbr, parameters={}, ) build = test_util.build( id=1, input=dict( gerrit_changes=expected_sbr.gerrit_changes, properties=expected_sbr.properties, ), tags=expected_sbr.tags, ) build.tags.append(buildset_tag) build.tags.sort() add_async.return_value = future(build) req = { 'client_operation_id': '42', 'bucket': 'luci.chromium.try', 'tags': [buildset_tag, 't:0'], 'parameters_json': json.dumps({ api_common.BUILDER_PARAMETER: 'linux', api_common.PROPERTIES_PARAMETER: props, }), } resp = self.call_api('put', req).json_body add_async.assert_called_once_with(expected_request) self.assertEqual(resp['build']['id'], '1') self.assertIn(buildset_tag, resp['build']['tags']) self.assertIn('t:0', resp['build']['tags'])
def test_out_props(self): props = bbutil.dict_to_struct({'a': 'b'}) build = test_util.build() model.BuildOutputProperties( key=model.BuildOutputProperties.key_for(build.key), properties=props.SerializeToString(), ).put() actual = self.to_proto(build, load_output_properties=True) self.assertEqual(actual.output.properties, props)
def _properties_to_dict(properties): """Implements properties_to_json.""" assert isinstance(properties, (dict, struct_pb2.Struct)), properties if isinstance(properties, dict): # pragma: no branch properties = bbutil.dict_to_struct(properties) # Note: this dict does not necessarily equal the original one. # In particular, an int may turn into a float. as_dict = json_format.MessageToDict(properties) for p in INTEGER_PROPERTIES: if isinstance(as_dict.get(p), float): as_dict[p] = int(as_dict[p]) return as_dict
def test_cancel_with_details(self, cancel): build = test_util.build(id=1) cancel.return_value = future(build) props = {'a': 'b'} model.BuildOutputProperties( key=model.BuildOutputProperties.key_for(build.key), properties=bbutil.dict_to_struct(props).SerializeToString(), ).put() result_details = {'properties': props} req = {'id': '1', 'result_details_json': json.dumps(result_details)} res = self.call_api('cancel', req).json_body cancel.assert_called_once_with(1, result_details=result_details) self.assertEqual(res['build']['result_details_json'], req['result_details_json'])
def _is_migrating_builder_prod_async( builder_cfg, build_proto): # pragma: no cover | TODO(nodir): delete this code """Returns True if the builder is prod according to the migration app. See also 'luci_migration_host' in the project config. If unknown, returns None. On failures, logs them and returns None. TODO(nodir): remove this function when Buildbot is turned down. """ ret = None master = None props_list = ( build_proto.input.properties, bbutil.dict_to_struct( flatten_swarmingcfg.read_properties(builder_cfg.recipe)), ) for prop_name in ('luci_migration_master_name', 'mastername'): for props in props_list: if prop_name in props: master = props[prop_name] break if master: # pragma: no branch break host = _clear_dash(builder_cfg.luci_migration_host) if master and host: try: url = 'https://%s/masters/%s/builders/%s/' % (host, master, builder_cfg.name) res = yield net.json_request_async(url, params={'format': 'json'}, scopes=net.EMAIL_SCOPE) ret = res.get('luci_is_prod') except net.NotFoundError: logging.warning('missing migration status for %r/%r', master, builder_cfg.name) except net.Error: logging.exception('failed to get migration status for %r/%r', master, builder_cfg.name) raise ndb.Return(ret)
def test_put(self, add_async): build = test_util.build(id=1, tags=[dict(key='a', value='b')]) add_async.return_value = future(build) props = {'foo': 'bar'} parameters_json = json.dumps({ api_common.BUILDER_PARAMETER: 'linux', api_common.PROPERTIES_PARAMETER: props, }) req = { 'client_operation_id': '42', 'bucket': 'luci.chromium.try', 'tags': ['a:b'], 'parameters_json': parameters_json, 'pubsub_callback': { 'topic': 'projects/foo/topic/bar', 'user_data': 'hello', 'auth_token': 'secret', }, } resp = self.call_api('put', req).json_body add_async.assert_called_once_with( creation.BuildRequest( schedule_build_request=rpc_pb2.ScheduleBuildRequest( builder=dict( project='chromium', bucket='try', builder='linux', ), tags=[dict(key='a', value='b')], request_id='42', notify=dict( pubsub_topic='projects/foo/topic/bar', user_data='hello', ), properties=bbutil.dict_to_struct(props), ), parameters={}, pubsub_callback_auth_token='secret', ) ) self.assertEqual(resp['build']['id'], '1') self.assertEqual(resp['build']['bucket'], req['bucket']) self.assertIn('a:b', resp['build']['tags'])
def test_succeed_with_result_details(self, succeed): build = test_util.build(id=1, tags=[dict(key='t', value='0')]) succeed.return_value = build props = {'p': '0'} model.BuildOutputProperties( key=model.BuildOutputProperties.key_for(build.key), properties=bbutil.dict_to_struct(props).SerializeToString(), ).put() result_details = {'properties': props} req = { 'id': '1', 'lease_key': 42, 'result_details_json': json.dumps(result_details), } res = self.call_api('succeed', req).json_body _, kwargs = service.succeed.call_args self.assertEqual(kwargs['result_details'], result_details) self.assertEqual(res['build']['result_details_json'], req['result_details_json']) self.assertIn('t:0', res['build']['tags'])
def test_succeed_with_result_details(self, succeed): props = {'p': '0'} bundle = test_util.build_bundle( id=1, tags=[dict(key='t', value='0')], output=dict(properties=bbutil.dict_to_struct(props)), ) succeed.return_value = bundle.build bundle.output_properties.put() result_details = {'properties': props} req = { 'id': '1', 'lease_key': 42, 'result_details_json': json.dumps(result_details), } res = self.call_api('succeed', req).json_body _, kwargs = service.succeed.call_args self.assertEqual(kwargs['result_details'], result_details) self.assertEqual( res['build']['result_details_json'], req['result_details_json'] ) self.assertIn('t:0', res['build']['tags'])
def test_cancel_batch(self, cancel): props = {'p': 0} build = test_util.build( id=1, output=dict(properties=bbutil.dict_to_struct(props))) cancel.side_effect = [ future(build), future_exception(errors.BuildIsCompletedError()), ] req = { 'build_ids': ['1', '2'], 'result_details_json': json.dumps(build.result_details), } res = self.call_api('cancel_batch', req).json_body res0 = res['results'][0] self.assertEqual(res0['build_id'], '1') self.assertEqual(res0['build']['id'], '1') cancel.assert_any_call(1, result_details=build.result_details) res1 = res['results'][1] self.assertEqual(res1['build_id'], '2') self.assertEqual(res1['error']['reason'], 'BUILD_IS_COMPLETED') cancel.assert_any_call(2, result_details=build.result_details)
def test_overall(self): self.patch('components.auth.get_current_identity', autospec=True, return_value=auth.Identity('user', '*****@*****.**')) build = self._test_build( id=1, number=1, scheduling_timeout=dict(seconds=3600), execution_timeout=dict(seconds=3600), builder=build_pb2.BuilderID(project='chromium', bucket='try', builder='linux'), exe=dict( cipd_package='infra/recipe_bundle', cipd_version='refs/heads/master', ), input=dict( properties=bbutil.dict_to_struct({ 'a': 'b', 'recipe': 'recipe', }), gerrit_changes=[ dict( host='chromium-review.googlesource.com', project='chromium/src', change=1234, patchset=5, ), ], ), infra=dict(swarming=dict( task_service_account='*****@*****.**', priority=108, task_dimensions=[ dict(key='cores', value='8'), dict(key='os', value='Ubuntu'), dict(key='pool', value='Chrome'), ], caches=[ dict(path='a', name='1'), ], ), ), ) actual = self.prepare_task_def(build) expected_swarming_props_def = { 'env': [{ 'key': 'BUILDBUCKET_EXPERIMENTAL', 'value': 'FALSE', }], 'execution_timeout_secs': '3600', 'extra_args': [ 'cook', '-recipe', 'recipe', '-properties', # Properties are tested by test_properties() above. swarming._setup_recipes(build)[1]['properties_json'], '-logdog-project', 'chromium', ], 'dimensions': [ { 'key': 'cores', 'value': '8' }, { 'key': 'os', 'value': 'Ubuntu' }, { 'key': 'pool', 'value': 'Chrome' }, ], 'caches': [{ 'path': 'cache/a', 'name': '1' }], 'cipd_input': { 'packages': [ { 'package_name': 'infra/tools/luci_runner', 'path': '.', 'version': 'luci-runner-version', }, { 'package_name': 'infra/tools/kitchen', 'path': '.', 'version': 'kitchen-version', }, { 'package_name': 'infra/recipe_bundle', 'path': 'kitchen-checkout', 'version': 'refs/heads/master', }, { 'package_name': 'infra/tools/git', 'path': swarming.USER_PACKAGE_DIR, 'version': 'git-version', }, { 'package_name': 'infra/cpython/python', 'path': '%s/python' % swarming.USER_PACKAGE_DIR, 'version': 'py-version', }, ], }, } expected = { 'name': 'bb-1-chromium-linux', 'priority': '108', 'tags': [ 'build_address:luci.chromium.try/linux/1', 'buildbucket_bucket:chromium/try', 'buildbucket_build_id:1', 'buildbucket_hostname:cr-buildbucket.appspot.com', 'buildbucket_template_canary:0', 'buildbucket_template_revision:template_rev', 'builder:linux', 'buildset:1', ('log_location:logdog://luci-logdog-dev.appspot.com/chromium/' 'buildbucket/cr-buildbucket.appspot.com/1/+/annotations'), 'luci_project:chromium', 'recipe_name:recipe', 'recipe_package:infra/recipe_bundle', ], 'pool_task_template': 'CANARY_NEVER', 'task_slices': [{ 'expiration_secs': '3600', 'properties': expected_swarming_props_def, 'wait_for_capacity': False, }], 'pubsub_topic': 'projects/testbed-test/topics/swarming', 'pubsub_userdata': json.dumps( { 'created_ts': 1448841600000000, 'swarming_hostname': 'swarming.example.com', 'build_id': 1L, }, sort_keys=True), 'service_account': '*****@*****.**', }
def test_properties(self): self.patch('components.auth.get_current_identity', autospec=True, return_value=auth.Identity('user', '*****@*****.**')) build = self._test_build( id=1, number=1, builder=build_pb2.BuilderID(project='chromium', bucket='try', builder='linux'), exe=dict( cipd_package='infra/recipe_bundle', cipd_version='refs/heads/master', ), input=dict( properties=bbutil.dict_to_struct({ 'a': 'b', 'recipe': 'recipe', }), gerrit_changes=[ dict( host='chromium-review.googlesource.com', project='chromium/src', change=1234, patchset=5, ), ], ), infra=dict(swarming=dict( task_service_account='*****@*****.**', priority=108, task_dimensions=[ dict(key='cores', value='8'), dict(key='os', value='Ubuntu'), dict(key='pool', value='Chrome'), ], ), ), ) _, extra_task_template_params = swarming._setup_recipes(build) actual = json.loads(extra_task_template_params['properties_json']) expected = { 'a': 'b', 'buildbucket': { 'hostname': 'cr-buildbucket.appspot.com', 'build': { 'project': 'chromium', 'bucket': 'luci.chromium.try', 'created_by': 'anonymous:anonymous', 'created_ts': 1448841600000000, 'id': '1', 'tags': [ 'build_address:luci.chromium.try/linux/1', 'builder:linux', 'buildset:1', ], }, }, 'buildername': 'linux', 'buildnumber': 1, 'recipe': 'recipe', 'repository': 'https://chromium.googlesource.com/chromium/src', '$recipe_engine/buildbucket': { 'hostname': 'cr-buildbucket.appspot.com', 'build': { 'id': '1', 'builder': { 'project': 'chromium', 'bucket': 'try', 'builder': 'linux', }, 'number': 1, 'tags': [{ 'value': '1', 'key': 'buildset' }], 'exe': { 'cipdPackage': 'infra/recipe_bundle', 'cipdVersion': 'refs/heads/master', }, 'input': { 'gerritChanges': [{ 'host': 'chromium-review.googlesource.com', 'project': 'chromium/src', 'change': '1234', 'patchset': '5', }], }, 'infra': { 'buildbucket': {}, 'swarming': { 'hostname': 'swarming.example.com', 'taskId': 'deadbeef', 'taskServiceAccount': '*****@*****.**', 'priority': 108, 'taskDimensions': [ { 'key': 'cores', 'value': '8' }, { 'key': 'os', 'value': 'Ubuntu' }, { 'key': 'pool', 'value': 'Chrome' }, ], }, 'logdog': { 'hostname': 'logdog.example.com', 'project': 'chromium', 'prefix': 'bb', }, }, 'createdBy': 'anonymous:anonymous', 'createTime': '2015-11-30T00:00:00Z', }, }, '$recipe_engine/runtime': { 'is_experimental': False, 'is_luci': True, }, } self.assertEqual(test_util.ununicode(actual), expected)
def test_retry(self, add_async): props = bbutil.dict_to_struct({ 'foo': 'bar', 'recipe': 'recipe', }) orig_bundle = test_util.build_bundle( id=1, input=dict( properties=props, gitiles_commit=dict( host='gitiles.example.com', project='chromium/src', id='a' * 40, ), ), ) orig_build = orig_bundle.build orig_build.parameters.pop('changes') orig_build.tags = ['a:b'] ndb.put_multi([orig_build, orig_bundle.input_properties]) retried_build_bundle = test_util.build_bundle( id=2, input=dict( properties=orig_build.proto.input.properties, gitiles_commit=orig_build.proto.input.gitiles_commit, ), ) retried_build_bundle.infra.put() retried_build = retried_build_bundle.build retried_build.retry_of = 1 add_async.return_value = future(retried_build) req = { 'id': '1', 'client_operation_id': '42', 'pubsub_callback': { 'topic': 'projects/foo/topic/bar', 'user_data': 'hello', 'auth_token': 'secret', }, } resp = self.call_api('retry', req).json_body add_async.assert_called_once_with( creation.BuildRequest( schedule_build_request=rpc_pb2.ScheduleBuildRequest( builder=orig_build.proto.builder, request_id='42', notify=dict( pubsub_topic='projects/foo/topic/bar', user_data='hello', ), properties=props, tags=[dict(key='a', value='b')], canary=common_pb2.NO, gitiles_commit=orig_build.proto.input.gitiles_commit, ), parameters={}, lease_expiration_date=None, retry_of=1, pubsub_callback_auth_token='secret', ) ) self.assertEqual(resp['build']['id'], '2') self.assertEqual(resp['build']['bucket'], 'luci.chromium.try') self.assertEqual(resp['build']['retry_of'], '1')
def put_request_message_to_build_request(put_request): """Converts PutRequest to BuildRequest. Raises errors.InvalidInputError if the put_request is invalid. """ lease_expiration_date = parse_datetime(put_request.lease_expiration_ts) errors.validate_lease_expiration_date(lease_expiration_date) # Read parameters. parameters = parse_json_object(put_request.parameters_json, 'parameters_json') parameters = parameters or {} validate_known_build_parameters(parameters) builder = parameters.get(model.BUILDER_PARAMETER) or '' # Validate tags. buildtags.validate_tags(put_request.tags, 'new', builder=builder) # Read properties. Remove them from parameters. props = parameters.pop(model.PROPERTIES_PARAMETER, None) if props is not None and not isinstance(props, dict): raise errors.InvalidInputError( '"properties" parameter must be a JSON object or null') props = props or {} changes = parameters.get(_PARAM_CHANGES) if changes: # pragma: no branch # Buildbucket-Buildbot integration passes repo_url of the first change in # build parameter "changes" as "repository" attribute of SourceStamp. # https://chromium.googlesource.com/chromium/tools/build/+/2c6023d # /scripts/master/buildbucket/changestore.py#140 # Buildbot passes repository of the build source stamp as "repository" # build property. Recipes, in partiular bot_update recipe module, rely on # "repository" property and it is an almost sane property to support in # swarmbucket. repo_url = changes[0].get('repo_url') if repo_url: # pragma: no branch props['repository'] = repo_url # Buildbot-Buildbucket integration converts emails in changes to blamelist # property. emails = [c.get('author', {}).get('email') for c in changes] props['blamelist'] = filter(None, emails) # Create a v2 request. sbr = rpc_pb2.ScheduleBuildRequest( builder=build_pb2.BuilderID(builder=builder), properties=bbutil.dict_to_struct(props), request_id=put_request.client_operation_id, experimental=bbutil.BOOLISH_TO_TRINARY[put_request.experimental], canary=api_common.CANARY_PREFERENCE_TO_TRINARY.get( put_request.canary_preference, common_pb2.UNSET), ) sbr.builder.project, sbr.builder.bucket = config.parse_bucket_id( put_request.bucket) # Parse tags. Extract gitiles commit and gerrit changes. tags, gitiles_commit, gerrit_changes = parse_v1_tags(put_request.tags) sbr.tags.extend(tags) if gitiles_commit: sbr.gitiles_commit.CopyFrom(gitiles_commit) # Gerrit changes explicitly passed via "gerrit_changes" parameter win. gerrit_change_list = parameters.pop('gerrit_changes', None) if gerrit_change_list is not None: if not isinstance(gerrit_change_list, list): # pragma: no cover raise errors.InvalidInputError('gerrit_changes must be a list') try: gerrit_changes = [ json_format.ParseDict(c, common_pb2.GerritChange()) for c in gerrit_change_list ] except json_format.ParseError as ex: # pragma: no cover raise errors.InvalidInputError('Invalid gerrit_changes: %s' % ex) sbr.gerrit_changes.extend(gerrit_changes) if (not gerrit_changes and not sbr.builder.bucket.startswith('master.')): # pragma: no cover changes = parameters.get('changes') if isinstance(changes, list) and changes and not gitiles_commit: legacy_revision = changes[0].get('revision') if legacy_revision: raise errors.InvalidInputError( 'legacy revision without gitiles buildset tag') # Populate Gerrit project from patch_project property. # V2 API users will have to provide this. patch_project = props.get('patch_project') if len(sbr.gerrit_changes) == 1 and isinstance(patch_project, basestring): sbr.gerrit_changes[0].project = patch_project # Read PubSub callback. pubsub_callback_auth_token = None if put_request.pubsub_callback: pubsub_callback_auth_token = put_request.pubsub_callback.auth_token pubsub_callback_to_notification_config(put_request.pubsub_callback, sbr.notify) # Validate the resulting v2 request before continuing. with _wrap_validation_error(): validation.validate_schedule_build_request(sbr, legacy=True) return creation.BuildRequest( schedule_build_request=sbr, parameters=parameters, lease_expiration_date=lease_expiration_date, pubsub_callback_auth_token=pubsub_callback_auth_token, override_builder_cfg=_override_builder_cfg_func(parameters), )
def test_properties(self): self.patch( 'components.auth.get_current_identity', autospec=True, return_value=auth.Identity('user', '*****@*****.**') ) now_ts = timestamp_pb2.Timestamp() now_ts.FromDatetime(utils.utcnow()) build = model.Build( tags=['t:1'], created_by=auth.Anonymous, proto=build_pb2.Build( id=1, builder=dict(project='chromium', bucket='try', builder='linux-rel'), number=1, status=common_pb2.SCHEDULED, created_by='anonymous:anonymous', create_time=now_ts, update_time=now_ts, input=dict( properties=bbutil.dict_to_struct({ 'recipe': 'recipe', 'a': 'b', }), gerrit_changes=[ dict( host='chromium-review.googlesource.com', project='chromium/src', change=1234, patchset=5, ) ], ), output=dict(), infra=dict( buildbucket=dict( requested_properties=bbutil.dict_to_struct({'a': 'b'}), ), recipe=dict(), ), ), ) actual = bbutil.struct_to_dict(swarming._compute_legacy_properties(build)) expected = { 'a': 'b', 'buildbucket': { 'hostname': 'cr-buildbucket.appspot.com', 'build': { 'project': 'chromium', 'bucket': 'luci.chromium.try', 'created_by': 'anonymous:anonymous', 'created_ts': 1448841600000000, 'id': '1', 'tags': ['t:1'], }, }, 'buildername': 'linux-rel', 'buildnumber': 1, 'recipe': 'recipe', 'repository': 'https://chromium.googlesource.com/chromium/src', '$recipe_engine/buildbucket': { 'hostname': 'cr-buildbucket.appspot.com', 'build': { 'id': '1', 'builder': { 'project': 'chromium', 'bucket': 'try', 'builder': 'linux-rel', }, 'number': 1, 'tags': [{'key': 't', 'value': '1'}], 'input': { 'gerritChanges': [{ 'host': 'chromium-review.googlesource.com', 'project': 'chromium/src', 'change': '1234', 'patchset': '5', }], }, 'infra': {'buildbucket': {},}, 'createdBy': 'anonymous:anonymous', 'createTime': '2015-11-30T00:00:00Z', }, }, '$recipe_engine/runtime': { 'is_experimental': False, 'is_luci': True, }, } self.assertEqual(test_util.ununicode(actual), expected)
def test_overall(self): self.patch( 'components.auth.get_current_identity', autospec=True, return_value=auth.Identity('user', '*****@*****.**') ) build = self._test_build( id=1, number=1, scheduling_timeout=dict(seconds=3600), execution_timeout=dict(seconds=3600), builder=build_pb2.BuilderID( project='chromium', bucket='try', builder='linux' ), exe=dict( cipd_package='infra/recipe_bundle', cipd_version='refs/heads/master', ), input=dict( properties=bbutil.dict_to_struct({ 'a': 'b', 'recipe': 'recipe', }), gerrit_changes=[ dict( host='chromium-review.googlesource.com', project='chromium/src', change=1234, patchset=5, ), ], ), infra=dict( logdog=dict( hostname='logs.example.com', project='chromium', prefix='bb', ), swarming=dict( task_service_account='*****@*****.**', priority=108, task_dimensions=[ dict(key='cores', value='8'), dict(key='os', value='Ubuntu'), dict(key='pool', value='Chrome'), ], caches=[ dict( path='vpython', name='vpython', env_var='VPYTHON_VIRTUALENV_ROOT' ), ], ), ), ) actual = self.compute_task_def(build) expected_args = launcher_pb2.BBAgentArgs( executable_path=swarming._KITCHEN_CHECKOUT + '/luciexe', cache_dir=swarming._CACHE_DIR, known_public_gerrit_hosts=['chromium-review.googlesource.com'], build=build.proto, ) expected_swarming_props_def = { 'env': [{ 'key': 'BUILDBUCKET_EXPERIMENTAL', 'value': 'FALSE', }], 'env_prefixes': [ { 'key': 'PATH', 'value': [ 'cipd_bin_packages', 'cipd_bin_packages/bin', 'cipd_bin_packages/python', 'cipd_bin_packages/python/bin', ], }, { 'key': 'VPYTHON_VIRTUALENV_ROOT', 'value': ['cache/vpython'], }, ], 'execution_timeout_secs': '3600', 'command': [ 'bbagent${EXECUTABLE_SUFFIX}', swarming._cli_encode_proto(expected_args), ], 'dimensions': [ {'key': 'cores', 'value': '8'}, {'key': 'os', 'value': 'Ubuntu'}, {'key': 'pool', 'value': 'Chrome'}, ], 'caches': [{'path': 'cache/vpython', 'name': 'vpython'},], 'cipd_input': { 'packages': [ { 'package_name': 'infra/tools/bbagent', 'path': '.', 'version': 'luci-runner-version', }, { 'package_name': 'infra/tools/kitchen', 'path': '.', 'version': 'kitchen-version', }, { 'package_name': 'infra/recipe_bundle', 'path': 'kitchen-checkout', 'version': 'refs/heads/master', }, { 'package_name': 'infra/tools/git', 'path': swarming.USER_PACKAGE_DIR, 'version': 'git-version', }, { 'package_name': 'infra/cpython/python', 'path': '%s/python' % swarming.USER_PACKAGE_DIR, 'version': 'py-version', }, ], }, } expected = { 'name': 'bb-1-chromium/try/linux-1', 'priority': '108', 'tags': [ 'build_address:luci.chromium.try/linux/1', 'buildbucket_bucket:chromium/try', 'buildbucket_build_id:1', 'buildbucket_hostname:cr-buildbucket.appspot.com', 'buildbucket_template_canary:0', 'builder:linux', 'buildset:1', 'luci_project:chromium', ], 'task_slices': [{ 'expiration_secs': '3600', 'properties': expected_swarming_props_def, 'wait_for_capacity': False, }], 'pubsub_topic': 'projects/testbed-test/topics/swarming', 'pubsub_userdata': json.dumps({ 'created_ts': 1448841600000000, 'swarming_hostname': 'swarming.example.com', 'build_id': 1L, }, sort_keys=True), 'service_account': '*****@*****.**', }
def test_build_to_dict(self): params_json = json.dumps( { api_common.BUILDER_PARAMETER: 'linux', api_common.PROPERTIES_PARAMETER: { 'build-defined-property': 1.0, }, }, sort_keys=True, ) tags = [ 'build_address:luci.chromium.try/linux/1', 'builder:linux', 'buildset:1', 'swarming_hostname:swarming.example.com', ('swarming_tag:log_location:' 'logdog://logdog.example.com/chromium/bb/+/annotations'), 'swarming_tag:luci_project:chromium', 'swarming_tag:os:Ubuntu', 'swarming_tag:recipe_name:recipe', 'swarming_tag:recipe_package:infra/recipe_bundle', 'swarming_task_id:deadbeef', ] result_details = { 'properties': { 'a': 'b' }, 'swarming': { 'bot_dimensions': { 'dim1': ['v1', 'v2'], 'os': ['Ubuntu'], }, }, 'error': { 'message': 'bad' }, 'ui': { 'info': 'bad' }, } expected = { 'project': 'chromium', 'bucket': 'luci.chromium.try', 'created_by': 'anonymous:anonymous', 'created_ts': '1483228800000000', 'experimental': False, 'completed_ts': '1483228800000000', 'id': '8991715593768927232', 'parameters_json': params_json, 'result_details_json': json.dumps(result_details, sort_keys=True), 'status': 'COMPLETED', 'result': 'FAILURE', 'failure_reason': 'INFRA_FAILURE', 'status_changed_ts': '1483228800000000', 'tags': tags, 'utcnow_ts': '1483228800000000', 'updated_ts': '1483228800000000', 'canary_preference': 'PROD', 'canary': False, 'service_account': '*****@*****.**', 'url': 'https://ci.example.com/8991715593768927232', } bundle = test_util.build_bundle( status=common_pb2.INFRA_FAILURE, summary_markdown='bad', input=dict(properties=bbutil.dict_to_struct({ 'recipe': 'recipe', 'build-defined-property': 1, 'builder-defined-property': 2, }), ), output=dict(properties=bbutil.dict_to_struct({'a': 'b'}), ), infra=dict( swarming=dict(bot_dimensions=[ dict(key='dim1', value='v1'), dict(key='dim1', value='v2'), dict(key='os', value='Ubuntu'), ], ), buildbucket=dict(requested_properties=bbutil.dict_to_struct({ 'build-defined-property': 1, }), ), )) self.assertEqual(expected, test_util.ununicode(api_common.build_to_dict(bundle)))
def test_in_props(self): props = bbutil.dict_to_struct({'a': 'b'}) build = test_util.build(input=dict(properties=props)) actual = self.to_proto(build, load_input_properties=True) self.assertEqual(actual.input.properties, props)
def test_pubsub_callback(self): build = test_util.build(id=1) build.pubsub_callback = model.PubSubCallback( topic='projects/example/topics/buildbucket', user_data='hello', auth_token='secret', ) out_props = model.BuildOutputProperties( key=model.BuildOutputProperties.key_for(build.key), ) out_props.serialize(bbutil.dict_to_struct({'a': 'b'})) @ndb.transactional def txn(): build.put() out_props.put() notifications.enqueue_notifications_async(build).get_result() txn() build = build.key.get() global_task_payload = { 'id': 1, 'mode': 'global', } callback_task_payload = { 'id': 1, 'mode': 'callback', } tq.enqueue_async.assert_called_with( 'backend-default', [ { 'url': '/internal/task/buildbucket/notify/1', 'payload': global_task_payload, 'retry_options': { 'task_age_limit': model.BUILD_TIMEOUT.total_seconds(), }, }, { 'url': '/internal/task/buildbucket/notify/1', 'payload': callback_task_payload, 'retry_options': { 'task_age_limit': model.BUILD_TIMEOUT.total_seconds(), }, }, ] ) self.app.post_json( '/internal/task/buildbucket/notify/1', params=global_task_payload, headers={'X-AppEngine-QueueName': 'backend-default'} ) pubsub.publish.assert_called_with( 'projects/testbed-test/topics/builds', json.dumps({ 'build': api_common.build_to_dict(build, out_props), 'hostname': 'buildbucket.example.com', }, sort_keys=True), {'build_id': '1'}, ) self.app.post_json( '/internal/task/buildbucket/notify/1', params=callback_task_payload, headers={'X-AppEngine-QueueName': 'backend-default'} ) pubsub.publish.assert_called_with( 'projects/example/topics/buildbucket', json.dumps({ 'build': api_common.build_to_dict(build, out_props), 'hostname': 'buildbucket.example.com', 'user_data': 'hello', }, sort_keys=True), { 'build_id': '1', 'auth_token': 'secret', }, )