def test_valid(self): msg = rpc_pb2.ScheduleBuildRequest( builder=dict(project='chromium', bucket='try', builder='linux-rel'), gitiles_commit=dict( host='gerrit.example.com', project='project', id='a' * 40, ref='refs/heads/master', position=1, ), gerrit_changes=[ dict(host='gerrit.example.com', change=1, patchset=1), dict(host='gerrit.example.com', change=2, patchset=2), ], tags=[ dict(key='a', value='a1'), dict(key='a', value='a2'), dict(key='b', value='b1'), ], dimensions=[ dict(key='d1', value='dv1'), dict(key='d2', value='dv2'), ], priority=100, notify=dict( pubsub_topic='projects/project_id/topics/topic_id', user_data='blob', ), ) msg.properties.update({'a': 1, '$recipe_engine/runtime': {'b': 1}}) self.assert_valid(msg)
def test_put_with_generic_buildset(self, add_async): tags = [ dict(key='buildset', value='x'), dict(key='t', value='0'), ] build = test_util.build(id=1, tags=tags) add_async.return_value = future(build) req = { 'client_operation_id': '42', 'bucket': 'luci.chromium.try', 'tags': ['buildset:x', 't:0'], 'parameters_json': json.dumps({api_common.BUILDER_PARAMETER: 'linux'}), } resp = self.call_api('put', req).json_body add_async.assert_called_once_with( creation.BuildRequest( schedule_build_request=rpc_pb2.ScheduleBuildRequest( builder=dict( project='chromium', bucket='try', builder='linux', ), tags=tags, request_id='42', properties=dict(), ), parameters={}, ) ) self.assertEqual(resp['build']['id'], '1') self.assertIn('buildset:x', resp['build']['tags']) self.assertIn('t:0', resp['build']['tags'])
def build_request(self, schedule_build_request_fields=None, **kwargs): schedule_build_request_fields = schedule_build_request_fields or {} sbr = rpc_pb2.ScheduleBuildRequest(**schedule_build_request_fields) sbr.builder.project = sbr.builder.project or 'chromium' sbr.builder.bucket = sbr.builder.bucket or 'try' sbr.builder.builder = sbr.builder.builder or 'linux' return creation.BuildRequest(schedule_build_request=sbr, **kwargs)
def test_forbidden(self): user.can_async.return_value = future(False) req = rpc_pb2.ScheduleBuildRequest(builder=dict(project='chromium', bucket='try', builder='linux'), ) self.call(self.api.ScheduleBuild, req, expected_code=prpc.StatusCode.PERMISSION_DENIED)
def test_priority(self): msg = rpc_pb2.ScheduleBuildRequest( builder=build_pb2.BuilderID(project='chromium', bucket='try', builder='linux-rel'), priority=256, ) self.assert_invalid(msg, r'priority: must be in \[0, 255\]')
def test_notify_pubsub_topic(self): msg = rpc_pb2.ScheduleBuildRequest( builder=build_pb2.BuilderID(project='chromium', bucket='try', builder='linux-rel'), notify=notification_pb2.NotificationConfig(), ) self.assert_invalid(msg, r'notify.pubsub_topic: required')
def test_cipd_version_latest(self): msg = rpc_pb2.ScheduleBuildRequest( builder=build_pb2.BuilderID(project='chromium', bucket='try', builder='linux-rel'), exe=dict(cipd_version='latest'), ) self.assert_valid(msg)
def test_tags(self): msg = rpc_pb2.ScheduleBuildRequest( builder=build_pb2.BuilderID( project='chromium', bucket='try', builder='linux-rel' ), tags=[dict()] ) self.assert_invalid(msg, r'tags: Invalid tag ":": starts with ":"')
def test_dimensions(self): msg = rpc_pb2.ScheduleBuildRequest( builder=build_pb2.BuilderID( project='chromium', bucket='try', builder='linux-rel' ), dimensions=[dict()] ) self.assert_invalid(msg, r'dimensions\[0\]\.key: required')
def test_cipd_version_invalid(self): msg = rpc_pb2.ScheduleBuildRequest( builder=build_pb2.BuilderID(project='chromium', bucket='try', builder='linux-rel'), exe=dict(cipd_version=':'), ) self.assert_invalid(msg, r'exe.cipd_version: invalid version ":"')
def test_expiration_not_suported(self): msg = rpc_pb2.ScheduleBuildRequest( builder=dict(project='chromium', bucket='try', builder='linux-rel'), dimensions=[ dict(key='a', value='b', expiration=dict(seconds=10)), ], ) self.assert_invalid(msg, r'dimensions\[0\]\.expiration: not supported')
def test_empty_property_value(self): msg = rpc_pb2.ScheduleBuildRequest( builder=dict(project='chromium', bucket='try', builder='linux-rel'), properties=dict(fields=dict(a=struct_pb2.Value())), ) self.assert_invalid( msg, r'properties\.a: value is not set; for null, initialize null_value' )
def test_cipd_package(self): msg = rpc_pb2.ScheduleBuildRequest( builder=build_pb2.BuilderID(project='chromium', bucket='try', builder='linux-rel'), exe=dict(cipd_package='something'), ) self.assert_invalid(msg, r'exe.cipd_package: disallowed')
def test_put_with_gerrit_change(self, add_async): buildset = 'patch/gerrit/gerrit.example.com/1234/5' buildset_tag = 'buildset:' + buildset props = {'patch_project': 'repo'} expected_sbr = rpc_pb2.ScheduleBuildRequest( builder=dict( project='chromium', bucket='try', builder='linux', ), gerrit_changes=[ dict( host='gerrit.example.com', project='repo', change=1234, patchset=5, ) ], tags=[dict(key='t', value='0')], request_id='42', properties=bbutil.dict_to_struct(props), ) expected_request = creation.BuildRequest( schedule_build_request=expected_sbr, parameters={}, ) build = test_util.build( id=1, input=dict( gerrit_changes=expected_sbr.gerrit_changes, properties=expected_sbr.properties, ), tags=expected_sbr.tags, ) build.tags.append(buildset_tag) build.tags.sort() add_async.return_value = future(build) req = { 'client_operation_id': '42', 'bucket': 'luci.chromium.try', 'tags': [buildset_tag, 't:0'], 'parameters_json': json.dumps({ api_common.BUILDER_PARAMETER: 'linux', api_common.PROPERTIES_PARAMETER: props, }), } resp = self.call_api('put', req).json_body add_async.assert_called_once_with(expected_request) self.assertEqual(resp['build']['id'], '1') self.assertIn(buildset_tag, resp['build']['tags']) self.assertIn('t:0', resp['build']['tags'])
def test_gitiles_commit_incomplete(self): msg = rpc_pb2.ScheduleBuildRequest( builder=build_pb2.BuilderID(project='chromium', bucket='try', builder='linux-rel'), gitiles_commit=common_pb2.GitilesCommit(host='gerrit.example.com', project='project'), ) self.assert_invalid(msg, r'gitiles_commit\.ref: required')
def test_gerrit_change(self): msg = rpc_pb2.ScheduleBuildRequest( builder=build_pb2.BuilderID(project='chromium', bucket='try', builder='linux-rel'), gerrit_changes=[ common_pb2.GerritChange(host='gerrit.example.com', change=2), ], ) self.assert_invalid(msg, r'gerrit_changes\[0\]\.patchset: required')
def test_repeating_dimension_key_and_expiration(self): msg = rpc_pb2.ScheduleBuildRequest( builder=dict(project='chromium', bucket='try', builder='linux-rel'), dimensions=[ dict(key='a', value='b'), dict(key='a', value='b'), ], ) self.assert_invalid( msg, r'dimensions: key "a" and expiration 0s are not unique')
def test_notify_user_data(self): msg = rpc_pb2.ScheduleBuildRequest( builder=build_pb2.BuilderID(project='chromium', bucket='try', builder='linux-rel'), notify=notification_pb2.NotificationConfig( pubsub_topic='x', user_data='a' * 5000, ), ) self.assert_invalid(msg, r'notify.user_data: must be <= 4096 bytes')
def test_put_with_commit(self, add_async): buildset = ( 'commit/gitiles/gitiles.example.com/chromium/src/+/' 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' ) buildset_tag = 'buildset:' + buildset gitiles_ref_tag = 'gitiles_ref:refs/heads/master' gitiles_commit = common_pb2.GitilesCommit( host='gitiles.example.com', project='chromium/src', ref='refs/heads/master', id='aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', ) build = test_util.build( id=1, input=dict(gitiles_commit=gitiles_commit), tags=[dict(key='t', value='0')], ) build.tags.append(buildset_tag) build.tags.append(gitiles_ref_tag) build.tags.sort() add_async.return_value = future(build) req = { 'client_operation_id': '42', 'bucket': 'luci.chromium.try', 'tags': [buildset_tag, gitiles_ref_tag, 't:0'], 'parameters_json': json.dumps({api_common.BUILDER_PARAMETER: 'linux'}), } resp = self.call_api('put', req).json_body add_async.assert_called_once_with( creation.BuildRequest( schedule_build_request=rpc_pb2.ScheduleBuildRequest( builder=dict( project='chromium', bucket='try', builder='linux', ), gitiles_commit=gitiles_commit, tags=[dict(key='t', value='0')], request_id='42', properties=dict(), ), parameters={}, ) ) self.assertEqual(resp['build']['id'], '1') self.assertIn(buildset_tag, resp['build']['tags']) self.assertIn(gitiles_ref_tag, resp['build']['tags']) self.assertIn('t:0', resp['build']['tags'])
def test_schedule(self, add_async): add_async.return_value = future( test_util.build( id=54, builder=dict(project='chromium', bucket='try', builder='linux'), ), ) req = rpc_pb2.ScheduleBuildRequest(builder=dict(project='chromium', bucket='try', builder='linux'), ) res = self.call(self.api.ScheduleBuild, req) self.assertEqual(res.id, 54) add_async.assert_called_once_with( creation.BuildRequest(schedule_build_request=req))
def test_reserved_properties(self): properties = [ {'buildbucket': 1}, {'buildername': 1}, {'$recipe_engine/runtime': {'is_luci': 1}}, {'$recipe_engine/runtime': {'is_experimental': 1}}, ] for p in properties: msg = rpc_pb2.ScheduleBuildRequest( builder=build_pb2.BuilderID( project='chromium', bucket='try', builder='linux-rel' ), ) msg.properties.update(p) self.assert_invalid(msg, r'property .+ is reserved')
def retry(self, request): """Retries an existing build.""" lease_expiration_date = parse_datetime(request.lease_expiration_ts) errors.validate_lease_expiration_date(lease_expiration_date) build = model.Build.get_by_id(request.id) if not build: raise errors.BuildNotFoundError('Build %s not found' % request.id) check_scheduling_permissions([build.bucket_id]) # Prepare v2 request. sbr = rpc_pb2.ScheduleBuildRequest( builder=build.proto.builder, request_id=request.client_operation_id, canary=common_pb2.YES if build.canary else common_pb2.NO, properties=build.proto.input.properties, gerrit_changes=build.proto.input.gerrit_changes[:], ) build.tags_to_protos(sbr.tags) if build.input_properties_bytes: # pragma: no branch sbr.properties.ParseFromString(build.input_properties_bytes) if build.proto.input.HasField('gitiles_commit'): # pragma: no branch sbr.gitiles_commit.CopyFrom(build.proto.input.gitiles_commit) # Read PubSub callback. pubsub_callback_auth_token = None if request.pubsub_callback: # pragma: no branch pubsub_callback_auth_token = request.pubsub_callback.auth_token pubsub_callback_to_notification_config(request.pubsub_callback, sbr.notify) with _wrap_validation_error(): validation.validate_notification_config(sbr.notify) # Remove properties from parameters. params = build.parameters.copy() params.pop(model.PROPERTIES_PARAMETER, None) # Create the build. build_req = creation.BuildRequest( schedule_build_request=sbr, parameters=params, lease_expiration_date=lease_expiration_date, retry_of=request.id, pubsub_callback_auth_token=pubsub_callback_auth_token, ) build = creation.add_async(build_req).get_result() return build_to_response_message(build, include_lease_key=True)
def test_put(self, add_async): build = test_util.build(id=1, tags=[dict(key='a', value='b')]) add_async.return_value = future(build) props = {'foo': 'bar'} parameters_json = json.dumps({ api_common.BUILDER_PARAMETER: 'linux', api_common.PROPERTIES_PARAMETER: props, }) req = { 'client_operation_id': '42', 'bucket': 'luci.chromium.try', 'tags': ['a:b'], 'parameters_json': parameters_json, 'pubsub_callback': { 'topic': 'projects/foo/topic/bar', 'user_data': 'hello', 'auth_token': 'secret', }, } resp = self.call_api('put', req).json_body add_async.assert_called_once_with( creation.BuildRequest( schedule_build_request=rpc_pb2.ScheduleBuildRequest( builder=dict( project='chromium', bucket='try', builder='linux', ), tags=[dict(key='a', value='b')], request_id='42', notify=dict( pubsub_topic='projects/foo/topic/bar', user_data='hello', ), properties=bbutil.dict_to_struct(props), ), parameters={}, pubsub_callback_auth_token='secret', ) ) self.assertEqual(resp['build']['id'], '1') self.assertEqual(resp['build']['bucket'], req['bucket']) self.assertIn('a:b', resp['build']['tags'])
def test_put_with_v2_gerrit_changes(self, add_async): changes = [ common_pb2.GerritChange( host='chromium.googlesource.com', project='project', change=1, patchset=1, ), common_pb2.GerritChange( host='chromium.googlesource.com', project='project', change=2, patchset=1, ), ] expected_sbr = rpc_pb2.ScheduleBuildRequest( builder=dict( project='chromium', bucket='try', builder='linux', ), properties=dict(), gerrit_changes=changes, ) expected_request = creation.BuildRequest( schedule_build_request=expected_sbr, parameters={}, ) add_async.return_value = future(test_util.build(id=1)) params = { api_common.BUILDER_PARAMETER: 'linux', 'gerrit_changes': [json_format.MessageToDict(c) for c in changes], } req = { 'bucket': 'luci.chromium.try', 'parameters_json': json.dumps(params), } self.call_api('put', req) add_async.assert_called_once_with(expected_request)
def test_retry(self, add_async): props = bbutil.dict_to_struct({ 'foo': 'bar', 'recipe': 'recipe', }) orig_bundle = test_util.build_bundle( id=1, input=dict( properties=props, gitiles_commit=dict( host='gitiles.example.com', project='chromium/src', id='a' * 40, ), ), ) orig_build = orig_bundle.build orig_build.parameters.pop('changes') orig_build.tags = ['a:b'] ndb.put_multi([orig_build, orig_bundle.input_properties]) retried_build_bundle = test_util.build_bundle( id=2, input=dict( properties=orig_build.proto.input.properties, gitiles_commit=orig_build.proto.input.gitiles_commit, ), ) retried_build_bundle.infra.put() retried_build = retried_build_bundle.build retried_build.retry_of = 1 add_async.return_value = future(retried_build) req = { 'id': '1', 'client_operation_id': '42', 'pubsub_callback': { 'topic': 'projects/foo/topic/bar', 'user_data': 'hello', 'auth_token': 'secret', }, } resp = self.call_api('retry', req).json_body add_async.assert_called_once_with( creation.BuildRequest( schedule_build_request=rpc_pb2.ScheduleBuildRequest( builder=orig_build.proto.builder, request_id='42', notify=dict( pubsub_topic='projects/foo/topic/bar', user_data='hello', ), properties=props, tags=[dict(key='a', value='b')], canary=common_pb2.NO, gitiles_commit=orig_build.proto.input.gitiles_commit, ), parameters={}, lease_expiration_date=None, retry_of=1, pubsub_callback_auth_token='secret', ) ) self.assertEqual(resp['build']['id'], '2') self.assertEqual(resp['build']['bucket'], 'luci.chromium.try') self.assertEqual(resp['build']['retry_of'], '1')
def put_request_message_to_build_request(put_request): """Converts PutRequest to BuildRequest. Raises errors.InvalidInputError if the put_request is invalid. """ lease_expiration_date = parse_datetime(put_request.lease_expiration_ts) errors.validate_lease_expiration_date(lease_expiration_date) # Read parameters. parameters = parse_json_object(put_request.parameters_json, 'parameters_json') parameters = parameters or {} validate_known_build_parameters(parameters) builder = parameters.get(model.BUILDER_PARAMETER) or '' # Validate tags. buildtags.validate_tags(put_request.tags, 'new', builder=builder) # Read properties. Remove them from parameters. props = parameters.pop(model.PROPERTIES_PARAMETER, None) if props is not None and not isinstance(props, dict): raise errors.InvalidInputError( '"properties" parameter must be a JSON object or null') props = props or {} changes = parameters.get(_PARAM_CHANGES) if changes: # pragma: no branch # Buildbucket-Buildbot integration passes repo_url of the first change in # build parameter "changes" as "repository" attribute of SourceStamp. # https://chromium.googlesource.com/chromium/tools/build/+/2c6023d # /scripts/master/buildbucket/changestore.py#140 # Buildbot passes repository of the build source stamp as "repository" # build property. Recipes, in partiular bot_update recipe module, rely on # "repository" property and it is an almost sane property to support in # swarmbucket. repo_url = changes[0].get('repo_url') if repo_url: # pragma: no branch props['repository'] = repo_url # Buildbot-Buildbucket integration converts emails in changes to blamelist # property. emails = [c.get('author', {}).get('email') for c in changes] props['blamelist'] = filter(None, emails) # Create a v2 request. sbr = rpc_pb2.ScheduleBuildRequest( builder=build_pb2.BuilderID(builder=builder), properties=bbutil.dict_to_struct(props), request_id=put_request.client_operation_id, experimental=bbutil.BOOLISH_TO_TRINARY[put_request.experimental], canary=api_common.CANARY_PREFERENCE_TO_TRINARY.get( put_request.canary_preference, common_pb2.UNSET), ) sbr.builder.project, sbr.builder.bucket = config.parse_bucket_id( put_request.bucket) # Parse tags. Extract gitiles commit and gerrit changes. tags, gitiles_commit, gerrit_changes = parse_v1_tags(put_request.tags) sbr.tags.extend(tags) if gitiles_commit: sbr.gitiles_commit.CopyFrom(gitiles_commit) # Gerrit changes explicitly passed via "gerrit_changes" parameter win. gerrit_change_list = parameters.pop('gerrit_changes', None) if gerrit_change_list is not None: if not isinstance(gerrit_change_list, list): # pragma: no cover raise errors.InvalidInputError('gerrit_changes must be a list') try: gerrit_changes = [ json_format.ParseDict(c, common_pb2.GerritChange()) for c in gerrit_change_list ] except json_format.ParseError as ex: # pragma: no cover raise errors.InvalidInputError('Invalid gerrit_changes: %s' % ex) sbr.gerrit_changes.extend(gerrit_changes) if (not gerrit_changes and not sbr.builder.bucket.startswith('master.')): # pragma: no cover changes = parameters.get('changes') if isinstance(changes, list) and changes and not gitiles_commit: legacy_revision = changes[0].get('revision') if legacy_revision: raise errors.InvalidInputError( 'legacy revision without gitiles buildset tag') # Populate Gerrit project from patch_project property. # V2 API users will have to provide this. patch_project = props.get('patch_project') if len(sbr.gerrit_changes) == 1 and isinstance(patch_project, basestring): sbr.gerrit_changes[0].project = patch_project # Read PubSub callback. pubsub_callback_auth_token = None if put_request.pubsub_callback: pubsub_callback_auth_token = put_request.pubsub_callback.auth_token pubsub_callback_to_notification_config(put_request.pubsub_callback, sbr.notify) # Validate the resulting v2 request before continuing. with _wrap_validation_error(): validation.validate_schedule_build_request(sbr, legacy=True) return creation.BuildRequest( schedule_build_request=sbr, parameters=parameters, lease_expiration_date=lease_expiration_date, pubsub_callback_auth_token=pubsub_callback_auth_token, override_builder_cfg=_override_builder_cfg_func(parameters), )
def test_put_batch(self, add_many_async): bundle1 = test_util.build_bundle(id=1, tags=[dict(key='a', value='b')]) bundle2 = test_util.build_bundle(id=2) bundle1.infra.put() bundle2.infra.put() config.put_bucket( 'chromium', 'a' * 40, test_util.parse_bucket_cfg( ''' name: "luci.chromium.try" acls { role: SCHEDULER identity: "anonymous:anonymous" } ''' ), ) add_many_async.return_value = future([ (bundle1.build, None), (bundle2.build, None), (None, errors.InvalidInputError('bad')), ]) req = { 'builds': [ { 'bucket': 'luci.chromium.try', 'tags': ['a:b'], 'client_operation_id': '0', }, { 'bucket': 'luci.chromium.try', 'client_operation_id': '1', }, { 'bucket': 'luci.chromium.try', 'tags': ['bad tag'], 'client_operation_id': '2', }, { 'bucket': 'luci.chromium.try', 'client_operation_id': '3', }, ], } resp = self.call_api('put_batch', req).json_body add_many_async.assert_called_once_with([ creation.BuildRequest( schedule_build_request=rpc_pb2.ScheduleBuildRequest( builder=dict(project='chromium', bucket='try'), tags=[dict(key='a', value='b')], request_id='0', properties=dict(), ), parameters={}, ), creation.BuildRequest( schedule_build_request=rpc_pb2.ScheduleBuildRequest( builder=dict(project='chromium', bucket='try'), request_id='1', properties=dict(), ), parameters={}, ), creation.BuildRequest( schedule_build_request=rpc_pb2.ScheduleBuildRequest( builder=dict(project='chromium', bucket='try'), request_id='3', properties=dict(), ), parameters={}, ), ]) res0 = resp['results'][0] self.assertEqual(res0['client_operation_id'], '0') self.assertEqual(res0['build']['id'], '1') self.assertEqual(res0['build']['bucket'], 'luci.chromium.try') res1 = resp['results'][1] self.assertEqual(res1['client_operation_id'], '1') self.assertEqual(res1['build']['id'], '2') self.assertEqual(res1['build']['bucket'], 'luci.chromium.try') res2 = resp['results'][2] self.assertEqual( res2, { 'client_operation_id': '2', 'error': { 'reason': 'INVALID_INPUT', 'message': u'Invalid tag "bad tag": does not contain ":"', }, } ) res3 = resp['results'][3] self.assertEqual( res3, { 'client_operation_id': '3', 'error': { 'reason': 'INVALID_INPUT', 'message': 'bad', }, } )
def test_no_builder_and_template_build_id(self): msg = rpc_pb2.ScheduleBuildRequest() self.assert_invalid(msg, 'builder or template_build_id is required')
def test_incomplete_builder(self): msg = rpc_pb2.ScheduleBuildRequest(builder=build_pb2.BuilderID( project='chromium', bucket='try'), ) self.assert_invalid(msg, 'builder.builder: required')
def test_no_builder_but_template_build_id(self): msg = rpc_pb2.ScheduleBuildRequest(template_build_id=1) self.assert_valid(msg)