def test_get_buckets_async(self): config.Bucket( id='master.tryserver.chromium.linux', project_id='chromium', revision='deadbeef', config_content=MASTER_TRYSERVER_CHROMIUM_LINUX_CONFIG_TEXT).put() config.Bucket( id='master.tryserver.chromium.win', project_id='chromium', revision='deadbeef', config_content=MASTER_TRYSERVER_CHROMIUM_WIN_CONFIG_TEXT).put() actual = config.get_buckets_async().get_result() expected = [ project_config_pb2.Bucket( name='master.tryserver.chromium.linux', acls=[ project_config_pb2.Acl(role=project_config_pb2.Acl.READER, group='all'), project_config_pb2.Acl( role=project_config_pb2.Acl.SCHEDULER, group='tryjob-access'), ]), project_config_pb2.Bucket( name='master.tryserver.chromium.win', acls=[ project_config_pb2.Acl(role=project_config_pb2.Acl.READER, group='all'), project_config_pb2.Acl( role=project_config_pb2.Acl.SCHEDULER, group='tryjob-access'), ]), ] self.assertEqual(actual, expected)
def test_validate_buildbucket_cfg_fail(self): self.cfg_validation_test( project_config_pb2.BuildbucketCfg(buckets=[ project_config_pb2.Bucket( name='a', acls=[ project_config_pb2.Acl( group='writers', identity='*****@*****.**', role=project_config_pb2.Acl.READER), project_config_pb2.Acl( role=project_config_pb2.Acl.READER), ]), project_config_pb2.Bucket( name='b', acls=[ project_config_pb2.Acl( identity='ldap', role=project_config_pb2.Acl.READER), project_config_pb2.Acl( group='*****@*****.**', role=project_config_pb2.Acl.READER), ]), project_config_pb2.Bucket(), ]), [ errmsg( 'Bucket a: acl #1: either group or identity must be set, ' 'not both'), errmsg('Bucket a: acl #2: group or identity must be set'), errmsg('Bucket b: acl #1: Identity has invalid format: ldap'), errmsg('Bucket b: acl #2: invalid group: [email protected]'), errmsg('Bucket #3: invalid name: Bucket not specified'), ])
def test_validate_buildbucket_cfg_unsorted(self): self.cfg_validation_test( project_config_pb2.BuildbucketCfg(buckets=[ project_config_pb2.Bucket(name='b'), project_config_pb2.Bucket(name='a') ]), [ validation_context.Message( severity=logging.WARNING, text='Buckets are not sorted by name'), ])
def test_get_bucket_async(self): config.Bucket( id='master.tryserver.chromium.linux', project_id='chromium', revision='deadbeef', config_content=MASTER_TRYSERVER_CHROMIUM_LINUX_CONFIG_TEXT, config_content_binary=text_to_binary( MASTER_TRYSERVER_CHROMIUM_LINUX_CONFIG_TEXT), ).put() project, cfg = config.get_bucket_async( 'master.tryserver.chromium.linux').get_result() self.assertEqual(project, 'chromium') self.assertEqual( cfg, project_config_pb2.Bucket( name='master.tryserver.chromium.linux', acls=[ project_config_pb2.Acl(role=project_config_pb2.Acl.READER, group='all'), project_config_pb2.Acl( role=project_config_pb2.Acl.SCHEDULER, group='tryjob-access'), ]), ) self.assertIsNone( config.get_bucket_async('non.existing').get_result()[0])
def test_cron_update_buckets_change_reservation(self): config.Bucket( id='bucket', project_id='foo', revision='deadbeef', config_content='name: "bucket"', ).put() buildbucket_cfg = project_config_pb2.BuildbucketCfg( buckets=[project_config_pb2.Bucket(name='bucket')]) self.mock(config_component, 'get_project_configs', mock.Mock()) config_component.get_project_configs.return_value = { 'bar': ('deadbeef', buildbucket_cfg), } config.cron_update_buckets() actual = config.Bucket.query().fetch() expected = [ config.Bucket( id='bucket', project_id='bar', revision='deadbeef', config_content='name: "bucket"\n', ) ] self.assertEqual(actual, expected)
def test_validate_buildbucket_cfg_duplicate_names(self): config.Bucket(id='master.tryserver.v8', project_id='v8', revision='deadbeef', config_content=MASTER_TRYSERVER_V8_CONFIG_TEXT).put() self.cfg_validation_test( project_config_pb2.BuildbucketCfg(buckets=[ project_config_pb2.Bucket(name='a'), project_config_pb2.Bucket(name='a'), project_config_pb2.Bucket( name='master.tryserver.chromium.linux'), project_config_pb2.Bucket(name='master.tryserver.v8'), ]), [ errmsg('Bucket a: duplicate bucket name'), errmsg('Bucket master.tryserver.v8: ' 'this name is already reserved by another project'), ])
def test_update_global_metrics(self, set_build_status_metric, get_buckets_async): get_buckets_async.return_value = future( [project_config_pb2.Bucket(name='x')]) metrics.update_global_metrics() set_build_status_metric.assert_any_call(metrics.CURRENTLY_PENDING, 'x', model.BuildStatus.SCHEDULED) set_build_status_metric.assert_any_call(metrics.CURRENTLY_RUNNING, 'x', model.BuildStatus.STARTED)
def test_validate_buildbucket_cfg_success(self): self.cfg_validation_test( project_config_pb2.BuildbucketCfg(buckets=[ project_config_pb2.Bucket( name='good.name', acls=[ project_config_pb2.Acl( group='writers', role=project_config_pb2.Acl.WRITER) ], ), project_config_pb2.Bucket( name='good.name2', acls=[ project_config_pb2.Acl( identity='*****@*****.**', role=project_config_pb2.Acl.READER), project_config_pb2.Acl( identity='user:[email protected]', role=project_config_pb2.Acl.READER), ], ) ]), [])
def test_send_all_metrics(self): buf = mock.Mock() self.mock(metrics_component, 'Buffer', lambda: buf) self.mock(config, 'get_buckets_async', mock.Mock()) config.get_buckets_async.return_value = future( [project_config_pb2.Bucket(name='x')]) self.mock(metrics, 'send_build_status_metric', mock.Mock()) metrics.send_all_metrics() metrics.send_build_status_metric.assert_any_call( buf, 'x', metrics.METRIC_PENDING_BUILDS, model.BuildStatus.SCHEDULED) metrics.send_build_status_metric.assert_any_call( buf, 'x', metrics.METRIC_RUNNING_BUILDS, model.BuildStatus.STARTED)
def setUp(self): super(SearchTest, self).setUp() self.current_identity = auth.Identity('service', 'unittest') self.patch('components.auth.get_current_identity', side_effect=lambda: self.current_identity) self.patch('user.can_async', return_value=future(True)) self.now = datetime.datetime(2015, 1, 1) self.patch('components.utils.utcnow', side_effect=lambda: self.now) self.chromium_try = project_config_pb2.Bucket(name='try') self.patch('config.get_bucket_async', return_value=future({'deadbeef': self.chromium_try})) self.patch( 'user.get_accessible_buckets_async', autospec=True, return_value=future({'chromium/try'}), ) self.patch('search.TagIndex.random_shard_index', return_value=0)
def text_to_binary(bucket_cfg_text): cfg = project_config_pb2.Bucket() protobuf.text_format.Merge(bucket_cfg_text, cfg) return cfg.SerializeToString()
def test_is_swarming_config(self): cfg = project_config_pb2.Bucket() self.assertFalse(config.is_swarming_config(cfg)) cfg.swarming.hostname = 'exists.now' self.assertTrue(config.is_swarming_config(cfg))
def parse_binary_bucket_config(cfg_bytes): cfg = project_config_pb2.Bucket() cfg.MergeFromString(cfg_bytes) return cfg
def parse_bucket_config(text): cfg = project_config_pb2.Bucket() protobuf.text_format.Merge(text, cfg) return cfg
def test_cron_update_buckets_with_existing(self): config.Bucket( id='master.tryserver.chromium.linux', project_id='chromium', revision='deadbeef', config_content=MASTER_TRYSERVER_CHROMIUM_LINUX_CONFIG_TEXT, ).put() # Will not be updated. config.Bucket( id='master.tryserver.v8', project_id='v8', revision='deadbeef', config_content=MASTER_TRYSERVER_V8_CONFIG_TEXT, ).put() # Will be deleted. config.Bucket( id='master.tryserver.chromium.win', project_id='chromium', revision='deadbeef', config_content=MASTER_TRYSERVER_CHROMIUM_WIN_CONFIG_TEXT, ).put() chromium_buildbucket_cfg = project_config_pb2.BuildbucketCfg(buckets=[ project_config_pb2.Bucket( name='master.tryserver.chromium.linux', acls=[ project_config_pb2.Acl(role=project_config_pb2.Acl.READER, group='all'), project_config_pb2.Acl( role=project_config_pb2.Acl.SCHEDULER, group='tryjob-access'), ], ), # Will be added. project_config_pb2.Bucket( name='master.tryserver.chromium.mac', acls=[ project_config_pb2.Acl(role=project_config_pb2.Acl.READER, group='all'), project_config_pb2.Acl( role=project_config_pb2.Acl.SCHEDULER, group='tryjob-access'), ], ), ]) v8_buildbucket_cfg = project_config_pb2.BuildbucketCfg( buckets=[ # Reservation will fail. project_config_pb2.Bucket( name='master.tryserver.chromium.linux', acls=[ project_config_pb2.Acl( role=project_config_pb2.Acl.WRITER, group='v8-team') ], ), # Will not be updated. project_config_pb2.Bucket( name='master.tryserver.v8', acls=[ project_config_pb2.Acl( role=project_config_pb2.Acl.WRITER, group='v8-team') ], ), ], ) self.mock(config_component, 'get_project_configs', mock.Mock()) config_component.get_project_configs.return_value = { 'chromium': ('new!', chromium_buildbucket_cfg), 'v8': ('deadbeef', v8_buildbucket_cfg), } config.cron_update_buckets() actual = config.Bucket.query().fetch() actual = sorted(actual, key=lambda b: b.key.id()) expected = [ config.Bucket( id='master.tryserver.chromium.linux', project_id='chromium', revision='new!', config_content=MASTER_TRYSERVER_CHROMIUM_LINUX_CONFIG_TEXT, ), config.Bucket( id='master.tryserver.chromium.mac', project_id='chromium', revision='new!', config_content=MASTER_TRYSERVER_CHROMIUM_MAC_CONFIG_TEXT, ), config.Bucket( id='master.tryserver.v8', project_id='v8', revision='deadbeef', config_content=MASTER_TRYSERVER_V8_CONFIG_TEXT, ), ] self.assertEqual(actual, expected)
def setUp(self): super(SwarmingTest, self).setUp() self.now = datetime.datetime(2015, 11, 30) self.patch('components.utils.utcnow', autospec=True, return_value=self.now) self.json_response = None def json_request_async(*_, **__): if self.json_response is not None: return future(self.json_response) self.fail('unexpected outbound request') # pragma: no cover self.patch('components.net.json_request_async', autospec=True, side_effect=json_request_async) bucket_cfg_text = ''' name: "bucket" swarming { hostname: "chromium-swarm.appspot.com" url_format: "https://example.com/{swarming_hostname}/{task_id}" builders { name: "linux_chromium_rel_ng" swarming_tags: "buildertag:yes" swarming_tags: "commontag:yes" dimensions: "cores:8" dimensions: "os:Linux" dimensions: "pool:Chrome" priority: 108 build_numbers: true recipe { repository: "https://example.com/repo" name: "recipe" properties_j: "predefined-property:\\\"x\\\"" properties_j: "predefined-property-bool:true" } caches { path: "a" name: "a" } caches { path: "git_cache" name: "git_chromium" } caches { path: "out" name: "build_chromium" } } } ''' self.bucket_cfg = project_config_pb2.Bucket() protobuf.text_format.Merge(bucket_cfg_text, self.bucket_cfg) self.patch('config.get_bucket_async', autospec=True, return_value=future(('chromium', self.bucket_cfg))) self.task_template = { 'name': 'buildbucket:${bucket}:${builder}', 'priority': '100', 'expiration_secs': '3600', 'tags': [ ('log_location:logdog://luci-logdog-dev.appspot.com/${project}/' 'buildbucket/${hostname}/${build_id}/+/annotations'), 'luci_project:${project}', ], 'properties': { 'execution_timeout_secs': '3600', 'extra_args': [ 'cook', '-repository', '${repository}', '-revision', '${revision}', '-recipe', '${recipe}', '-properties', '${properties_json}', '-logdog-project', '${project}', ], 'caches': [ { 'path': '${cache_dir}/builder', 'name': 'builder_${builder_hash}' }, ], 'cipd_input': { 'packages': [ { 'package_name': 'infra/test/bar/${os_ver}', 'path': '.', 'version': 'latest', }, { 'package_name': 'infra/test/foo/${platform}', 'path': 'third_party', 'version': 'stable', }, ], }, }, 'numerical_value_for_coverage_in_format_obj': 42, } self.task_template_canary = self.task_template.copy() self.task_template_canary['name'] += '-canary' def get_self_config_async(path, *_args, **_kwargs): if path not in ( 'swarming_task_template.json', 'swarming_task_template_canary.json'): # pragma: no cover self.fail() if path == 'swarming_task_template.json': template = self.task_template else: template = self.task_template_canary return future( ('template_rev', json.dumps(template) if template is not None else None)) self.patch('components.config.get_self_config_async', side_effect=get_self_config_async) self.patch('components.auth.delegate_async', return_value=future('blah')) self.patch( 'google.appengine.api.app_identity.get_default_version_hostname', return_value='cr-buildbucket.appspot.com')
def test_cron_update_buckets(self): chromium_buildbucket_cfg = project_config_pb2.BuildbucketCfg(buckets=[ project_config_pb2.Bucket( name='master.tryserver.chromium.linux', acls=[ project_config_pb2.Acl(role=project_config_pb2.Acl.READER, group='all'), project_config_pb2.Acl( role=project_config_pb2.Acl.SCHEDULER, group='tryjob-access'), ], ), project_config_pb2.Bucket( name='master.tryserver.chromium.win', acls=[ project_config_pb2.Acl(role=project_config_pb2.Acl.READER, group='all'), project_config_pb2.Acl( role=project_config_pb2.Acl.SCHEDULER, group='tryjob-access'), ], ), ]) v8_buildbucket_cfg = project_config_pb2.BuildbucketCfg(buckets=[ project_config_pb2.Bucket( name='master.tryserver.v8', acls=[ project_config_pb2.Acl(role=project_config_pb2.Acl.WRITER, group='v8-team') ], ), ]) test_buildbucket_cfg = project_config_pb2.BuildbucketCfg(buckets=[ project_config_pb2.Bucket( name='master.tryserver.test', acls=[ project_config_pb2.Acl(role=project_config_pb2.Acl.WRITER, identity='*****@*****.**') ], ), ]) self.mock(config_component, 'get_project_configs', mock.Mock()) config_component.get_project_configs.return_value = { 'chromium': ('deadbeef', chromium_buildbucket_cfg), 'v8': (None, v8_buildbucket_cfg), 'test': ('babe', test_buildbucket_cfg), } config.cron_update_buckets() actual = config.Bucket.query().fetch() actual = sorted(actual, key=lambda b: b.key) expected = [ config.Bucket( id='master.tryserver.chromium.linux', project_id='chromium', revision='deadbeef', config_content=MASTER_TRYSERVER_CHROMIUM_LINUX_CONFIG_TEXT, ), config.Bucket( id='master.tryserver.chromium.win', project_id='chromium', revision='deadbeef', config_content=MASTER_TRYSERVER_CHROMIUM_WIN_CONFIG_TEXT, ), config.Bucket(id='master.tryserver.test', project_id='test', revision='babe', config_content=MASTER_TRYSERVER_TEST_CONFIG_TEXT), config.Bucket( id='master.tryserver.v8', project_id='v8', revision='sha1:cfc761d7a953a72ddea8f3d4c9a28e69777ca22c', config_content=MASTER_TRYSERVER_V8_CONFIG_TEXT, ), ] self.assertEqual(actual, expected)