def test_get_buckets_async(self):
config.Bucket(
id='master.tryserver.chromium.linux',
project_id='chromium',
revision='deadbeef',
config_content=MASTER_TRYSERVER_CHROMIUM_LINUX_CONFIG_TEXT).put()
config.Bucket(
id='master.tryserver.chromium.win',
project_id='chromium',
revision='deadbeef',
config_content=MASTER_TRYSERVER_CHROMIUM_WIN_CONFIG_TEXT).put()
actual = config.get_buckets_async().get_result()
expected = [
project_config_pb2.Bucket(
name='master.tryserver.chromium.linux',
acls=[
project_config_pb2.Acl(role=project_config_pb2.Acl.READER,
group='all'),
project_config_pb2.Acl(
role=project_config_pb2.Acl.SCHEDULER,
group='tryjob-access'),
]),
project_config_pb2.Bucket(
name='master.tryserver.chromium.win',
acls=[
project_config_pb2.Acl(role=project_config_pb2.Acl.READER,
group='all'),
project_config_pb2.Acl(
role=project_config_pb2.Acl.SCHEDULER,
group='tryjob-access'),
]),
]
self.assertEqual(actual, expected)
def test_validate_buildbucket_cfg_fail(self):
self.cfg_validation_test(
project_config_pb2.BuildbucketCfg(buckets=[
project_config_pb2.Bucket(
name='a',
acls=[
project_config_pb2.Acl(
group='writers',
identity='*****@*****.**',
role=project_config_pb2.Acl.READER),
project_config_pb2.Acl(
role=project_config_pb2.Acl.READER),
]),
project_config_pb2.Bucket(
name='b',
acls=[
project_config_pb2.Acl(
identity='ldap',
role=project_config_pb2.Acl.READER),
project_config_pb2.Acl(
group='*****@*****.**',
role=project_config_pb2.Acl.READER),
]),
project_config_pb2.Bucket(),
]), [
errmsg(
'Bucket a: acl #1: either group or identity must be set, '
'not both'),
errmsg('Bucket a: acl #2: group or identity must be set'),
errmsg('Bucket b: acl #1: Identity has invalid format: ldap'),
errmsg('Bucket b: acl #2: invalid group: [email protected]'),
errmsg('Bucket #3: invalid name: Bucket not specified'),
])
def test_validate_buildbucket_cfg_unsorted(self):
self.cfg_validation_test(
project_config_pb2.BuildbucketCfg(buckets=[
project_config_pb2.Bucket(name='b'),
project_config_pb2.Bucket(name='a')
]), [
validation_context.Message(
severity=logging.WARNING,
text='Buckets are not sorted by name'),
])
def test_get_bucket_async(self):
config.Bucket(
id='master.tryserver.chromium.linux',
project_id='chromium',
revision='deadbeef',
config_content=MASTER_TRYSERVER_CHROMIUM_LINUX_CONFIG_TEXT,
config_content_binary=text_to_binary(
MASTER_TRYSERVER_CHROMIUM_LINUX_CONFIG_TEXT),
).put()
project, cfg = config.get_bucket_async(
'master.tryserver.chromium.linux').get_result()
self.assertEqual(project, 'chromium')
self.assertEqual(
cfg,
project_config_pb2.Bucket(
name='master.tryserver.chromium.linux',
acls=[
project_config_pb2.Acl(role=project_config_pb2.Acl.READER,
group='all'),
project_config_pb2.Acl(
role=project_config_pb2.Acl.SCHEDULER,
group='tryjob-access'),
]),
)
self.assertIsNone(
config.get_bucket_async('non.existing').get_result()[0])
def test_cron_update_buckets_change_reservation(self):
config.Bucket(
id='bucket',
project_id='foo',
revision='deadbeef',
config_content='name: "bucket"',
).put()
buildbucket_cfg = project_config_pb2.BuildbucketCfg(
buckets=[project_config_pb2.Bucket(name='bucket')])
self.mock(config_component, 'get_project_configs', mock.Mock())
config_component.get_project_configs.return_value = {
'bar': ('deadbeef', buildbucket_cfg),
}
config.cron_update_buckets()
actual = config.Bucket.query().fetch()
expected = [
config.Bucket(
id='bucket',
project_id='bar',
revision='deadbeef',
config_content='name: "bucket"\n',
)
]
self.assertEqual(actual, expected)
def test_validate_buildbucket_cfg_duplicate_names(self):
config.Bucket(id='master.tryserver.v8',
project_id='v8',
revision='deadbeef',
config_content=MASTER_TRYSERVER_V8_CONFIG_TEXT).put()
self.cfg_validation_test(
project_config_pb2.BuildbucketCfg(buckets=[
project_config_pb2.Bucket(name='a'),
project_config_pb2.Bucket(name='a'),
project_config_pb2.Bucket(
name='master.tryserver.chromium.linux'),
project_config_pb2.Bucket(name='master.tryserver.v8'),
]), [
errmsg('Bucket a: duplicate bucket name'),
errmsg('Bucket master.tryserver.v8: '
'this name is already reserved by another project'),
])
def test_update_global_metrics(self, set_build_status_metric,
get_buckets_async):
get_buckets_async.return_value = future(
[project_config_pb2.Bucket(name='x')])
metrics.update_global_metrics()
set_build_status_metric.assert_any_call(metrics.CURRENTLY_PENDING, 'x',
model.BuildStatus.SCHEDULED)
set_build_status_metric.assert_any_call(metrics.CURRENTLY_RUNNING, 'x',
model.BuildStatus.STARTED)
def test_validate_buildbucket_cfg_success(self):
self.cfg_validation_test(
project_config_pb2.BuildbucketCfg(buckets=[
project_config_pb2.Bucket(
name='good.name',
acls=[
project_config_pb2.Acl(
group='writers',
role=project_config_pb2.Acl.WRITER)
],
),
project_config_pb2.Bucket(
name='good.name2',
acls=[
project_config_pb2.Acl(
identity='*****@*****.**',
role=project_config_pb2.Acl.READER),
project_config_pb2.Acl(
identity='user:[email protected]',
role=project_config_pb2.Acl.READER),
],
)
]), [])
def test_send_all_metrics(self):
buf = mock.Mock()
self.mock(metrics_component, 'Buffer', lambda: buf)
self.mock(config, 'get_buckets_async', mock.Mock())
config.get_buckets_async.return_value = future(
[project_config_pb2.Bucket(name='x')])
self.mock(metrics, 'send_build_status_metric', mock.Mock())
metrics.send_all_metrics()
metrics.send_build_status_metric.assert_any_call(
buf, 'x', metrics.METRIC_PENDING_BUILDS,
model.BuildStatus.SCHEDULED)
metrics.send_build_status_metric.assert_any_call(
buf, 'x', metrics.METRIC_RUNNING_BUILDS, model.BuildStatus.STARTED)
def setUp(self):
super(SearchTest, self).setUp()
self.current_identity = auth.Identity('service', 'unittest')
self.patch('components.auth.get_current_identity',
side_effect=lambda: self.current_identity)
self.patch('user.can_async', return_value=future(True))
self.now = datetime.datetime(2015, 1, 1)
self.patch('components.utils.utcnow', side_effect=lambda: self.now)
self.chromium_try = project_config_pb2.Bucket(name='try')
self.patch('config.get_bucket_async',
return_value=future({'deadbeef': self.chromium_try}))
self.patch(
'user.get_accessible_buckets_async',
autospec=True,
return_value=future({'chromium/try'}),
)
self.patch('search.TagIndex.random_shard_index', return_value=0)
def text_to_binary(bucket_cfg_text):
cfg = project_config_pb2.Bucket()
protobuf.text_format.Merge(bucket_cfg_text, cfg)
return cfg.SerializeToString()
def test_is_swarming_config(self):
cfg = project_config_pb2.Bucket()
self.assertFalse(config.is_swarming_config(cfg))
cfg.swarming.hostname = 'exists.now'
self.assertTrue(config.is_swarming_config(cfg))
def parse_binary_bucket_config(cfg_bytes):
cfg = project_config_pb2.Bucket()
cfg.MergeFromString(cfg_bytes)
return cfg
def parse_bucket_config(text):
cfg = project_config_pb2.Bucket()
protobuf.text_format.Merge(text, cfg)
return cfg
def test_cron_update_buckets_with_existing(self):
config.Bucket(
id='master.tryserver.chromium.linux',
project_id='chromium',
revision='deadbeef',
config_content=MASTER_TRYSERVER_CHROMIUM_LINUX_CONFIG_TEXT,
).put()
# Will not be updated.
config.Bucket(
id='master.tryserver.v8',
project_id='v8',
revision='deadbeef',
config_content=MASTER_TRYSERVER_V8_CONFIG_TEXT,
).put()
# Will be deleted.
config.Bucket(
id='master.tryserver.chromium.win',
project_id='chromium',
revision='deadbeef',
config_content=MASTER_TRYSERVER_CHROMIUM_WIN_CONFIG_TEXT,
).put()
chromium_buildbucket_cfg = project_config_pb2.BuildbucketCfg(buckets=[
project_config_pb2.Bucket(
name='master.tryserver.chromium.linux',
acls=[
project_config_pb2.Acl(role=project_config_pb2.Acl.READER,
group='all'),
project_config_pb2.Acl(
role=project_config_pb2.Acl.SCHEDULER,
group='tryjob-access'),
],
),
# Will be added.
project_config_pb2.Bucket(
name='master.tryserver.chromium.mac',
acls=[
project_config_pb2.Acl(role=project_config_pb2.Acl.READER,
group='all'),
project_config_pb2.Acl(
role=project_config_pb2.Acl.SCHEDULER,
group='tryjob-access'),
],
),
])
v8_buildbucket_cfg = project_config_pb2.BuildbucketCfg(
buckets=[
# Reservation will fail.
project_config_pb2.Bucket(
name='master.tryserver.chromium.linux',
acls=[
project_config_pb2.Acl(
role=project_config_pb2.Acl.WRITER,
group='v8-team')
],
),
# Will not be updated.
project_config_pb2.Bucket(
name='master.tryserver.v8',
acls=[
project_config_pb2.Acl(
role=project_config_pb2.Acl.WRITER,
group='v8-team')
],
),
], )
self.mock(config_component, 'get_project_configs', mock.Mock())
config_component.get_project_configs.return_value = {
'chromium': ('new!', chromium_buildbucket_cfg),
'v8': ('deadbeef', v8_buildbucket_cfg),
}
config.cron_update_buckets()
actual = config.Bucket.query().fetch()
actual = sorted(actual, key=lambda b: b.key.id())
expected = [
config.Bucket(
id='master.tryserver.chromium.linux',
project_id='chromium',
revision='new!',
config_content=MASTER_TRYSERVER_CHROMIUM_LINUX_CONFIG_TEXT,
),
config.Bucket(
id='master.tryserver.chromium.mac',
project_id='chromium',
revision='new!',
config_content=MASTER_TRYSERVER_CHROMIUM_MAC_CONFIG_TEXT,
),
config.Bucket(
id='master.tryserver.v8',
project_id='v8',
revision='deadbeef',
config_content=MASTER_TRYSERVER_V8_CONFIG_TEXT,
),
]
self.assertEqual(actual, expected)
def setUp(self):
super(SwarmingTest, self).setUp()
self.now = datetime.datetime(2015, 11, 30)
self.patch('components.utils.utcnow',
autospec=True,
return_value=self.now)
self.json_response = None
def json_request_async(*_, **__):
if self.json_response is not None:
return future(self.json_response)
self.fail('unexpected outbound request') # pragma: no cover
self.patch('components.net.json_request_async',
autospec=True,
side_effect=json_request_async)
bucket_cfg_text = '''
name: "bucket"
swarming {
hostname: "chromium-swarm.appspot.com"
url_format: "https://example.com/{swarming_hostname}/{task_id}"
builders {
name: "linux_chromium_rel_ng"
swarming_tags: "buildertag:yes"
swarming_tags: "commontag:yes"
dimensions: "cores:8"
dimensions: "os:Linux"
dimensions: "pool:Chrome"
priority: 108
build_numbers: true
recipe {
repository: "https://example.com/repo"
name: "recipe"
properties_j: "predefined-property:\\\"x\\\""
properties_j: "predefined-property-bool:true"
}
caches {
path: "a"
name: "a"
}
caches {
path: "git_cache"
name: "git_chromium"
}
caches {
path: "out"
name: "build_chromium"
}
}
}
'''
self.bucket_cfg = project_config_pb2.Bucket()
protobuf.text_format.Merge(bucket_cfg_text, self.bucket_cfg)
self.patch('config.get_bucket_async',
autospec=True,
return_value=future(('chromium', self.bucket_cfg)))
self.task_template = {
'name':
'buildbucket:${bucket}:${builder}',
'priority':
'100',
'expiration_secs':
'3600',
'tags': [
('log_location:logdog://luci-logdog-dev.appspot.com/${project}/'
'buildbucket/${hostname}/${build_id}/+/annotations'),
'luci_project:${project}',
],
'properties': {
'execution_timeout_secs':
'3600',
'extra_args': [
'cook',
'-repository',
'${repository}',
'-revision',
'${revision}',
'-recipe',
'${recipe}',
'-properties',
'${properties_json}',
'-logdog-project',
'${project}',
],
'caches': [
{
'path': '${cache_dir}/builder',
'name': 'builder_${builder_hash}'
},
],
'cipd_input': {
'packages': [
{
'package_name': 'infra/test/bar/${os_ver}',
'path': '.',
'version': 'latest',
},
{
'package_name': 'infra/test/foo/${platform}',
'path': 'third_party',
'version': 'stable',
},
],
},
},
'numerical_value_for_coverage_in_format_obj':
42,
}
self.task_template_canary = self.task_template.copy()
self.task_template_canary['name'] += '-canary'
def get_self_config_async(path, *_args, **_kwargs):
if path not in (
'swarming_task_template.json',
'swarming_task_template_canary.json'): # pragma: no cover
self.fail()
if path == 'swarming_task_template.json':
template = self.task_template
else:
template = self.task_template_canary
return future(
('template_rev',
json.dumps(template) if template is not None else None))
self.patch('components.config.get_self_config_async',
side_effect=get_self_config_async)
self.patch('components.auth.delegate_async',
return_value=future('blah'))
self.patch(
'google.appengine.api.app_identity.get_default_version_hostname',
return_value='cr-buildbucket.appspot.com')
def test_cron_update_buckets(self):
chromium_buildbucket_cfg = project_config_pb2.BuildbucketCfg(buckets=[
project_config_pb2.Bucket(
name='master.tryserver.chromium.linux',
acls=[
project_config_pb2.Acl(role=project_config_pb2.Acl.READER,
group='all'),
project_config_pb2.Acl(
role=project_config_pb2.Acl.SCHEDULER,
group='tryjob-access'),
],
),
project_config_pb2.Bucket(
name='master.tryserver.chromium.win',
acls=[
project_config_pb2.Acl(role=project_config_pb2.Acl.READER,
group='all'),
project_config_pb2.Acl(
role=project_config_pb2.Acl.SCHEDULER,
group='tryjob-access'),
],
),
])
v8_buildbucket_cfg = project_config_pb2.BuildbucketCfg(buckets=[
project_config_pb2.Bucket(
name='master.tryserver.v8',
acls=[
project_config_pb2.Acl(role=project_config_pb2.Acl.WRITER,
group='v8-team')
],
),
])
test_buildbucket_cfg = project_config_pb2.BuildbucketCfg(buckets=[
project_config_pb2.Bucket(
name='master.tryserver.test',
acls=[
project_config_pb2.Acl(role=project_config_pb2.Acl.WRITER,
identity='*****@*****.**')
],
),
])
self.mock(config_component, 'get_project_configs', mock.Mock())
config_component.get_project_configs.return_value = {
'chromium': ('deadbeef', chromium_buildbucket_cfg),
'v8': (None, v8_buildbucket_cfg),
'test': ('babe', test_buildbucket_cfg),
}
config.cron_update_buckets()
actual = config.Bucket.query().fetch()
actual = sorted(actual, key=lambda b: b.key)
expected = [
config.Bucket(
id='master.tryserver.chromium.linux',
project_id='chromium',
revision='deadbeef',
config_content=MASTER_TRYSERVER_CHROMIUM_LINUX_CONFIG_TEXT,
),
config.Bucket(
id='master.tryserver.chromium.win',
project_id='chromium',
revision='deadbeef',
config_content=MASTER_TRYSERVER_CHROMIUM_WIN_CONFIG_TEXT,
),
config.Bucket(id='master.tryserver.test',
project_id='test',
revision='babe',
config_content=MASTER_TRYSERVER_TEST_CONFIG_TEXT),
config.Bucket(
id='master.tryserver.v8',
project_id='v8',
revision='sha1:cfc761d7a953a72ddea8f3d4c9a28e69777ca22c',
config_content=MASTER_TRYSERVER_V8_CONFIG_TEXT,
),
]
self.assertEqual(actual, expected)
