def test_validate_acl_cfg(self):
cfg = """
invalid_field: "admins"
"""
result = validation.validate_config(config.self_config_set(), "acl.cfg", cfg)
self.assertEqual(len(result.messages), 1)
self.assertEqual(result.messages[0].severity, logging.ERROR)
self.assertTrue(result.messages[0].text.startswith("Could not parse config"))
cfg = """
project_access_group: "admins"
"""
result = validation.validate_config(config.self_config_set(), "acl.cfg", cfg)
self.assertEqual(len(result.messages), 0)
def _fetch_configs(paths):
"""Fetches a bunch of config files in parallel and validates them.
Returns:
dict {path -> (Revision tuple, <config>)}.
Raises:
CannotLoadConfigError if some config is missing or invalid.
"""
paths = sorted(paths)
futures = [
config.get_self_config_async(
p, dest_type=_CONFIG_SCHEMAS[p]['proto_class'], store_last_good=False)
for p in paths
]
configs_url = _get_configs_url()
out = {}
for path, future in zip(paths, futures):
rev, conf = future.get_result()
try:
validation.validate(config.self_config_set(), path, conf)
except ValueError as exc:
raise config.CannotLoadConfigError(
'Config %s at rev %s failed to pass validation: %s' %
(path, rev, exc))
out[path] = (Revision(rev, _gitiles_url(configs_url, rev, path)), conf)
return out
def test_validate_services_registry(self):
cfg = '''
services {
id: "a"
access: "*****@*****.**"
access: "user:[email protected]"
access: "group:abc"
}
services {
owners: "not an email"
metadata_url: "not an url"
access: "**&"
access: "group:**&"
access: "a:b"
}
services {
id: "b"
}
services {
id: "a-unsorted"
}
'''
result = validation.validate_config(config.self_config_set(),
'services.cfg', cfg)
self.assertEqual([m.text for m in result.messages], [
'Service #2: id is not specified',
'Service #2: invalid email: "not an email"',
'Service #2: metadata_url: hostname not specified',
'Service #2: metadata_url: scheme must be "https"',
'Service #2: access #1: invalid email: "**&"',
'Service #2: access #2: invalid group: **&',
'Service #2: access #3: Identity has invalid format: b',
'Services are not sorted by id. First offending id: a-unsorted',
])
def _fetch_configs(paths):
"""Fetches a bunch of config files in parallel and validates them.
Returns:
dict {path -> (Revision tuple, <config>)}.
Raises:
CannotLoadConfigError if some config is missing or invalid.
"""
paths = sorted(paths)
configs_url = _get_configs_url()
out = {}
configs = utils.async_apply(
paths, lambda p: config.get_self_config_async(
p,
dest_type=_CONFIG_SCHEMAS[p]['proto_class'],
store_last_good=False))
for path, (rev, conf) in configs:
if conf is None:
default = _CONFIG_SCHEMAS[path].get('default')
if default is None:
raise CannotLoadConfigError('Config %s is missing' % path)
rev, conf = '0' * 40, default
try:
validation.validate(config.self_config_set(), path, conf)
except ValueError as exc:
raise CannotLoadConfigError(
'Config %s at rev %s failed to pass validation: %s' %
(path, rev, exc))
out[path] = (Revision(rev, _gitiles_url(configs_url, rev, path)), conf)
return out
def test_validate_acl_cfg(self):
cfg = '''
invalid_field: "admins"
'''
result = validation.validate_config(
config.self_config_set(), 'acl.cfg', cfg)
self.assertEqual(len(result.messages), 1)
self.assertEqual(result.messages[0].severity, logging.ERROR)
self.assertTrue('no field named "invalid_field"' in result.messages[0].text)
cfg = '''
project_access_group: "admins"
'''
result = validation.validate_config(
config.self_config_set(), 'acl.cfg', cfg)
self.assertEqual(len(result.messages), 0)
def _fetch_configs(paths):
"""Fetches a bunch of config files in parallel and validates them.
Returns:
dict {path -> (Revision tuple, <config>)}.
Raises:
CannotLoadConfigError if some config is missing or invalid.
"""
paths = sorted(paths)
futures = [
config.get_self_config_async(
p,
dest_type=_CONFIG_SCHEMAS[p]['proto_class'],
store_last_good=False) for p in paths
]
configs_url = _get_configs_url()
ndb.Future.wait_all(futures)
out = {}
for path, future in zip(paths, futures):
rev, conf = future.get_result()
if conf is None:
raise CannotLoadConfigError('Config %s is missing' % path)
try:
validation.validate(config.self_config_set(), path, conf)
except ValueError as exc:
raise CannotLoadConfigError(
'Config %s at rev %s failed to pass validation: %s' %
(path, rev, exc))
out[path] = (Revision(rev, _gitiles_url(configs_url, rev, path)), conf)
return out
def test_validate_acl_cfg(self):
cfg = '''
invalid_field: "admins"
'''
result = validation.validate_config(config.self_config_set(),
'acl.cfg', cfg)
self.assertEqual(len(result.messages), 1)
self.assertEqual(result.messages[0].severity, logging.ERROR)
self.assertTrue(
result.messages[0].text.startswith('Could not parse config'))
cfg = '''
project_access_group: "admins"
'''
result = validation.validate_config(config.self_config_set(),
'acl.cfg', cfg)
self.assertEqual(len(result.messages), 0)
def self_config_set():
"""Returns buildbucket's service config set."""
try:
return config.self_config_set()
except AttributeError: # pragma: no cover | does not get run on some bots
# Raised in testbed environment because cfg_path is called
# during decoration.
return 'services/testbed-test'
def _get_configs_url():
"""Returns URL where luci-config fetches configs from."""
try:
return config.get_config_set_location(config.self_config_set())
except net.Error:
logging.info(
'Could not get configs URL. Possibly config directory for this '
'instance of swarming does not exist')
def test_validate_schemas(self):
cfg = '''
schemas {
name: "services/config:foo"
url: "https://foo"
}
schemas {
name: "projects:foo"
url: "https://foo"
}
schemas {
name: "projects/refs:foo"
url: "https://foo"
}
# Invalid schemas.
schemas {
}
schemas {
name: "services/config:foo"
url: "https://foo"
}
schemas {
name: "no_colon"
url: "http://foo"
}
schemas {
name: "bad_prefix:foo"
url: "https://foo"
}
schemas {
name: "projects:foo/../a.cfg"
url: "https://foo"
}
'''
result = validation.validate_config(
config.self_config_set(), 'schemas.cfg', cfg)
self.assertEqual(
[m.text for m in result.messages],
[
'Schema #4: name is not specified',
'Schema #4: url: not specified',
'Schema services/config:foo: duplicate schema name',
'Schema no_colon: name must contain ":"',
'Schema no_colon: url: scheme must be "https"',
(
'Schema bad_prefix:foo: left side of ":" must be a service config '
'set, "projects" or "projects/refs"'),
(
'Schema projects:foo/../a.cfg: '
'must not contain ".." or "." components: foo/../a.cfg'),
]
)
def test_validate_schemas(self):
cfg = '''
schemas {
name: "services/config:foo"
url: "https://foo"
}
schemas {
name: "projects:foo"
url: "https://foo"
}
schemas {
name: "projects/refs:foo"
url: "https://foo"
}
# Invalid schemas.
schemas {
}
schemas {
name: "services/config:foo"
url: "https://foo"
}
schemas {
name: "no_colon"
url: "http://foo"
}
schemas {
name: "bad_prefix:foo"
url: "https://foo"
}
schemas {
name: "projects:foo/../a.cfg"
url: "https://foo"
}
'''
result = validation.validate_config(
config.self_config_set(), 'schemas.cfg', cfg)
self.assertEqual(
[m.text for m in result.messages],
[
'Schema #4: name is not specified',
'Schema #4: url: not specified',
'Schema services/config:foo: duplicate schema name',
'Schema no_colon: name must contain ":"',
'Schema no_colon: url: scheme must be "https"',
(
'Schema bad_prefix:foo: left side of ":" must be a service config '
'set, "projects" or "projects/refs"'),
(
'Schema projects:foo/../a.cfg: '
'must not contain ".." or "." components: foo/../a.cfg'),
]
)
def test_validate_project_registry(self):
cfg = '''
projects {
id: "a"
gitiles_location {
repo: "https://a.googlesource.com/ok"
ref: "refs/heads/main"
path: "infra/config/generated"
}
}
projects {
id: "b"
}
projects {
id: "a"
gitiles_location {
repo: "https://a.googlesource.com/project/"
ref: "refs/heads/infra/config"
path: "/generated"
}
}
projects {
gitiles_location {
repo: "https://a.googlesource.com/project.git"
ref: "branch"
}
}
projects {
id: "c"
gitiles_location {
repo: "https://a.googlesource.com/missed/ref"
}
}
'''
result = validation.validate_config(config.self_config_set(),
'projects.cfg', cfg)
self.assertEqual(
[m.text for m in result.messages],
[
'Project b: gitiles_location: repo: not specified',
'Project b: gitiles_location: ref is not set',
'Project a: id is not unique',
'Project a: gitiles_location: repo: must not end with "/"',
'Project a: gitiles_location: path must not start with "/"',
'Project #4: id is not specified',
'Project #4: gitiles_location: repo: must not end with ".git"',
'Project #4: gitiles_location: ref must start with "refs/"',
'Project c: gitiles_location: ref is not set',
'Projects are not sorted by id. First offending id: a',
],
)
def test_validate_services_registry(self):
cfg = '''
services {
id: "a"
access: "*****@*****.**"
access: "user:[email protected]"
access: "group:abc"
}
services {
owners: "not an email"
config_location {
storage_type: GITILES
url: "../some"
}
metadata_url: "not an url"
access: "**&"
access: "group:**&"
access: "a:b"
}
services {
id: "b"
config_location {
storage_type: GITILES
url: "https://gitiles.host.com/project"
}
}
services {
id: "a-unsorted"
}
'''
result = validation.validate_config(
config.self_config_set(), 'services.cfg', cfg)
self.assertEqual(
[m.text for m in result.messages],
[
'Service #2: id is not specified',
('Service #2: config_location: '
'storage_type must not be set if relative url is used'),
'Service #2: invalid email: "not an email"',
'Service #2: metadata_url: hostname not specified',
'Service #2: metadata_url: scheme must be "https"',
'Service #2: access #1: invalid email: "**&"',
'Service #2: access #2: invalid group: **&',
'Service #2: access #3: Identity has invalid format: b',
'Services are not sorted by id. First offending id: a-unsorted',
]
)
def test_validate_services_registry(self):
cfg = '''
services {
id: "a"
access: "*****@*****.**"
access: "user:[email protected]"
access: "group:abc"
}
services {
owners: "not an email"
config_location {
storage_type: GITILES
url: "../some"
}
metadata_url: "not an url"
access: "**&"
access: "group:**&"
access: "a:b"
}
services {
id: "b"
config_location {
storage_type: GITILES
url: "https://gitiles.host.com/project"
}
}
services {
id: "a-unsorted"
}
'''
result = validation.validate_config(
config.self_config_set(), 'services.cfg', cfg)
self.assertEqual(
[m.text for m in result.messages],
[
'Service #2: id is not specified',
('Service #2: config_location: '
'storage_type must not be set if relative url is used'),
'Service #2: invalid email: "not an email"',
'Service #2: metadata_url: hostname not specified',
'Service #2: metadata_url: scheme must be "https"',
'Service #2: access #1: invalid email: "**&"',
'Service #2: access #2: invalid group: **&',
'Service #2: access #3: Identity has invalid format: b',
'Services are not sorted by id. First offending id: a-unsorted',
]
)
def test_validate_project_registry(self):
cfg = '''
projects {
id: "a"
config_location {
storage_type: GITILES
url: "https://a.googlesource.com/project/+/refs/heads/master"
}
}
projects {
id: "b"
}
projects {
id: "a"
config_location {
storage_type: GITILES
url: "https://no-project.googlesource.com"
}
}
projects {
config_location {
storage_type: GITILES
url: "https://example.googlesource.com/bad_plus/+"
}
}
projects {
id: "c"
config_location {
storage_type: GITILES
url: "https://example.googlesource.com/no_ref/"
}
}
'''
result = validation.validate_config(config.self_config_set(),
'projects.cfg', cfg)
self.assertEqual([m.text for m in result.messages], [
'Project b: config_location: storage_type is not set',
'Project a: id is not unique',
('Project a: config_location: Invalid Gitiles repo url: '
'https://no-project.googlesource.com'),
'Project #4: id is not specified',
('Project #4: config_location: Invalid Gitiles repo url: '
'https://example.googlesource.com/bad_plus/+'),
'Project c: config_location: ref/commit is not specified',
'Projects are not sorted by id. First offending id: a',
])
def test_validate_project_registry(self):
cfg = """
projects {
id: "a"
config_location {
storage_type: GITILES
url: "https://a.googlesource.com/project"
}
}
projects {
id: "b"
}
projects {
id: "a"
config_location {
storage_type: GITILES
url: "https://no-project.googlesource.com"
}
}
projects {
config_location {
storage_type: GITILES
url: "https://no-project.googlesource.com/bad_plus/+"
}
}
"""
result = validation.validate_config(config.self_config_set(), "projects.cfg", cfg)
self.assertEqual(
[m.text for m in result.messages],
[
"Project b: config_location: storage_type is not set",
"Project a: id is not unique",
("Project a: config_location: Invalid Gitiles repo url: " "https://no-project.googlesource.com"),
"Project #4: id is not specified",
(
"Project #4: config_location: Invalid Gitiles repo url: "
"https://no-project.googlesource.com/bad_plus/+"
),
"Projects are not sorted by id. First offending id: a",
],
)
def test_validate_project_registry(self):
cfg = '''
projects {
id: "a"
config_location {
storage_type: GITILES
url: "https://a.googlesource.com/project"
}
}
projects {
id: "b"
}
projects {
id: "a"
config_location {
storage_type: GITILES
url: "https://no-project.googlesource.com"
}
}
projects {
config_location {
storage_type: GITILES
url: "https://no-project.googlesource.com/bad_plus/+"
}
}
'''
result = validation.validate_config(
config.self_config_set(), 'projects.cfg', cfg)
self.assertEqual(
[m.text for m in result.messages],
[
'Project b: config_location: storage_type is not set',
'Project a: id is not unique',
('Project a: config_location: Invalid Gitiles repo url: '
'https://no-project.googlesource.com'),
'Project #4: id is not specified',
('Project #4: config_location: Invalid Gitiles repo url: '
'https://no-project.googlesource.com/bad_plus/+'),
'Projects are not sorted by id. First offending id: a',
]
)
def _get_configs_url(): """Returns URL where luci-config fetches configs from.""" url = config.get_config_set_location(config.self_config_set()) return url or 'about:blank'
def _get_configs_url():
"""Returns URL where luci-config fetches configs from."""
url = config.get_config_set_location(config.self_config_set())
return url or 'about:blank'