def test_validate_acl_cfg(self): cfg = """ invalid_field: "admins" """ result = validation.validate_config(config.self_config_set(), "acl.cfg", cfg) self.assertEqual(len(result.messages), 1) self.assertEqual(result.messages[0].severity, logging.ERROR) self.assertTrue(result.messages[0].text.startswith("Could not parse config")) cfg = """ project_access_group: "admins" """ result = validation.validate_config(config.self_config_set(), "acl.cfg", cfg) self.assertEqual(len(result.messages), 0)
def _fetch_configs(paths): """Fetches a bunch of config files in parallel and validates them. Returns: dict {path -> (Revision tuple, <config>)}. Raises: CannotLoadConfigError if some config is missing or invalid. """ paths = sorted(paths) futures = [ config.get_self_config_async( p, dest_type=_CONFIG_SCHEMAS[p]['proto_class'], store_last_good=False) for p in paths ] configs_url = _get_configs_url() out = {} for path, future in zip(paths, futures): rev, conf = future.get_result() try: validation.validate(config.self_config_set(), path, conf) except ValueError as exc: raise config.CannotLoadConfigError( 'Config %s at rev %s failed to pass validation: %s' % (path, rev, exc)) out[path] = (Revision(rev, _gitiles_url(configs_url, rev, path)), conf) return out
def test_validate_services_registry(self): cfg = ''' services { id: "a" access: "*****@*****.**" access: "user:[email protected]" access: "group:abc" } services { owners: "not an email" metadata_url: "not an url" access: "**&" access: "group:**&" access: "a:b" } services { id: "b" } services { id: "a-unsorted" } ''' result = validation.validate_config(config.self_config_set(), 'services.cfg', cfg) self.assertEqual([m.text for m in result.messages], [ 'Service #2: id is not specified', 'Service #2: invalid email: "not an email"', 'Service #2: metadata_url: hostname not specified', 'Service #2: metadata_url: scheme must be "https"', 'Service #2: access #1: invalid email: "**&"', 'Service #2: access #2: invalid group: **&', 'Service #2: access #3: Identity has invalid format: b', 'Services are not sorted by id. First offending id: a-unsorted', ])
def _fetch_configs(paths): """Fetches a bunch of config files in parallel and validates them. Returns: dict {path -> (Revision tuple, <config>)}. Raises: CannotLoadConfigError if some config is missing or invalid. """ paths = sorted(paths) configs_url = _get_configs_url() out = {} configs = utils.async_apply( paths, lambda p: config.get_self_config_async( p, dest_type=_CONFIG_SCHEMAS[p]['proto_class'], store_last_good=False)) for path, (rev, conf) in configs: if conf is None: default = _CONFIG_SCHEMAS[path].get('default') if default is None: raise CannotLoadConfigError('Config %s is missing' % path) rev, conf = '0' * 40, default try: validation.validate(config.self_config_set(), path, conf) except ValueError as exc: raise CannotLoadConfigError( 'Config %s at rev %s failed to pass validation: %s' % (path, rev, exc)) out[path] = (Revision(rev, _gitiles_url(configs_url, rev, path)), conf) return out
def test_validate_acl_cfg(self): cfg = ''' invalid_field: "admins" ''' result = validation.validate_config( config.self_config_set(), 'acl.cfg', cfg) self.assertEqual(len(result.messages), 1) self.assertEqual(result.messages[0].severity, logging.ERROR) self.assertTrue('no field named "invalid_field"' in result.messages[0].text) cfg = ''' project_access_group: "admins" ''' result = validation.validate_config( config.self_config_set(), 'acl.cfg', cfg) self.assertEqual(len(result.messages), 0)
def _fetch_configs(paths): """Fetches a bunch of config files in parallel and validates them. Returns: dict {path -> (Revision tuple, <config>)}. Raises: CannotLoadConfigError if some config is missing or invalid. """ paths = sorted(paths) futures = [ config.get_self_config_async( p, dest_type=_CONFIG_SCHEMAS[p]['proto_class'], store_last_good=False) for p in paths ] configs_url = _get_configs_url() ndb.Future.wait_all(futures) out = {} for path, future in zip(paths, futures): rev, conf = future.get_result() if conf is None: raise CannotLoadConfigError('Config %s is missing' % path) try: validation.validate(config.self_config_set(), path, conf) except ValueError as exc: raise CannotLoadConfigError( 'Config %s at rev %s failed to pass validation: %s' % (path, rev, exc)) out[path] = (Revision(rev, _gitiles_url(configs_url, rev, path)), conf) return out
def test_validate_acl_cfg(self): cfg = ''' invalid_field: "admins" ''' result = validation.validate_config(config.self_config_set(), 'acl.cfg', cfg) self.assertEqual(len(result.messages), 1) self.assertEqual(result.messages[0].severity, logging.ERROR) self.assertTrue( result.messages[0].text.startswith('Could not parse config')) cfg = ''' project_access_group: "admins" ''' result = validation.validate_config(config.self_config_set(), 'acl.cfg', cfg) self.assertEqual(len(result.messages), 0)
def self_config_set(): """Returns buildbucket's service config set.""" try: return config.self_config_set() except AttributeError: # pragma: no cover | does not get run on some bots # Raised in testbed environment because cfg_path is called # during decoration. return 'services/testbed-test'
def _get_configs_url(): """Returns URL where luci-config fetches configs from.""" try: return config.get_config_set_location(config.self_config_set()) except net.Error: logging.info( 'Could not get configs URL. Possibly config directory for this ' 'instance of swarming does not exist')
def test_validate_schemas(self): cfg = ''' schemas { name: "services/config:foo" url: "https://foo" } schemas { name: "projects:foo" url: "https://foo" } schemas { name: "projects/refs:foo" url: "https://foo" } # Invalid schemas. schemas { } schemas { name: "services/config:foo" url: "https://foo" } schemas { name: "no_colon" url: "http://foo" } schemas { name: "bad_prefix:foo" url: "https://foo" } schemas { name: "projects:foo/../a.cfg" url: "https://foo" } ''' result = validation.validate_config( config.self_config_set(), 'schemas.cfg', cfg) self.assertEqual( [m.text for m in result.messages], [ 'Schema #4: name is not specified', 'Schema #4: url: not specified', 'Schema services/config:foo: duplicate schema name', 'Schema no_colon: name must contain ":"', 'Schema no_colon: url: scheme must be "https"', ( 'Schema bad_prefix:foo: left side of ":" must be a service config ' 'set, "projects" or "projects/refs"'), ( 'Schema projects:foo/../a.cfg: ' 'must not contain ".." or "." components: foo/../a.cfg'), ] )
def test_validate_project_registry(self): cfg = ''' projects { id: "a" gitiles_location { repo: "https://a.googlesource.com/ok" ref: "refs/heads/main" path: "infra/config/generated" } } projects { id: "b" } projects { id: "a" gitiles_location { repo: "https://a.googlesource.com/project/" ref: "refs/heads/infra/config" path: "/generated" } } projects { gitiles_location { repo: "https://a.googlesource.com/project.git" ref: "branch" } } projects { id: "c" gitiles_location { repo: "https://a.googlesource.com/missed/ref" } } ''' result = validation.validate_config(config.self_config_set(), 'projects.cfg', cfg) self.assertEqual( [m.text for m in result.messages], [ 'Project b: gitiles_location: repo: not specified', 'Project b: gitiles_location: ref is not set', 'Project a: id is not unique', 'Project a: gitiles_location: repo: must not end with "/"', 'Project a: gitiles_location: path must not start with "/"', 'Project #4: id is not specified', 'Project #4: gitiles_location: repo: must not end with ".git"', 'Project #4: gitiles_location: ref must start with "refs/"', 'Project c: gitiles_location: ref is not set', 'Projects are not sorted by id. First offending id: a', ], )
def test_validate_services_registry(self): cfg = ''' services { id: "a" access: "*****@*****.**" access: "user:[email protected]" access: "group:abc" } services { owners: "not an email" config_location { storage_type: GITILES url: "../some" } metadata_url: "not an url" access: "**&" access: "group:**&" access: "a:b" } services { id: "b" config_location { storage_type: GITILES url: "https://gitiles.host.com/project" } } services { id: "a-unsorted" } ''' result = validation.validate_config( config.self_config_set(), 'services.cfg', cfg) self.assertEqual( [m.text for m in result.messages], [ 'Service #2: id is not specified', ('Service #2: config_location: ' 'storage_type must not be set if relative url is used'), 'Service #2: invalid email: "not an email"', 'Service #2: metadata_url: hostname not specified', 'Service #2: metadata_url: scheme must be "https"', 'Service #2: access #1: invalid email: "**&"', 'Service #2: access #2: invalid group: **&', 'Service #2: access #3: Identity has invalid format: b', 'Services are not sorted by id. First offending id: a-unsorted', ] )
def test_validate_project_registry(self): cfg = ''' projects { id: "a" config_location { storage_type: GITILES url: "https://a.googlesource.com/project/+/refs/heads/master" } } projects { id: "b" } projects { id: "a" config_location { storage_type: GITILES url: "https://no-project.googlesource.com" } } projects { config_location { storage_type: GITILES url: "https://example.googlesource.com/bad_plus/+" } } projects { id: "c" config_location { storage_type: GITILES url: "https://example.googlesource.com/no_ref/" } } ''' result = validation.validate_config(config.self_config_set(), 'projects.cfg', cfg) self.assertEqual([m.text for m in result.messages], [ 'Project b: config_location: storage_type is not set', 'Project a: id is not unique', ('Project a: config_location: Invalid Gitiles repo url: ' 'https://no-project.googlesource.com'), 'Project #4: id is not specified', ('Project #4: config_location: Invalid Gitiles repo url: ' 'https://example.googlesource.com/bad_plus/+'), 'Project c: config_location: ref/commit is not specified', 'Projects are not sorted by id. First offending id: a', ])
def test_validate_project_registry(self): cfg = """ projects { id: "a" config_location { storage_type: GITILES url: "https://a.googlesource.com/project" } } projects { id: "b" } projects { id: "a" config_location { storage_type: GITILES url: "https://no-project.googlesource.com" } } projects { config_location { storage_type: GITILES url: "https://no-project.googlesource.com/bad_plus/+" } } """ result = validation.validate_config(config.self_config_set(), "projects.cfg", cfg) self.assertEqual( [m.text for m in result.messages], [ "Project b: config_location: storage_type is not set", "Project a: id is not unique", ("Project a: config_location: Invalid Gitiles repo url: " "https://no-project.googlesource.com"), "Project #4: id is not specified", ( "Project #4: config_location: Invalid Gitiles repo url: " "https://no-project.googlesource.com/bad_plus/+" ), "Projects are not sorted by id. First offending id: a", ], )
def test_validate_project_registry(self): cfg = ''' projects { id: "a" config_location { storage_type: GITILES url: "https://a.googlesource.com/project" } } projects { id: "b" } projects { id: "a" config_location { storage_type: GITILES url: "https://no-project.googlesource.com" } } projects { config_location { storage_type: GITILES url: "https://no-project.googlesource.com/bad_plus/+" } } ''' result = validation.validate_config( config.self_config_set(), 'projects.cfg', cfg) self.assertEqual( [m.text for m in result.messages], [ 'Project b: config_location: storage_type is not set', 'Project a: id is not unique', ('Project a: config_location: Invalid Gitiles repo url: ' 'https://no-project.googlesource.com'), 'Project #4: id is not specified', ('Project #4: config_location: Invalid Gitiles repo url: ' 'https://no-project.googlesource.com/bad_plus/+'), 'Projects are not sorted by id. First offending id: a', ] )
def _get_configs_url(): """Returns URL where luci-config fetches configs from.""" url = config.get_config_set_location(config.self_config_set()) return url or 'about:blank'