Ejemplo n.º 1
0
    def test_validate_acl_cfg(self):
        cfg = """
      invalid_field: "admins"
    """
        result = validation.validate_config(config.self_config_set(), "acl.cfg", cfg)
        self.assertEqual(len(result.messages), 1)
        self.assertEqual(result.messages[0].severity, logging.ERROR)
        self.assertTrue(result.messages[0].text.startswith("Could not parse config"))

        cfg = """
      project_access_group: "admins"
    """
        result = validation.validate_config(config.self_config_set(), "acl.cfg", cfg)
        self.assertEqual(len(result.messages), 0)
Ejemplo n.º 2
0
def _fetch_configs(paths):
  """Fetches a bunch of config files in parallel and validates them.

  Returns:
    dict {path -> (Revision tuple, <config>)}.

  Raises:
    CannotLoadConfigError if some config is missing or invalid.
  """
  paths = sorted(paths)
  futures = [
    config.get_self_config_async(
        p, dest_type=_CONFIG_SCHEMAS[p]['proto_class'], store_last_good=False)
    for p in paths
  ]
  configs_url = _get_configs_url()
  out = {}
  for path, future in zip(paths, futures):
    rev, conf = future.get_result()
    try:
      validation.validate(config.self_config_set(), path, conf)
    except ValueError as exc:
      raise config.CannotLoadConfigError(
          'Config %s at rev %s failed to pass validation: %s' %
          (path, rev, exc))
    out[path] = (Revision(rev, _gitiles_url(configs_url, rev, path)), conf)
  return out
Ejemplo n.º 3
0
    def test_validate_services_registry(self):
        cfg = '''
      services {
        id: "a"
        access: "*****@*****.**"
        access: "user:[email protected]"
        access: "group:abc"
      }
      services {
        owners: "not an email"
        metadata_url: "not an url"
        access: "**&"
        access: "group:**&"
        access: "a:b"
      }
      services {
        id: "b"
      }
      services {
        id: "a-unsorted"
      }
    '''
        result = validation.validate_config(config.self_config_set(),
                                            'services.cfg', cfg)

        self.assertEqual([m.text for m in result.messages], [
            'Service #2: id is not specified',
            'Service #2: invalid email: "not an email"',
            'Service #2: metadata_url: hostname not specified',
            'Service #2: metadata_url: scheme must be "https"',
            'Service #2: access #1: invalid email: "**&"',
            'Service #2: access #2: invalid group: **&',
            'Service #2: access #3: Identity has invalid format: b',
            'Services are not sorted by id. First offending id: a-unsorted',
        ])
Ejemplo n.º 4
0
def _fetch_configs(paths):
    """Fetches a bunch of config files in parallel and validates them.

  Returns:
    dict {path -> (Revision tuple, <config>)}.

  Raises:
    CannotLoadConfigError if some config is missing or invalid.
  """
    paths = sorted(paths)
    configs_url = _get_configs_url()
    out = {}
    configs = utils.async_apply(
        paths, lambda p: config.get_self_config_async(
            p,
            dest_type=_CONFIG_SCHEMAS[p]['proto_class'],
            store_last_good=False))
    for path, (rev, conf) in configs:
        if conf is None:
            default = _CONFIG_SCHEMAS[path].get('default')
            if default is None:
                raise CannotLoadConfigError('Config %s is missing' % path)
            rev, conf = '0' * 40, default
        try:
            validation.validate(config.self_config_set(), path, conf)
        except ValueError as exc:
            raise CannotLoadConfigError(
                'Config %s at rev %s failed to pass validation: %s' %
                (path, rev, exc))
        out[path] = (Revision(rev, _gitiles_url(configs_url, rev, path)), conf)
    return out
Ejemplo n.º 5
0
  def test_validate_acl_cfg(self):
    cfg = '''
      invalid_field: "admins"
    '''
    result = validation.validate_config(
        config.self_config_set(), 'acl.cfg', cfg)
    self.assertEqual(len(result.messages), 1)
    self.assertEqual(result.messages[0].severity, logging.ERROR)
    self.assertTrue('no field named "invalid_field"' in result.messages[0].text)

    cfg = '''
      project_access_group: "admins"
    '''
    result = validation.validate_config(
        config.self_config_set(), 'acl.cfg', cfg)
    self.assertEqual(len(result.messages), 0)
Ejemplo n.º 6
0
def _fetch_configs(paths):
    """Fetches a bunch of config files in parallel and validates them.

  Returns:
    dict {path -> (Revision tuple, <config>)}.

  Raises:
    CannotLoadConfigError if some config is missing or invalid.
  """
    paths = sorted(paths)
    futures = [
        config.get_self_config_async(
            p,
            dest_type=_CONFIG_SCHEMAS[p]['proto_class'],
            store_last_good=False) for p in paths
    ]
    configs_url = _get_configs_url()
    ndb.Future.wait_all(futures)
    out = {}
    for path, future in zip(paths, futures):
        rev, conf = future.get_result()
        if conf is None:
            raise CannotLoadConfigError('Config %s is missing' % path)
        try:
            validation.validate(config.self_config_set(), path, conf)
        except ValueError as exc:
            raise CannotLoadConfigError(
                'Config %s at rev %s failed to pass validation: %s' %
                (path, rev, exc))
        out[path] = (Revision(rev, _gitiles_url(configs_url, rev, path)), conf)
    return out
Ejemplo n.º 7
0
    def test_validate_acl_cfg(self):
        cfg = '''
      invalid_field: "admins"
    '''
        result = validation.validate_config(config.self_config_set(),
                                            'acl.cfg', cfg)
        self.assertEqual(len(result.messages), 1)
        self.assertEqual(result.messages[0].severity, logging.ERROR)
        self.assertTrue(
            result.messages[0].text.startswith('Could not parse config'))

        cfg = '''
      project_access_group: "admins"
    '''
        result = validation.validate_config(config.self_config_set(),
                                            'acl.cfg', cfg)
        self.assertEqual(len(result.messages), 0)
Ejemplo n.º 8
0
def self_config_set():
    """Returns buildbucket's service config set."""
    try:
        return config.self_config_set()
    except AttributeError:  # pragma: no cover | does not get run on some bots
        # Raised in testbed environment because cfg_path is called
        # during decoration.
        return 'services/testbed-test'
Ejemplo n.º 9
0
def _get_configs_url():
    """Returns URL where luci-config fetches configs from."""
    try:
        return config.get_config_set_location(config.self_config_set())
    except net.Error:
        logging.info(
            'Could not get configs URL. Possibly config directory for this '
            'instance of swarming does not exist')
Ejemplo n.º 10
0
  def test_validate_schemas(self):
    cfg = '''
      schemas {
        name: "services/config:foo"
        url: "https://foo"
      }
      schemas {
        name: "projects:foo"
        url: "https://foo"
      }
      schemas {
        name: "projects/refs:foo"
        url: "https://foo"
      }
      # Invalid schemas.
      schemas {
      }
      schemas {
        name: "services/config:foo"
        url: "https://foo"
      }
      schemas {
        name: "no_colon"
        url: "http://foo"
      }
      schemas {
        name: "bad_prefix:foo"
        url: "https://foo"
      }
      schemas {
        name: "projects:foo/../a.cfg"
        url: "https://foo"
      }
    '''
    result = validation.validate_config(
        config.self_config_set(), 'schemas.cfg', cfg)

    self.assertEqual(
        [m.text for m in result.messages],
        [
          'Schema #4: name is not specified',
          'Schema #4: url: not specified',
          'Schema services/config:foo: duplicate schema name',
          'Schema no_colon: name must contain ":"',
          'Schema no_colon: url: scheme must be "https"',
          (
            'Schema bad_prefix:foo: left side of ":" must be a service config '
            'set, "projects" or "projects/refs"'),
          (
            'Schema projects:foo/../a.cfg: '
            'must not contain ".." or "." components: foo/../a.cfg'),
        ]
    )
Ejemplo n.º 11
0
  def test_validate_schemas(self):
    cfg = '''
      schemas {
        name: "services/config:foo"
        url: "https://foo"
      }
      schemas {
        name: "projects:foo"
        url: "https://foo"
      }
      schemas {
        name: "projects/refs:foo"
        url: "https://foo"
      }
      # Invalid schemas.
      schemas {
      }
      schemas {
        name: "services/config:foo"
        url: "https://foo"
      }
      schemas {
        name: "no_colon"
        url: "http://foo"
      }
      schemas {
        name: "bad_prefix:foo"
        url: "https://foo"
      }
      schemas {
        name: "projects:foo/../a.cfg"
        url: "https://foo"
      }
    '''
    result = validation.validate_config(
        config.self_config_set(), 'schemas.cfg', cfg)

    self.assertEqual(
        [m.text for m in result.messages],
        [
          'Schema #4: name is not specified',
          'Schema #4: url: not specified',
          'Schema services/config:foo: duplicate schema name',
          'Schema no_colon: name must contain ":"',
          'Schema no_colon: url: scheme must be "https"',
          (
            'Schema bad_prefix:foo: left side of ":" must be a service config '
            'set, "projects" or "projects/refs"'),
          (
            'Schema projects:foo/../a.cfg: '
            'must not contain ".." or "." components: foo/../a.cfg'),
        ]
    )
Ejemplo n.º 12
0
    def test_validate_project_registry(self):
        cfg = '''
      projects {
        id: "a"
        gitiles_location {
          repo: "https://a.googlesource.com/ok"
          ref: "refs/heads/main"
          path: "infra/config/generated"
        }
      }
      projects {
        id: "b"
      }
      projects {
        id: "a"
        gitiles_location {
          repo: "https://a.googlesource.com/project/"
          ref: "refs/heads/infra/config"
          path: "/generated"
        }
      }
      projects {
        gitiles_location {
          repo: "https://a.googlesource.com/project.git"
          ref: "branch"
        }
      }
      projects {
        id: "c"
        gitiles_location {
          repo: "https://a.googlesource.com/missed/ref"
        }
      }
    '''
        result = validation.validate_config(config.self_config_set(),
                                            'projects.cfg', cfg)

        self.assertEqual(
            [m.text for m in result.messages],
            [
                'Project b: gitiles_location: repo: not specified',
                'Project b: gitiles_location: ref is not set',
                'Project a: id is not unique',
                'Project a: gitiles_location: repo: must not end with "/"',
                'Project a: gitiles_location: path must not start with "/"',
                'Project #4: id is not specified',
                'Project #4: gitiles_location: repo: must not end with ".git"',
                'Project #4: gitiles_location: ref must start with "refs/"',
                'Project c: gitiles_location: ref is not set',
                'Projects are not sorted by id. First offending id: a',
            ],
        )
Ejemplo n.º 13
0
  def test_validate_services_registry(self):
    cfg = '''
      services {
        id: "a"
        access: "*****@*****.**"
        access: "user:[email protected]"
        access: "group:abc"
      }
      services {
        owners: "not an email"
        config_location {
          storage_type: GITILES
          url: "../some"
        }
        metadata_url: "not an url"
        access: "**&"
        access: "group:**&"
        access: "a:b"
      }
      services {
        id: "b"
        config_location {
          storage_type: GITILES
          url: "https://gitiles.host.com/project"
        }
      }
      services {
        id: "a-unsorted"
      }
    '''
    result = validation.validate_config(
        config.self_config_set(), 'services.cfg', cfg)

    self.assertEqual(
        [m.text for m in result.messages],
        [
          'Service #2: id is not specified',
          ('Service #2: config_location: '
           'storage_type must not be set if relative url is used'),
          'Service #2: invalid email: "not an email"',
          'Service #2: metadata_url: hostname not specified',
          'Service #2: metadata_url: scheme must be "https"',
          'Service #2: access #1: invalid email: "**&"',
          'Service #2: access #2: invalid group: **&',
          'Service #2: access #3: Identity has invalid format: b',
          'Services are not sorted by id. First offending id: a-unsorted',
        ]
    )
Ejemplo n.º 14
0
  def test_validate_services_registry(self):
    cfg = '''
      services {
        id: "a"
        access: "*****@*****.**"
        access: "user:[email protected]"
        access: "group:abc"
      }
      services {
        owners: "not an email"
        config_location {
          storage_type: GITILES
          url: "../some"
        }
        metadata_url: "not an url"
        access: "**&"
        access: "group:**&"
        access: "a:b"
      }
      services {
        id: "b"
        config_location {
          storage_type: GITILES
          url: "https://gitiles.host.com/project"
        }
      }
      services {
        id: "a-unsorted"
      }
    '''
    result = validation.validate_config(
        config.self_config_set(), 'services.cfg', cfg)

    self.assertEqual(
        [m.text for m in result.messages],
        [
          'Service #2: id is not specified',
          ('Service #2: config_location: '
           'storage_type must not be set if relative url is used'),
          'Service #2: invalid email: "not an email"',
          'Service #2: metadata_url: hostname not specified',
          'Service #2: metadata_url: scheme must be "https"',
          'Service #2: access #1: invalid email: "**&"',
          'Service #2: access #2: invalid group: **&',
          'Service #2: access #3: Identity has invalid format: b',
          'Services are not sorted by id. First offending id: a-unsorted',
        ]
    )
Ejemplo n.º 15
0
    def test_validate_project_registry(self):
        cfg = '''
      projects {
        id: "a"
        config_location {
          storage_type: GITILES
          url: "https://a.googlesource.com/project/+/refs/heads/master"
        }
      }
      projects {
        id: "b"
      }
      projects {
        id: "a"
        config_location {
          storage_type: GITILES
          url: "https://no-project.googlesource.com"
        }
      }
      projects {
        config_location {
          storage_type: GITILES
          url: "https://example.googlesource.com/bad_plus/+"
        }
      }
      projects {
        id: "c"
        config_location {
          storage_type: GITILES
          url: "https://example.googlesource.com/no_ref/"
        }
      }
    '''
        result = validation.validate_config(config.self_config_set(),
                                            'projects.cfg', cfg)

        self.assertEqual([m.text for m in result.messages], [
            'Project b: config_location: storage_type is not set',
            'Project a: id is not unique',
            ('Project a: config_location: Invalid Gitiles repo url: '
             'https://no-project.googlesource.com'),
            'Project #4: id is not specified',
            ('Project #4: config_location: Invalid Gitiles repo url: '
             'https://example.googlesource.com/bad_plus/+'),
            'Project c: config_location: ref/commit is not specified',
            'Projects are not sorted by id. First offending id: a',
        ])
Ejemplo n.º 16
0
    def test_validate_project_registry(self):
        cfg = """
      projects {
        id: "a"
        config_location {
          storage_type: GITILES
          url: "https://a.googlesource.com/project"
        }
      }
      projects {
        id: "b"
      }
      projects {
        id: "a"
        config_location {
          storage_type: GITILES
          url: "https://no-project.googlesource.com"
        }
      }
      projects {
        config_location {
          storage_type: GITILES
          url: "https://no-project.googlesource.com/bad_plus/+"
        }
      }
    """
        result = validation.validate_config(config.self_config_set(), "projects.cfg", cfg)

        self.assertEqual(
            [m.text for m in result.messages],
            [
                "Project b: config_location: storage_type is not set",
                "Project a: id is not unique",
                ("Project a: config_location: Invalid Gitiles repo url: " "https://no-project.googlesource.com"),
                "Project #4: id is not specified",
                (
                    "Project #4: config_location: Invalid Gitiles repo url: "
                    "https://no-project.googlesource.com/bad_plus/+"
                ),
                "Projects are not sorted by id. First offending id: a",
            ],
        )
Ejemplo n.º 17
0
  def test_validate_project_registry(self):
    cfg = '''
      projects {
        id: "a"
        config_location {
          storage_type: GITILES
          url: "https://a.googlesource.com/project"
        }
      }
      projects {
        id: "b"
      }
      projects {
        id: "a"
        config_location {
          storage_type: GITILES
          url: "https://no-project.googlesource.com"
        }
      }
      projects {
        config_location {
          storage_type: GITILES
          url: "https://no-project.googlesource.com/bad_plus/+"
        }
      }
    '''
    result = validation.validate_config(
        config.self_config_set(), 'projects.cfg', cfg)

    self.assertEqual(
        [m.text for m in result.messages],
        [
          'Project b: config_location: storage_type is not set',
          'Project a: id is not unique',
          ('Project a: config_location: Invalid Gitiles repo url: '
           'https://no-project.googlesource.com'),
          'Project #4: id is not specified',
          ('Project #4: config_location: Invalid Gitiles repo url: '
           'https://no-project.googlesource.com/bad_plus/+'),
          'Projects are not sorted by id. First offending id: a',
        ]
    )
Ejemplo n.º 18
0
def _get_configs_url():
  """Returns URL where luci-config fetches configs from."""
  url = config.get_config_set_location(config.self_config_set())
  return url or 'about:blank'
Ejemplo n.º 19
0
def _get_configs_url():
    """Returns URL where luci-config fetches configs from."""
    url = config.get_config_set_location(config.self_config_set())
    return url or 'about:blank'