def validate_access_list(acl_list, ctx): """Validates a list of Acl messages.""" for i, acl in enumerate(acl_list): with ctx.prefix('acl #%d: ', i + 1): if acl.group and acl.identity: ctx.error('either group or identity must be set, not both') elif acl.group: if not auth.is_valid_group_name(acl.group): ctx.error('invalid group: %s', acl.group) elif acl.identity: validate_identity(acl.identity, ctx) else: ctx.error('group or identity must be set')
def validate_binding(self, binding, custom_roles_map): """Emits errors if the binding is invalid.""" self.validate_role_ref(binding.role, custom_roles_map) for p in binding.principals: if p.startswith('group:'): group = p[len('group:'):] if not auth.is_valid_group_name(group): self.error('invalid group name: "%s"', group) else: try: auth.Identity.from_bytes(p) except ValueError: self.error('invalid principal format: "%s"', p)
def validate_dimension_acls(cfg, ctx): """Validates DimensionACLs message stored in settings.cfg.""" ctx = ctx or validation.Context.raise_on_error() # Pick '<key>:*' entries upfront to check that regular '<key>:<val>' entries # aren't colliding with wildcard entries. stars = set() star_dups = set() for e in cfg.entry: for dim in e.dimension: if not dim.endswith(':*'): continue if dim in stars: star_dups.add(dim) else: stars.add(dim) seen = set() for i, e in enumerate(cfg.entry): with ctx.prefix('entry #%d: ', i + 1): if not e.dimension: ctx.error('at least one dimension is required') for dim in e.dimension: with ctx.prefix('dimension "%s": ', dim): key, _, val = dim.partition(':') if not val: ctx.error('not a "<key>:<value>" pair') if not re.match(DIMENSION_KEY_RE, key): ctx.error('key %r doesn\'t match %s' % (key, DIMENSION_KEY_RE)) if val == '*': if dim in star_dups: ctx.error('was specified multiple times') star_dups.remove( dim) # to avoid rereporting same error else: if dim in seen: ctx.error('was already specified') if '%s:*' % key in stars: ctx.error('was already specified via "%s:*"' % key) seen.add(dim) if not e.usable_by: ctx.error('"usable_by" is required') elif not auth.is_valid_group_name(e.usable_by): ctx.error('"usable_by" specifies invalid group name "%s"' % e.usable_by)
def validate_buildbucket_cfg(cfg, ctx): is_sorted = True bucket_names = set() for i, bucket in enumerate(cfg.buckets): with ctx.prefix('Bucket %s: ', bucket.name or ('#%d' % (i + 1))): try: errors.validate_bucket_name(bucket.name) except errors.InvalidInputError as ex: ctx.error('invalid name: %s', ex.message) else: if bucket.name in bucket_names: ctx.error('duplicate bucket name') else: bucket_names.add(bucket.name) if ctx.project_id: # pragma: no branch bucket_entity = Bucket.get_by_id(bucket.name) if bucket_entity and bucket_entity.project_id != ctx.project_id: ctx.error( 'this name is already reserved by another project' ) if is_sorted and i > 0 and cfg.buckets[i - 1].name: if bucket.name < cfg.buckets[i - 1].name: is_sorted = False for i, acl in enumerate(bucket.acls): with ctx.prefix('acl #%d: ', i + 1): if acl.group and acl.identity: ctx.error( 'either group or identity must be set, not both') elif acl.group: if not auth.is_valid_group_name(acl.group): ctx.error('invalid group: %s', acl.group) elif acl.identity: validate_identity(acl.identity, ctx) else: ctx.error('group or identity must be set') if not is_sorted: ctx.warning('Buckets are not sorted by name')
def validate_buildbucket_cfg(cfg, ctx): is_sorted = True bucket_names = set() for i, bucket in enumerate(cfg.buckets): with ctx.prefix('Bucket %s: ', bucket.name or ('#%d' % (i + 1))): try: errors.validate_bucket_name(bucket.name) except errors.InvalidInputError as ex: ctx.error('invalid name: %s', ex.message) else: if bucket.name in bucket_names: ctx.error('duplicate bucket name') else: bucket_names.add(bucket.name) if ctx.project_id: # pragma: no branch bucket_entity = Bucket.get_by_id(bucket.name) if bucket_entity and bucket_entity.project_id != ctx.project_id: ctx.error('this name is already reserved by another project') if is_sorted and i > 0 and cfg.buckets[i - 1].name: if bucket.name < cfg.buckets[i - 1].name: is_sorted = False for i, acl in enumerate(bucket.acls): with ctx.prefix('acl #%d: ', i + 1): if acl.group and acl.identity: ctx.error('either group or identity must be set, not both') elif acl.group: if not auth.is_valid_group_name(acl.group): ctx.error('invalid group: %s', acl.group) elif acl.identity: validate_identity(acl.identity, ctx) else: ctx.error('group or identity must be set') if bucket.HasField('swarming'): # pragma: no cover with ctx.prefix('swarming: '): swarmingcfg.validate_cfg(bucket.swarming, ctx) if not is_sorted: ctx.warning('Buckets are not sorted by name')
def load_tarball(content, systems, groups, domain): """Unzips tarball with groups and deserializes them. Args: content: byte buffer with *.tar.gz data. systems: names of external group systems expected to be in the bundle. groups: list of group name to extract, or None to extract all. domain: email domain to append to naked user ids. Returns: Dict {system name -> {group name -> list of identities}}. Raises: BundleImportError on errors. """ bundles = collections.defaultdict(dict) try: # Expected filenames are <external system name>/<group name>, skip # everything else. for filename, fileobj in extract_tar_archive(content): chunks = filename.split('/') if len(chunks) != 2 or not auth.is_valid_group_name(filename): logging.warning('Skipping file %s, not a valid name', filename) continue if groups is not None and filename not in groups: continue system = chunks[0] if system not in systems: logging.warning('Skipping file %s, not allowed', filename) continue # Do not catch BundleBadFormatError here and in effect reject the whole # bundle if at least one group file is broken. That way all existing # groups will stay intact. Simply ignoring broken group here will cause # the importer to remove it completely. bundles[system][filename] = load_group_file(fileobj.read(), domain) except tarfile.TarError as exc: raise BundleUnpackError('Not a valid tar archive: %s' % exc) return dict(bundles.iteritems())
def role_change_from_proto(proto, package_path): """RoleChange proto message -> acl.RoleChange object. Raises ValueError on format errors. """ if not acl.is_valid_role(proto.role): raise ValueError('Invalid role %s' % proto.role) user = None group = None if proto.principal.startswith('group:'): group = proto.principal[len('group:'):] if not auth.is_valid_group_name(group): raise ValueError('Invalid group name: "%s"' % group) else: # Raises ValueError if proto.user has invalid format, e.g. not 'user:...'. user = auth.Identity.from_bytes(proto.principal) return acl.RoleChange( package_path=package_path, revoke=(proto.action != RoleChange.Action.GRANT), role=proto.role, user=user, group=group)
def modify_roles(changes, caller, now): """Transactionally modifies ACLs for a bunch of packages and roles. Args: changes: list of RoleChange objects describing what modifications to apply. Order matters, modifications are applied in the order provided. caller: Identity that made this change. now: datetime with current time. Raises: ValueError if changes list contains bad changes. """ if not changes: return # Validate format of changes. for c in changes: if not isinstance(c, RoleChange): raise ValueError('Expecting RoleChange, got %s instead' % type(c).__name__) if not impl.is_valid_package_path(c.package_path): raise ValueError('Invalid package_path: %s' % c.package_path) if not is_valid_role(c.role): raise ValueError('Invalid role: %s' % c.role) if not c.user and not c.group: raise ValueError( 'RoleChange.user or RoleChange.group should be set') if c.user and c.group: raise ValueError( 'Only one of RoleChange.user or RoleChange.group should be set' ) if c.user and not isinstance(c.user, auth.Identity): raise ValueError('RoleChange.user must be auth.Identity') if c.group and not auth.is_valid_group_name(c.group): raise ValueError('Invalid RoleChange.group value') @ndb.transactional def run(): # (package_path, role) pair -> list of RoleChanges to apply to it. to_apply = collections.defaultdict(list) for c in changes: to_apply[(c.package_path, c.role)].append(c) # Grab all existing PackageACL entities, make new empty ones if missing. # Build mapping (package_path, role) -> PackageACL. entities = {} path_role_pairs = sorted(to_apply.keys()) keys = [package_acl_key(path, role) for path, role in path_role_pairs] for i, entity in enumerate(ndb.get_multi(keys)): entities[path_role_pairs[i]] = entity or PackageACL(key=keys[i]) # Helper to apply RoleChange to a list of users and groups. def apply_change(c, users, groups): if c.user: assert not c.group if c.revoke and c.user in users: users.remove(c.user) elif not c.revoke and c.user not in users: users.append(c.user) if c.group: assert not c.user if c.revoke and c.group in groups: groups.remove(c.group) elif not c.revoke and c.group not in groups: groups.append(c.group) # Apply all the changes. Collect a list of modified entities. to_put = [] for package_path, role in path_role_pairs: package_acl = entities[(package_path, role)] change_list = to_apply[(package_path, role)] # Mutate lists of users and groups. users = list(package_acl.users) groups = list(package_acl.groups) for c in change_list: apply_change(c, users, groups) # Nothing changed? if users == package_acl.users and groups == package_acl.groups: continue # Store the previous revision in the log. if package_acl.rev: to_put.append( PackageACLRevision(key=ndb.Key(PackageACLRevision, package_acl.rev, parent=package_acl.key), users=package_acl.users, groups=package_acl.groups, modified_by=package_acl.modified_by, modified_ts=package_acl.modified_ts)) # Store modified PackageACL, bump revision. package_acl.users = users package_acl.groups = groups package_acl.modified_by = caller package_acl.modified_ts = now package_acl.rev += 1 to_put.append(package_acl) # Apply all pending changes. ndb.put_multi(to_put) run()
def subtoken_from_jsonish(d): """Given JSON dict with request body returns delegation_pb2.Subtoken msg. Raises: ValueError if some fields are invalid. """ msg = delegation_pb2.Subtoken() # 'audience' is an optional list of 'group:...' or identity names. if 'audience' in d: aud = d['audience'] if not isinstance(aud, list): raise ValueError('"audience" must be a list of strings') for e in aud: if not isinstance(e, basestring): raise ValueError('"audience" must be a list of strings') if e.startswith('group:'): if not auth.is_valid_group_name(e.lstrip('group:')): raise ValueError('Invalid group name in "audience": %s' % e) else: try: auth.Identity.from_bytes(e) except ValueError as exc: raise ValueError( 'Invalid identity name "%s" in "audience": %s' % (e, exc)) msg.audience.append(str(e)) # 'services' is an optional list of identity names. if 'services' in d: services = d['services'] if not isinstance(services, list): raise ValueError('"services" must be a list of strings') for e in services: if not isinstance(e, basestring): raise ValueError('"services" must be a list of strings') try: auth.Identity.from_bytes(e) except ValueError as exc: raise ValueError( 'Invalid identity name "%s" in "services": %s' % (e, exc)) msg.services.append(str(e)) # 'validity_duration' is optional positive number within some defined bounds. if 'validity_duration' in d: dur = d['validity_duration'] if not isinstance(dur, (int, float)): raise ValueError('"validity_duration" must be a positive number') if dur < MIN_VALIDITY_DURATION_SEC or dur > MAX_VALIDITY_DURATION_SEC: raise ValueError( '"validity_duration" must be between %d and %d sec' % (MIN_VALIDITY_DURATION_SEC, MAX_VALIDITY_DURATION_SEC)) msg.validity_duration = int(dur) # 'impersonate' is an optional identity string. if 'impersonate' in d: imp = d['impersonate'] try: auth.Identity.from_bytes(imp) except ValueError as exc: raise ValueError( 'Invalid identity name "%s" in "impersonate": %s' % (imp, exc)) msg.issuer_id = str(imp) return msg
def modify_roles(changes, caller, now): """Transactionally modifies ACLs for a bunch of packages and roles. Args: changes: list of RoleChange objects describing what modifications to apply. Order matters, modifications are applied in the order provided. caller: Identity that made this change. now: datetime with current time. Raises: ValueError if changes list contains bad changes. """ if not changes: return # Validate format of changes. for c in changes: if not isinstance(c, RoleChange): raise ValueError( 'Expecting RoleChange, got %s instead' % type(c).__name__) if not impl.is_valid_package_path(c.package_path): raise ValueError('Invalid package_path: %s' % c.package_path) if not is_valid_role(c.role): raise ValueError('Invalid role: %s' % c.role) if not c.user and not c.group: raise ValueError('RoleChange.user or RoleChange.group should be set') if c.user and c.group: raise ValueError( 'Only one of RoleChange.user or RoleChange.group should be set') if c.user and not isinstance(c.user, auth.Identity): raise ValueError('RoleChange.user must be auth.Identity') if c.group and not auth.is_valid_group_name(c.group): raise ValueError('Invalid RoleChange.group value') @ndb.transactional def run(): # (package_path, role) pair -> list of RoleChanges to apply to it. to_apply = collections.defaultdict(list) for c in changes: to_apply[(c.package_path, c.role)].append(c) # Grab all existing PackageACL entities, make new empty ones if missing. # Build mapping (package_path, role) -> PackageACL. entities = {} path_role_pairs = sorted(to_apply.keys()) keys = [package_acl_key(path, role) for path, role in path_role_pairs] for i, entity in enumerate(ndb.get_multi(keys)): entities[path_role_pairs[i]] = entity or PackageACL(key=keys[i]) # Helper to apply RoleChange to a list of users and groups. def apply_change(c, users, groups): if c.user: assert not c.group if c.revoke and c.user in users: users.remove(c.user) elif not c.revoke and c.user not in users: users.append(c.user) if c.group: assert not c.user if c.revoke and c.group in groups: groups.remove(c.group) elif not c.revoke and c.group not in groups: groups.append(c.group) # Apply all the changes. Collect a list of modified entities. to_put = [] for package_path, role in path_role_pairs: package_acl = entities[(package_path, role)] change_list = to_apply[(package_path, role)] # Mutate lists of users and groups. users = list(package_acl.users) groups = list(package_acl.groups) for c in change_list: apply_change(c, users, groups) # Nothing changed? if users == package_acl.users and groups == package_acl.groups: continue # Store the previous revision in the log. if package_acl.rev: to_put.append( PackageACLRevision( key=ndb.Key( PackageACLRevision, package_acl.rev, parent=package_acl.key), users=package_acl.users, groups=package_acl.groups, modified_by=package_acl.modified_by, modified_ts=package_acl.modified_ts)) # Store modified PackageACL, bump revision. package_acl.users = users package_acl.groups = groups package_acl.modified_by = caller package_acl.modified_ts = now package_acl.rev += 1 to_put.append(package_acl) # Apply all pending changes. ndb.put_multi(to_put) run()
def _validate_pools_cfg(cfg, ctx): """Validates pools.cfg file.""" template_map = _resolve_task_template_inclusions( ctx, cfg.task_template) deployment_map = _resolve_task_template_deployments( ctx, template_map, cfg.task_template_deployment) bot_monitorings = _resolve_bot_monitoring(ctx, cfg.bot_monitoring) bot_monitoring_unreferred = set(bot_monitorings) # Currently optional if cfg.HasField("default_external_services"): _validate_external_services(ctx, cfg.default_external_services) pools = set() for i, msg in enumerate(cfg.pool): with ctx.prefix('pool #%d (%s): ', i, '|'.join(msg.name) or 'unnamed'): # Validate names. if not msg.name: ctx.error('at least one pool name must be given') for name in msg.name: if not local_config.validate_dimension_value(name): ctx.error('bad pool name "%s", not a valid dimension value', name) elif name in pools: ctx.error('pool "%s" was already declared', name) else: pools.add(name) # Validate schedulers.user. for u in msg.schedulers.user: _validate_ident(ctx, 'user', u) # Validate schedulers.group. for g in msg.schedulers.group: if not auth.is_valid_group_name(g): ctx.error('bad group name "%s"', g) # Validate schedulers.trusted_delegation. seen_peers = set() for d in msg.schedulers.trusted_delegation: with ctx.prefix('trusted_delegation #%d (%s): ', i, d.peer_id): if not d.peer_id: ctx.error('"peer_id" is required') else: peer_id = _validate_ident(ctx, 'peer_id', d.peer_id) if peer_id in seen_peers: ctx.error('peer "%s" was specified twice', d.peer_id) elif peer_id: seen_peers.add(peer_id) for i, tag in enumerate(d.require_any_of.tag): if ':' not in tag: ctx.error('bad tag #%d "%s" - must be <key>:<value>', i, tag) # Validate service accounts. for i, account in enumerate(msg.allowed_service_account): if not service_accounts.is_service_account(account): ctx.error('bad allowed_service_account #%d "%s"', i, account) # Validate service account groups. for i, group in enumerate(msg.allowed_service_account_group): if not auth.is_valid_group_name(group): ctx.error('bad allowed_service_account_group #%d "%s"', i, group) # Validate external schedulers. for i, es in enumerate(msg.external_schedulers): if not es.address: ctx.error('%sth external scheduler config had no address', i) _resolve_deployment(ctx, msg, template_map, deployment_map) if msg.bot_monitoring: if msg.bot_monitoring not in bot_monitorings: ctx.error('refer to missing bot_monitoring %r', msg.bot_monitoring) else: bot_monitoring_unreferred.discard(msg.bot_monitoring) if bot_monitoring_unreferred: ctx.error( 'bot_monitoring not referred to: %s', ', '.join(sorted(bot_monitoring_unreferred)))
def validate_group(group, ctx): if not auth.is_valid_group_name(group): ctx.error('invalid group: %s', group)
def validate_group(group, ctx): if not auth.is_valid_group_name(group): ctx.error("invalid group: %s", group)
def _validate_pools_cfg(cfg, ctx): """Validates pools.cfg file.""" template_map = _resolve_task_template_inclusions(ctx, cfg.task_template) deployment_map = _resolve_task_template_deployments( ctx, template_map, cfg.task_template_deployment) pools = set() for i, msg in enumerate(cfg.pool): with ctx.prefix('pool #%d (%s): ', i, '|'.join(msg.name) or 'unnamed'): # Validate names. if not msg.name: ctx.error('at least one pool name must be given') for name in msg.name: if not local_config.validate_dimension_value(name): ctx.error( 'bad pool name "%s", not a valid dimension value', name) elif name in pools: ctx.error('pool "%s" was already declared', name) else: pools.add(name) # Validate schedulers.user. for u in msg.schedulers.user: _validate_ident(ctx, 'user', u) # Validate schedulers.group. for g in msg.schedulers.group: if not auth.is_valid_group_name(g): ctx.error('bad group name "%s"', g) # Validate schedulers.trusted_delegation. seen_peers = set() for d in msg.schedulers.trusted_delegation: with ctx.prefix('trusted_delegation #%d (%s): ', i, d.peer_id): if not d.peer_id: ctx.error('"peer_id" is required') else: peer_id = _validate_ident(ctx, 'peer_id', d.peer_id) if peer_id in seen_peers: ctx.error('peer "%s" was specified twice', d.peer_id) elif peer_id: seen_peers.add(peer_id) for i, tag in enumerate(d.require_any_of.tag): if ':' not in tag: ctx.error( 'bad tag #%d "%s" - must be <key>:<value>', i, tag) # Validate service accounts. for i, account in enumerate(msg.allowed_service_account): if not service_accounts.is_service_account(account): ctx.error('bad allowed_service_account #%d "%s"', i, account) # Validate service account groups. for i, group in enumerate(msg.allowed_service_account_group): if not auth.is_valid_group_name(group): ctx.error('bad allowed_service_account_group #%d "%s"', i, group) _resolve_deployment(ctx, msg, template_map, deployment_map)