def test_create_resource_is_ok(self):
"""Test the resource_util.create_resource() creates the types."""
expect_org = Organization(12345)
actual_org = resource_util.create_resource(
12345, ResourceType.ORGANIZATION)
self.assertEqual(expect_org, actual_org)
expect_proj = Project('abcd', 54321)
actual_proj = resource_util.create_resource(
'abcd', ResourceType.PROJECT, project_number=54321)
self.assertEqual(expect_proj, actual_proj)
self.assertEqual(expect_proj.project_number, actual_proj.project_number)
def get_folder_iam_policies(self, resource_name, timestamp):
"""Get the folder policies.
This does not raise any errors if there's a database or json parse
error because we want to return as many folders as possible.
Args:
resource_name (str): The resource type.
timestamp (str): The timestamp of the snapshot.
Returns:
dict: A dict keyed by the folders
(gcp_type.folder.Folder) and their iam policies (dict).
"""
folder_iam_policies = {}
query = select_data.FOLDER_IAM_POLICIES.format(timestamp, timestamp)
rows = self.execute_sql_with_fetch(resource_name, query, ())
for row in rows:
try:
folder = gcp_folder.Folder(
folder_id=row.get('folder_id'),
display_name=row.get('display_name'),
lifecycle_state=row.get('lifecycle_state'),
parent=resource_util.create_resource(
resource_id=row.get('parent_id'),
resource_type=row.get('parent_type')))
iam_policy = json.loads(row.get('iam_policy'))
folder_iam_policies[folder] = iam_policy
except ValueError:
LOGGER.warn('Error parsing json:\n %s', row.get('iam_policy'))
return folder_iam_policies
def add_org_policy(self, org_def):
"""Creates org policy and rule mapping.
Sample org structure:
org 1234
/ \
f-1 p-c
/ \
p-a p-b
Rules can be applied at any node above. When a policy is being audited,
it the rulebook will start at the lowest level (the project) and will
walk up the hierarchy until it reaches the first instance with rules and
these are the only rules that are checked.
Args:
org_def (dict): A dictionary of resource ids and enforced rules.
Raises:
RuleDoesntExistError: Raised if a rule included in the group does not
exist.
GroupDoesntExistError: Raised if a group included in an org policy
does not exist.
InvalidOrgDefinition: Raised if org policy doesn't have resources.
"""
resources = org_def.get('resources', [])
if not resources:
raise InvalidOrgDefinition(
'Org policy does not have any resources')
for resource in resources:
resource_type = resource_mod.ResourceType.verify(
resource.get('type'))
ids = resource.get('resource_ids', [])
rules = resource.get('rules', {})
groups = rules.get('group_ids', [])
expanded_rules = set()
for group_id in groups:
if group_id not in self.rule_groups_map:
raise GroupDoesntExistError('Group "%s" does not exist' %
group_id)
expanded_group = self.rule_groups_map.get(group_id, [])
expanded_rules.update(expanded_group)
for rule_id in rules.get('rule_ids', []):
if rule_id not in self.rules_map:
raise RuleDoesntExistError('Rule id "%s" does not exist' %
rule_id)
expanded_rules.add(rule_id)
for resource_id in ids:
gcp_resource = resource_util.create_resource(
resource_id=resource_id, resource_type=resource_type)
self.org_policy_rules_map[gcp_resource] = sorted(
expanded_rules)
def add_rule(self, rule_def, rule_index):
"""Add a rule to the rule book.
Args:
rule_def (dict): A dictionary containing rule definition
properties.
rule_index (int): The index of the rule from the rule definitions.
Assigned automatically when the rule book is built.
"""
with self._lock:
for resource in rule_def.get('resource'):
resource_ids = resource.get('resource_ids')
resource_type = None
try:
resource_type = resource_mod.ResourceType.verify(
resource.get('type'))
except resource_errors.InvalidResourceTypeError:
raise audit_errors.InvalidRulesSchemaError(
'Missing resource type in rule {}'.format(rule_index))
if not resource_ids or len(resource_ids) < 1:
raise audit_errors.InvalidRulesSchemaError(
'Missing resource ids in rule {}'.format(rule_index))
check_serverconfig_valid_node_versions = rule_def.get(
'check_serverconfig_valid_node_versions', False)
check_serverconfig_valid_master_versions = rule_def.get(
'check_serverconfig_valid_master_versions', False)
allowed_nodepool_versions = rule_def.get(
'allowed_nodepool_versions', [])
allowed_versions = []
for allowed_version in allowed_nodepool_versions:
allowed_versions.append(VersionRule(**allowed_version))
# For each resource id associated with the rule, create a
# mapping of resource => rules.
for resource_id in resource_ids:
gcp_resource = resource_util.create_resource(
resource_id=resource_id,
resource_type=resource_type)
rule = Rule(
rule_def.get('name'),
rule_index,
check_serverconfig_valid_node_versions,
check_serverconfig_valid_master_versions,
allowed_versions)
resource_rules = self.resource_rules_map.setdefault(
gcp_resource, ResourceRules(resource=gcp_resource))
if rule not in resource_rules.rules:
resource_rules.rules.add(rule)
def find_violations(self, resource, policy_binding):
"""Find policy binding violations in the rule book.
Args:
resource (Resource): The GCP resource associated with the
policy binding.
This is where we start looking for rule violations and
we move up the resource hierarchy (if permitted by the
resource's "inherit_from_parents" property).
policy_binding (IamPolicyBinding): An IamPolicyBinding.
Returns:
iterable: A generator of the rule violations.
"""
violations = itertools.chain()
resource_ancestors = [resource]
resource_ancestors.extend(
self.org_res_rel_dao.find_ancestors(resource,
self.snapshot_timestamp))
for curr_resource in resource_ancestors:
wildcard_resource = resource_util.create_resource(
resource_id='*', resource_type=curr_resource.type)
resource_rules = self._get_resource_rules(curr_resource)
resource_rules.extend(self._get_resource_rules(wildcard_resource))
# Set to None, because if the direct resource (e.g. project)
# doesn't have a specific rule, we still should check the
# ancestry to see if the resource's parents have any rules
# that apply to the children.
inherit_from_parents = None
for resource_rule in resource_rules:
if not self._rule_applies_to_resource(resource, curr_resource,
resource_rule):
continue
violations = itertools.chain(
violations,
resource_rule.find_mismatches(resource, policy_binding))
inherit_from_parents = resource_rule.inherit_from_parents
# If the rule does not inherit the parents' rules, stop.
# Due to the way rules are structured, we only define the
# "inherit" property once per rule. So even though a rule
# may apply to multiple resources, it will only have one
# value for "inherit_from_parents".
if not inherit_from_parents and inherit_from_parents is not None:
break
return violations
def map_row_to_object(row):
"""Instantiate a Folder from a database row.
TODO: Make this go away when we start using an ORM.
Args:
row (dict): The database row to map to the Folder object.
Returns:
Folder: A Folder from the database row.
"""
return gcp_folder.Folder(folder_id=row.get('folder_id'),
name=row.get('name'),
display_name=row.get('display_name'),
lifecycle_state=row.get('lifecycle_state'),
parent=resource_util.create_resource(
resource_id=row.get('parent_id'),
resource_type=row.get('parent_type')))
def _find_violations(self, policies):
"""Find violations in the policies.
Args:
policies (list): The list of policies to find violations in.
Returns:
list: A list of all violations
"""
all_violations = []
LOGGER.info('Finding firewall policy violations...')
for resource_id, p_policies in policies.items():
resource = resource_util.create_resource(
resource_id=resource_id, resource_type='project')
LOGGER.debug('%s => %s', resource, p_policies)
violations = self.rules_engine.find_policy_violations(
resource, p_policies)
all_violations.extend(violations)
return all_violations
def map_row_to_object(row):
"""Instantiate a Project from database row.
TODO: Make this go away when we start using an ORM.
ProjectDao has a special case because the database schema doesn't
match the GCP API fields.
Args:
row: The database row to map.
Returns:
A Project, created from the row.
"""
return project.Project(project_id=row['project_id'],
project_number=row['project_number'],
display_name=row['project_name'],
lifecycle_state=row['lifecycle_state'],
parent=resource_util.create_resource(
resource_id=row['parent_id'],
resource_type=row['parent_type']))
def find_violations(self, ke_cluster):
"""Find violations in the rule book.
Args:
ke_cluster (KeCluster): KE Cluster and ServerConfig data.
Returns:
list: RuleViolation
"""
LOGGER.debug('Looking for KE violations: %r', ke_cluster)
violations = []
resource_ancestors = []
project = self.project_dao.get_project(ke_cluster.project_id,
self.snapshot_timestamp)
resource_ancestors.append(project)
resource_ancestors.extend(
self.org_res_rel_dao.find_ancestors(
project, self.snapshot_timestamp))
LOGGER.debug('Ancestors of resource: %r', resource_ancestors)
checked_wildcards = set()
for curr_resource in resource_ancestors:
resource_rule = self.get_resource_rules(curr_resource)
if resource_rule:
violations.extend(
resource_rule.find_policy_violations(ke_cluster))
wildcard_resource = resource_util.create_resource(
resource_id='*', resource_type=curr_resource.type)
if wildcard_resource in checked_wildcards:
continue
checked_wildcards.add(wildcard_resource)
resource_rule = self.get_resource_rules(wildcard_resource)
if resource_rule:
violations.extend(
resource_rule.find_policy_violations(ke_cluster))
LOGGER.debug('Returning violations: %r', violations)
return violations
def add_rule(self, rule_def, rule_index):
"""Add a rule to the rule book.
The rule supplied to this method is the dictionary parsed from
the rules definition file.
For example, this rule...
# rules yaml:
rules:
- name: a rule
mode: whitelist
resource:
- type: project
applies_to: self
resource_ids:
- my-project-123
inherit_from_parents: true
bindings:
- role: roles/editor
members:
- users:[email protected]
... gets parsed into:
{
'name': 'a rule',
'mode': 'whitelist',
'resource': {
'type': 'project',
'applies_to': self,
'resource_ids': ['my-project-id']
},
'inherit_from_parents': true,
'bindings': [
{
'role': 'roles/editor',
'members': ['users:[email protected]']
}
]
}
Args:
rule_def (dict): Contains rule definition properties.
rule_index (int): The index of the rule from the rule definitions.
Assigned automatically when the rule book is built.
"""
self._rules_sema.acquire()
try:
resources = rule_def.get('resource')
for resource in resources:
resource_ids = resource.get('resource_ids')
resource_type = None
# TODO: collect these errors and output them in a log.
# TODO: log the error and keep going
try:
resource_type = resource_mod.ResourceType.verify(
resource.get('type'))
except resource_errors.InvalidResourceTypeError:
raise audit_errors.InvalidRulesSchemaError(
'Missing resource type in rule {}'.format(rule_index))
if not resource_ids or len(resource_ids) < 1:
raise audit_errors.InvalidRulesSchemaError(
'Missing resource ids in rule {}'.format(rule_index))
# For each resource id associated with the rule, create a
# mapping of resource => rules.
for resource_id in resource_ids:
gcp_resource = resource_util.create_resource(
resource_id=resource_id, resource_type=resource_type)
rule_bindings = [
iam_policy.IamPolicyBinding.create_from(b)
for b in rule_def.get('bindings')
]
rule = scanner_rules.Rule(rule_name=rule_def.get('name'),
rule_index=rule_index,
bindings=rule_bindings,
mode=rule_def.get('mode'))
rule_applies_to = resource.get('applies_to')
rule_key = (gcp_resource, rule_applies_to)
# See if we have a mapping of the resource and rule
resource_rules = self.resource_rules_map.get(rule_key)
# If no mapping exists, create it.
if not resource_rules:
resource_rules = ResourceRules(
resource=gcp_resource,
applies_to=rule_applies_to,
inherit_from_parents=rule_def.get(
'inherit_from_parents', False))
self.resource_rules_map[rule_key] = resource_rules
# If the rule isn't in the mapping, add it.
if rule not in resource_rules.rules:
resource_rules.rules.add(rule)
finally:
self._rules_sema.release()
def test_create_nonexist_resource_returns_None(self):
"""Test that nonexistent resource type creates None."""
self.assertIsNone(
resource_util.create_resource('fake-id', 'nonexist'))
def find_violations(self, iap_resource):
"""Find violations in the rule book.
Args:
iap_resource (IapResource): IAP data
Returns:
list: RuleViolation
"""
LOGGER.debug('Looking for IAP violations: %r', iap_resource)
violations = []
resource = iap_resource.backend_service
resource_ancestors = [resource]
project = self.project_dao.get_project(resource.project_id,
self.snapshot_timestamp)
resource_ancestors.append(project)
resource_ancestors.extend(
self.org_res_rel_dao.find_ancestors(project,
self.snapshot_timestamp))
LOGGER.debug('Ancestors of resource: %r', resource_ancestors)
for curr_resource in resource_ancestors:
wildcard_resource = resource_util.create_resource(
resource_id='*', resource_type=curr_resource.type)
resource_rules = self.get_resource_rules(curr_resource)
resource_rules.extend(self.get_resource_rules(wildcard_resource))
LOGGER.debug('Resource rules for %r: %r', curr_resource,
resource_rules)
# Set to None, because if the direct resource (e.g. project)
# doesn't have a specific rule, we still should check the
# ancestry to see if the resource's parents have any rules
# that apply to the children.
inherit_from_parents = None
for resource_rule in resource_rules:
# Check whether rules match if the applies_to condition is met:
# SELF: check rules if the starting resource == current resource
# CHILDREN: check rules if starting resource != current resource
# SELF_AND_CHILDREN: always check rules
applies_to_self = (resource_rule.applies_to
== scanner_rules.RuleAppliesTo.SELF
and resource == curr_resource)
applies_to_children = (resource_rule.applies_to
== scanner_rules.RuleAppliesTo.CHILDREN
and resource != curr_resource)
applies_to_both = (resource_rule.applies_to == scanner_rules.
RuleAppliesTo.SELF_AND_CHILDREN)
rule_applies_to_resource = (applies_to_self
or applies_to_children
or applies_to_both)
LOGGER.debug('Does %r apply to resource? %r', resource_rule,
rule_applies_to_resource)
if not rule_applies_to_resource:
continue
violations.extend(
resource_rule.find_mismatches(resource, iap_resource))
inherit_from_parents = resource_rule.inherit_from_parents
# If the rule does not inherit the parents' rules, stop.
# Due to the way rules are structured, we only define the
# "inherit" property once per rule. So even though a rule
# may apply to multiple resources, it will only have one
# value for "inherit_from_parents".
# TODO: Revisit to remove pylint disable
# pylint: disable=compare-to-zero
if inherit_from_parents is False:
break
# pylint: enable=compare-to-zero
LOGGER.debug('Returning violations: %r', violations)
return violations
def add_rule(self, rule_def, rule_index): # pylint: disable=too-many-locals
"""Add a rule to the rule book.
Args:
rule_def (dict): rule definition properties
rule_index (int): index of the rule from the rule definitions,
assigned automatically when the rule book is built
"""
self._rules_sema.acquire()
try:
for resource in rule_def.get('resource'):
resource_ids = resource.get('resource_ids')
resource_type = None
try:
resource_type = resource_mod.ResourceType.verify(
resource.get('type'))
except resource_errors.InvalidResourceTypeError:
raise audit_errors.InvalidRulesSchemaError(
'Missing resource type in rule {}'.format(rule_index))
if not resource_ids or len(resource_ids) < 1:
raise audit_errors.InvalidRulesSchemaError(
'Missing resource ids in rule {}'.format(rule_index))
allowed_alternate_services = [
regex_util.escape_and_globify(glob) for glob in
rule_def.get('allowed_alternate_services', '').split(',')
if glob
]
allowed_direct_access_sources = [
regex_util.escape_and_globify(glob)
for glob in rule_def.get('allowed_direct_access_sources',
'').split(',') if glob
]
allowed_iap_enabled = regex_util.escape_and_globify(
rule_def.get('allowed_iap_enabled', '*'))
# For each resource id associated with the rule, create a
# mapping of resource => rules.
for resource_id in resource_ids:
gcp_resource = resource_util.create_resource(
resource_id=resource_id, resource_type=resource_type)
rule = Rule(
rule_name=rule_def.get('name'),
rule_index=rule_index,
allowed_alternate_services=allowed_alternate_services,
allowed_direct_access_sources=(
allowed_direct_access_sources),
allowed_iap_enabled=allowed_iap_enabled)
rule_applies_to = resource.get('applies_to')
rule_key = (gcp_resource, rule_applies_to)
# See if we have a mapping of the resource and rule
resource_rules = self.resource_rules_map.get(rule_key)
# If no mapping exists, create it.
if not resource_rules:
resource_rules = ResourceRules(
resource=gcp_resource,
applies_to=rule_applies_to,
inherit_from_parents=rule_def.get(
'inherit_from_parents', False))
self.resource_rules_map[rule_key] = resource_rules
# If the rule isn't in the mapping, add it.
if rule not in resource_rules.rules:
resource_rules.rules.add(rule)
finally:
self._rules_sema.release()
