def _retrieve(self):
"""Runs the data collection.
Returns:
list: KE Cluster data.
"""
model_manager = self.service_config.model_manager
scoped_session, data_access = model_manager.get(self.model_name)
with scoped_session as session:
ke_clusters = []
for cluster in data_access.scanner_iter(
session, 'kubernetes_cluster'):
proj = project.Project(
project_id=cluster.parent.name,
full_name=cluster.parent.full_name,
)
ke_clusters.append(
ke_cluster.KeCluster.from_json(proj, cluster.data))
# Retrieve the service config via a separate query because session
# in the middle of yield_per() can not support simultaneous queries.
with scoped_session as session:
for cluster in ke_clusters:
position = (cluster.full_name.find('kubernetes_cluster'))
ke_cluster_type_name = (cluster.full_name[position:][:-1])
service_config = list(
data_access.scanner_iter(
session,
'kubernetes_service_config',
parent_type_name=ke_cluster_type_name))[0]
cluster.server_config = json.loads(service_config.data)
return ke_clusters
def test_find_violations_inapplicable_resource(self):
# rules are set on org 234
org = organization.Organization(
'000',
display_name='Organization 000',
full_name='organization/000/',
data='fake_org_data_000',
)
proj = project.Project(
'111',
project_number=111,
display_name='My project 111',
parent=org,
full_name='organization/000/project/111/',
data='fake_project_data_111',
)
rules_local_path = get_datafile_path(__file__,
'bigquery_test_rules_4.yaml')
rules_engine = bqe.BigqueryRulesEngine(rules_local_path)
rules_engine.build_rule_book()
fake_bq_acls = create_list_of_bq_objects_from_data()
actual_violations_list = []
for bqt in fake_bq_acls:
violation = rules_engine.find_violations(proj, bqt)
actual_violations_list.extend(violation)
self.assertEqual([], actual_violations_list)
def _retrieve(self):
"""Retrieves the data for scanner.
Returns:
List[Resource]: resources to check for violations.
Raises:
ValueError: if resources have an unexpected type.
"""
resources = []
scoped_session, data_access = self.service_config.model_manager.get(
self.model_name)
with scoped_session as session:
for resource_type in lre.SUPPORTED_LOCATION_RESOURCE_TYPES:
for resource in data_access.scanner_iter(
session, resource_type):
if resource.parent.type != 'project':
raise ValueError(
'Unexpected type of parent resource type: '
'got %s, want project' % resource.parent.type)
proj = project.Project(
project_id=resource.parent.name,
full_name=resource.parent.full_name,
)
resources.append(
resource_util.create_resource_from_json(
resource_type, proj, resource.data))
return resources
def test_instance(self):
"""Test instance.Key."""
url_1 = ('https://www.googleapis.com/compute/v1/'
'projects/foo/zones/us-central1-a/instances/bar')
obj_1 = instance.Instance('bar',
parent=project.Project('foo'),
locations=['us-central1-a'],
name='bar')
key_1 = key.Key(instance.KEY_OBJECT_KIND, {
'project_id': 'foo',
'zone': 'us-central1-a',
'name': 'bar'
})
self.assertEqual(key_1, obj_1.key)
self.assertEqual(key_1, instance.Key.from_url(url_1))
url_invalid_1 = ('https://www.googleapis.com/compute/v1/'
'zones/bar/instances/baz')
url_invalid_2 = ('https://www.googleapis.com/compute/v1/'
'projects/foo/instances/bar')
url_invalid_3 = ('https://www.googleapis.com/compute/v1/'
'projects/foo/zones/bar')
self.assertRaises(ValueError, instance.Key.from_url, url_invalid_1)
self.assertRaises(ValueError, instance.Key.from_url, url_invalid_2)
self.assertRaises(ValueError, instance.Key.from_url, url_invalid_3)
def _retrieve(self):
"""Retrieve the network interfaces for vm instances.
Return:
list: A list that contains nested lists of per-instance
InstanceNetworksInterface objects.
"""
model_manager = self.service_config.model_manager
scoped_session, data_access = model_manager.get(self.model_name)
with scoped_session as session:
network_interfaces = []
for instance_from_data_model in data_access.scanner_iter(
session, 'instance'):
proj = project.Project(
project_id=instance_from_data_model.parent.name,
full_name=instance_from_data_model.parent.full_name,
)
ins = instance.Instance.from_json(
parent=proj, json_string=instance_from_data_model.data)
network_interfaces.append(ins.create_network_interfaces())
if not network_interfaces:
LOGGER.warn('No VM network interfaces found. Exiting.')
return None, 0
return network_interfaces
def get_instance_networks_interfaces(self):
"""Get network info from a particular snapshot.
Returns:
list: A list that contains nested lists of per-instance
InstanceNetworksInterface objects.
"""
model_manager = self.service_config.model_manager
scoped_session, data_access = model_manager.get(self.model_name)
instance_from_data_models = []
with scoped_session as session:
for instance_from_data_model in data_access.scanner_iter(
session, 'instance'):
instance_from_data_models.append(instance_from_data_model)
network_interfaces = []
for instance_from_data_model in instance_from_data_models:
proj = project.Project(
project_id=instance_from_data_model.parent.name,
full_name=instance_from_data_model.parent.full_name)
ins = instance.Instance.from_json(
parent=proj, json_string=instance_from_data_model.data)
network_interfaces.append(ins.create_network_interfaces())
if not network_interfaces:
LOGGER.warning('No VM network interfaces found.')
return []
return network_interfaces
def _retrieve(self):
"""Retrieves the data for scanner.
Returns:
list: BigQuery ACL data
Raises:
ValueError: if resources have an unexpected type.
"""
model_manager = self.service_config.model_manager
scoped_session, data_access = model_manager.get(self.model_name)
with scoped_session as session:
bq_acl_data = []
policies = []
for policy in data_access.scanner_iter(session, 'dataset_policy'):
policies.append(policy)
for policy in policies:
# dataset_policy are always in a dataset, which is always in a
# project.
dataset = policy.parent
if dataset.type != 'dataset':
raise ValueError(
'Unexpected type of dataset_policy parent: '
'got %s, want dataset' % dataset.type
)
if dataset.parent.type != 'project':
raise ValueError(
'Unexpected type of dataset_policy grandparent: '
'got %s, want project' % dataset.parent.type
)
proj = project.Project(
project_id=dataset.parent.name,
full_name=dataset.parent.full_name,
data=policy.data,
)
# There is no functional use for project_id in this scanner,
# other than to identify where the dataset comes from,
# which can now be done with full_name.
# In case you are tempted to get the project_id anyways,
# do not use project_id = policy.parent.parent.name
# which will cause db session conflict.
# Instead, parse the project_id from the full_name.
bq_acls = list(BigqueryAccessControls.from_json(
project_id=None,
dataset_id=dataset.name,
full_name=policy.full_name,
acls=policy.data))
for bq_acl in bq_acls:
data = BigqueryAccessControlsData(
parent_project=proj,
bigquery_acl=bq_acl,
)
bq_acl_data.append(data)
return bq_acl_data
def _retrieve(self):
"""Retrieves the data for scanner.
Returns:
Dict[Resource, List[lien]]: mapping of a resource to the liens it
contains.
Raises:
ValueError: if resources have an unexpected type.
"""
scoped_session, data_access = self.service_config.model_manager.get(
self.model_name)
with scoped_session as session:
parent_resource_to_liens = {}
# liens can only be defined on a project currently
for project_resource in data_access.scanner_iter(
session, 'project'):
proj = project.Project(
project_id=project_resource.name,
full_name=project_resource.full_name,
)
parent_resource_to_liens[proj] = []
for lien_resource in data_access.scanner_iter(session, 'lien'):
parent_resource = lien_resource.parent
if lien_resource.parent.type != 'project':
raise ValueError(
'Unexpected type of lien resource parent: '
'got %s, want project' % parent_resource.parent.type
)
proj = project.Project(
project_id=parent_resource.name,
full_name=parent_resource.full_name,
)
parent_resource_to_liens[proj].append(lien.Lien.from_json(
parent=proj,
json_string=lien_resource.data))
return parent_resource_to_liens
def setUp(self):
"""Set up parent GCP resources for tests."""
self.org_234 = organization.Organization(
'234',
display_name='Organization 234',
full_name='organization/234/',
data='fake_org_data_234')
self.proj_1 = project.Project(
'proj-1',
project_number=11223344,
display_name='My project 1',
parent=self.org_234,
full_name='organization/234/project/proj-1/',
data='fake_project_data_2341')
def _get_instances(self, parent_type_name):
"""Retrieves instances.
Args:
parent_type_name (str): The parent resource type and name to pull.
Returns:
list: Instance
"""
instances = []
with self.scoped_session as session:
for instance in self.data_access.scanner_iter(
session, 'instance', parent_type_name=parent_type_name):
project = project_type.Project(
project_id=instance.parent.name,
full_name=instance.parent.full_name,
)
instances.append(
instance_type.Instance.from_json(
parent=project, json_string=instance.data))
return instances
def setUp(self):
"""Set up."""
self.rule_index = 0
self.bqe = bqe
self.bqe.LOGGER = mock.MagicMock()
self.fake_timestamp = '12345'
self.org = organization.Organization(
'234',
display_name='Organization 234',
full_name='organization/234/',
data='fake_org_data_234',
)
self.project = project.Project(
'p1',
project_number=11223344,
display_name='My project 1',
parent=self.org,
full_name='organization/234/project/p1/',
data='fake_project_data_2341',
)
from google.cloud.forseti.common.gcp_type import project
from google.cloud.forseti.common.gcp_type import resource_util
from google.cloud.forseti.scanner.audit import location_rules_engine
ORGANIZATION = organization.Organization(
'234',
display_name='Organization 234',
full_name='organization/234/',
data='fake_org_data_234',
)
PROJECT = project.Project(
'p1',
project_number=11223344,
display_name='Project with lien',
parent=ORGANIZATION,
full_name='organization/234/project/p1/',
data='fake_project_data_2341',
)
_BUCKET_JSON = """{
"id": "p1-bucket1",
"parent": "projects/p1",
"location": "EUROPE-WEST1"
}
"""
BUCKET = resource_util.create_resource_from_json(
'bucket', PROJECT, _BUCKET_JSON)
_CLOUD_SQL_INSTANCE_JSON = """{
from google.cloud.forseti.common.gcp_type import project
from google.cloud.forseti.common.gcp_type import resource
from google.cloud.forseti.common.gcp_type import table
from google.cloud.forseti.scanner.audit import retention_rules_engine as rre
ORGANIZATION = organization.Organization(
'123456',
display_name='Default Organization',
full_name='organization/123456/',
data='fake_org_data_123456',
)
PROJECT1 = project.Project(
'def-project-1',
project_number=11223344,
display_name='default project 1',
parent=ORGANIZATION,
full_name='organization/123456/project/def-project-1/',
data='fake_project_data_11223344',
)
PROJECT2 = project.Project(
'def-project-2',
project_number=55667788,
display_name='default project 2',
parent=ORGANIZATION,
full_name='organization/123456/project/def-project-2/',
data='fake_project_data_55667788',
)
PROJECT3 = project.Project(
'def-project-3',
from google.cloud.forseti.common.gcp_type import role
from google.cloud.forseti.common.gcp_type import organization
from google.cloud.forseti.common.gcp_type import project
from google.cloud.forseti.scanner.audit import role_rules_engine as rre
ORGANIZATION = organization.Organization(
'123456',
display_name='Default Organization',
full_name='organization/123456/',
data='fake_org_data_123456',
)
PROJECT1 = project.Project(
'def-project-1',
project_number=11223344,
display_name='default project 1',
parent=ORGANIZATION,
full_name='organization/123456/project/def-project-1/',
data='fake_project_data_11223344',
)
PROJECT2 = project.Project(
'def-project-2',
project_number=55667788,
display_name='default project 2',
parent=ORGANIZATION,
full_name='organization/123456/project/def-project-2/',
data='fake_project_data_55667788',
)
PROJECT3 = project.Project(
'def-project-3',
),
# Preempted by deny rule.
'preempted_deny':
firewall_rule_type.FirewallRule(
project_id='foo',
firewall_rule_name='preempted_deny',
firewall_rule_priority=1,
firewall_rule_network='global/networks/default',
firewall_rule_source_ranges=json.dumps(['preempted']),
firewall_rule_denied=json.dumps([{
'IPProtocol': 'tcp',
}]),
),
}
PROJECTS = {
'foo': project_type.Project(project_id='foo'),
}
INSTANCES = {
'i1':
instance_type.Instance('i1',
parent=PROJECTS['foo'],
name='i1',
tags={'items': ['tag_i1']},
locations=['wl-redqueen1-a'],
data=("""{
"name": "i2",
"selfLink": "https://www.googleapis.com/compute/v1/projects/foo/zones/wl-redqueen1-a/instances/i1",
"tags": {"items": ["tag_i1"]}
}""")),
'i2':
instance_type.Instance('i2',
