def _retrieve(self): """Runs the data collection. Returns: list: KE Cluster data. """ model_manager = self.service_config.model_manager scoped_session, data_access = model_manager.get(self.model_name) with scoped_session as session: ke_clusters = [] for cluster in data_access.scanner_iter( session, 'kubernetes_cluster'): proj = project.Project( project_id=cluster.parent.name, full_name=cluster.parent.full_name, ) ke_clusters.append( ke_cluster.KeCluster.from_json(proj, cluster.data)) # Retrieve the service config via a separate query because session # in the middle of yield_per() can not support simultaneous queries. with scoped_session as session: for cluster in ke_clusters: position = (cluster.full_name.find('kubernetes_cluster')) ke_cluster_type_name = (cluster.full_name[position:][:-1]) service_config = list( data_access.scanner_iter( session, 'kubernetes_service_config', parent_type_name=ke_cluster_type_name))[0] cluster.server_config = json.loads(service_config.data) return ke_clusters
def test_find_violations_inapplicable_resource(self): # rules are set on org 234 org = organization.Organization( '000', display_name='Organization 000', full_name='organization/000/', data='fake_org_data_000', ) proj = project.Project( '111', project_number=111, display_name='My project 111', parent=org, full_name='organization/000/project/111/', data='fake_project_data_111', ) rules_local_path = get_datafile_path(__file__, 'bigquery_test_rules_4.yaml') rules_engine = bqe.BigqueryRulesEngine(rules_local_path) rules_engine.build_rule_book() fake_bq_acls = create_list_of_bq_objects_from_data() actual_violations_list = [] for bqt in fake_bq_acls: violation = rules_engine.find_violations(proj, bqt) actual_violations_list.extend(violation) self.assertEqual([], actual_violations_list)
def _retrieve(self): """Retrieves the data for scanner. Returns: List[Resource]: resources to check for violations. Raises: ValueError: if resources have an unexpected type. """ resources = [] scoped_session, data_access = self.service_config.model_manager.get( self.model_name) with scoped_session as session: for resource_type in lre.SUPPORTED_LOCATION_RESOURCE_TYPES: for resource in data_access.scanner_iter( session, resource_type): if resource.parent.type != 'project': raise ValueError( 'Unexpected type of parent resource type: ' 'got %s, want project' % resource.parent.type) proj = project.Project( project_id=resource.parent.name, full_name=resource.parent.full_name, ) resources.append( resource_util.create_resource_from_json( resource_type, proj, resource.data)) return resources
def test_instance(self): """Test instance.Key.""" url_1 = ('https://www.googleapis.com/compute/v1/' 'projects/foo/zones/us-central1-a/instances/bar') obj_1 = instance.Instance('bar', parent=project.Project('foo'), locations=['us-central1-a'], name='bar') key_1 = key.Key(instance.KEY_OBJECT_KIND, { 'project_id': 'foo', 'zone': 'us-central1-a', 'name': 'bar' }) self.assertEqual(key_1, obj_1.key) self.assertEqual(key_1, instance.Key.from_url(url_1)) url_invalid_1 = ('https://www.googleapis.com/compute/v1/' 'zones/bar/instances/baz') url_invalid_2 = ('https://www.googleapis.com/compute/v1/' 'projects/foo/instances/bar') url_invalid_3 = ('https://www.googleapis.com/compute/v1/' 'projects/foo/zones/bar') self.assertRaises(ValueError, instance.Key.from_url, url_invalid_1) self.assertRaises(ValueError, instance.Key.from_url, url_invalid_2) self.assertRaises(ValueError, instance.Key.from_url, url_invalid_3)
def _retrieve(self): """Retrieve the network interfaces for vm instances. Return: list: A list that contains nested lists of per-instance InstanceNetworksInterface objects. """ model_manager = self.service_config.model_manager scoped_session, data_access = model_manager.get(self.model_name) with scoped_session as session: network_interfaces = [] for instance_from_data_model in data_access.scanner_iter( session, 'instance'): proj = project.Project( project_id=instance_from_data_model.parent.name, full_name=instance_from_data_model.parent.full_name, ) ins = instance.Instance.from_json( parent=proj, json_string=instance_from_data_model.data) network_interfaces.append(ins.create_network_interfaces()) if not network_interfaces: LOGGER.warn('No VM network interfaces found. Exiting.') return None, 0 return network_interfaces
def get_instance_networks_interfaces(self): """Get network info from a particular snapshot. Returns: list: A list that contains nested lists of per-instance InstanceNetworksInterface objects. """ model_manager = self.service_config.model_manager scoped_session, data_access = model_manager.get(self.model_name) instance_from_data_models = [] with scoped_session as session: for instance_from_data_model in data_access.scanner_iter( session, 'instance'): instance_from_data_models.append(instance_from_data_model) network_interfaces = [] for instance_from_data_model in instance_from_data_models: proj = project.Project( project_id=instance_from_data_model.parent.name, full_name=instance_from_data_model.parent.full_name) ins = instance.Instance.from_json( parent=proj, json_string=instance_from_data_model.data) network_interfaces.append(ins.create_network_interfaces()) if not network_interfaces: LOGGER.warning('No VM network interfaces found.') return [] return network_interfaces
def _retrieve(self): """Retrieves the data for scanner. Returns: list: BigQuery ACL data Raises: ValueError: if resources have an unexpected type. """ model_manager = self.service_config.model_manager scoped_session, data_access = model_manager.get(self.model_name) with scoped_session as session: bq_acl_data = [] policies = [] for policy in data_access.scanner_iter(session, 'dataset_policy'): policies.append(policy) for policy in policies: # dataset_policy are always in a dataset, which is always in a # project. dataset = policy.parent if dataset.type != 'dataset': raise ValueError( 'Unexpected type of dataset_policy parent: ' 'got %s, want dataset' % dataset.type ) if dataset.parent.type != 'project': raise ValueError( 'Unexpected type of dataset_policy grandparent: ' 'got %s, want project' % dataset.parent.type ) proj = project.Project( project_id=dataset.parent.name, full_name=dataset.parent.full_name, data=policy.data, ) # There is no functional use for project_id in this scanner, # other than to identify where the dataset comes from, # which can now be done with full_name. # In case you are tempted to get the project_id anyways, # do not use project_id = policy.parent.parent.name # which will cause db session conflict. # Instead, parse the project_id from the full_name. bq_acls = list(BigqueryAccessControls.from_json( project_id=None, dataset_id=dataset.name, full_name=policy.full_name, acls=policy.data)) for bq_acl in bq_acls: data = BigqueryAccessControlsData( parent_project=proj, bigquery_acl=bq_acl, ) bq_acl_data.append(data) return bq_acl_data
def _retrieve(self): """Retrieves the data for scanner. Returns: Dict[Resource, List[lien]]: mapping of a resource to the liens it contains. Raises: ValueError: if resources have an unexpected type. """ scoped_session, data_access = self.service_config.model_manager.get( self.model_name) with scoped_session as session: parent_resource_to_liens = {} # liens can only be defined on a project currently for project_resource in data_access.scanner_iter( session, 'project'): proj = project.Project( project_id=project_resource.name, full_name=project_resource.full_name, ) parent_resource_to_liens[proj] = [] for lien_resource in data_access.scanner_iter(session, 'lien'): parent_resource = lien_resource.parent if lien_resource.parent.type != 'project': raise ValueError( 'Unexpected type of lien resource parent: ' 'got %s, want project' % parent_resource.parent.type ) proj = project.Project( project_id=parent_resource.name, full_name=parent_resource.full_name, ) parent_resource_to_liens[proj].append(lien.Lien.from_json( parent=proj, json_string=lien_resource.data)) return parent_resource_to_liens
def setUp(self): """Set up parent GCP resources for tests.""" self.org_234 = organization.Organization( '234', display_name='Organization 234', full_name='organization/234/', data='fake_org_data_234') self.proj_1 = project.Project( 'proj-1', project_number=11223344, display_name='My project 1', parent=self.org_234, full_name='organization/234/project/proj-1/', data='fake_project_data_2341')
def _get_instances(self, parent_type_name): """Retrieves instances. Args: parent_type_name (str): The parent resource type and name to pull. Returns: list: Instance """ instances = [] with self.scoped_session as session: for instance in self.data_access.scanner_iter( session, 'instance', parent_type_name=parent_type_name): project = project_type.Project( project_id=instance.parent.name, full_name=instance.parent.full_name, ) instances.append( instance_type.Instance.from_json( parent=project, json_string=instance.data)) return instances
def setUp(self): """Set up.""" self.rule_index = 0 self.bqe = bqe self.bqe.LOGGER = mock.MagicMock() self.fake_timestamp = '12345' self.org = organization.Organization( '234', display_name='Organization 234', full_name='organization/234/', data='fake_org_data_234', ) self.project = project.Project( 'p1', project_number=11223344, display_name='My project 1', parent=self.org, full_name='organization/234/project/p1/', data='fake_project_data_2341', )
from google.cloud.forseti.common.gcp_type import project from google.cloud.forseti.common.gcp_type import resource_util from google.cloud.forseti.scanner.audit import location_rules_engine ORGANIZATION = organization.Organization( '234', display_name='Organization 234', full_name='organization/234/', data='fake_org_data_234', ) PROJECT = project.Project( 'p1', project_number=11223344, display_name='Project with lien', parent=ORGANIZATION, full_name='organization/234/project/p1/', data='fake_project_data_2341', ) _BUCKET_JSON = """{ "id": "p1-bucket1", "parent": "projects/p1", "location": "EUROPE-WEST1" } """ BUCKET = resource_util.create_resource_from_json( 'bucket', PROJECT, _BUCKET_JSON) _CLOUD_SQL_INSTANCE_JSON = """{
from google.cloud.forseti.common.gcp_type import project from google.cloud.forseti.common.gcp_type import resource from google.cloud.forseti.common.gcp_type import table from google.cloud.forseti.scanner.audit import retention_rules_engine as rre ORGANIZATION = organization.Organization( '123456', display_name='Default Organization', full_name='organization/123456/', data='fake_org_data_123456', ) PROJECT1 = project.Project( 'def-project-1', project_number=11223344, display_name='default project 1', parent=ORGANIZATION, full_name='organization/123456/project/def-project-1/', data='fake_project_data_11223344', ) PROJECT2 = project.Project( 'def-project-2', project_number=55667788, display_name='default project 2', parent=ORGANIZATION, full_name='organization/123456/project/def-project-2/', data='fake_project_data_55667788', ) PROJECT3 = project.Project( 'def-project-3',
from google.cloud.forseti.common.gcp_type import role from google.cloud.forseti.common.gcp_type import organization from google.cloud.forseti.common.gcp_type import project from google.cloud.forseti.scanner.audit import role_rules_engine as rre ORGANIZATION = organization.Organization( '123456', display_name='Default Organization', full_name='organization/123456/', data='fake_org_data_123456', ) PROJECT1 = project.Project( 'def-project-1', project_number=11223344, display_name='default project 1', parent=ORGANIZATION, full_name='organization/123456/project/def-project-1/', data='fake_project_data_11223344', ) PROJECT2 = project.Project( 'def-project-2', project_number=55667788, display_name='default project 2', parent=ORGANIZATION, full_name='organization/123456/project/def-project-2/', data='fake_project_data_55667788', ) PROJECT3 = project.Project( 'def-project-3',
), # Preempted by deny rule. 'preempted_deny': firewall_rule_type.FirewallRule( project_id='foo', firewall_rule_name='preempted_deny', firewall_rule_priority=1, firewall_rule_network='global/networks/default', firewall_rule_source_ranges=json.dumps(['preempted']), firewall_rule_denied=json.dumps([{ 'IPProtocol': 'tcp', }]), ), } PROJECTS = { 'foo': project_type.Project(project_id='foo'), } INSTANCES = { 'i1': instance_type.Instance('i1', parent=PROJECTS['foo'], name='i1', tags={'items': ['tag_i1']}, locations=['wl-redqueen1-a'], data=("""{ "name": "i2", "selfLink": "https://www.googleapis.com/compute/v1/projects/foo/zones/wl-redqueen1-a/instances/i1", "tags": {"items": ["tag_i1"]} }""")), 'i2': instance_type.Instance('i2',