def transform_gcp_firewall(fw_response: Resource) -> List[Dict]: """ Adjust the firewall response objects into a format that is easy to write to Neo4j. Also see _transform_fw_entry and _parse_port_string_to_rule(). :param fw_response: Firewall response object from the GCP API :return: List of transformed firewall rule objects. """ fw_list: List[Dict] = [] prefix = fw_response['id'] for fw in fw_response.get('items', []): fw_partial_uri = f"{prefix}/{fw['name']}" fw['id'] = fw_partial_uri fw['vpc_partial_uri'] = _parse_compute_full_uri_to_partial_uri( fw['network']) fw['transformed_allow_list'] = [] fw['transformed_deny_list'] = [] # Mark whether this FW is defined on a target service account. # In future we will need to ingest GCP IAM objects but for now we simply mark the presence of svc accounts here. fw['has_target_service_accounts'] = True if 'targetServiceAccounts' in fw else False for allow_rule in fw.get('allowed', []): transformed_allow_rules = _transform_fw_entry(allow_rule, fw_partial_uri, is_allow_rule=True) fw['transformed_allow_list'].extend(transformed_allow_rules) for deny_rule in fw.get('denied', []): transformed_deny_rules = _transform_fw_entry(deny_rule, fw_partial_uri, is_allow_rule=False) fw['transformed_deny_list'].extend(transformed_deny_rules) fw_list.append(fw) return fw_list
def _attach_target_tags(neo4j_session: neo4j.Session, fw: Resource, gcp_update_tag: int) -> None: """ Attach target tags to the firewall object :param neo4j_session: The neo4j session :param fw: The firewall object :param gcp_update_tag: The timestamp :return: Nothing """ query = """ MATCH (fw:GCPFirewall{id:{FwPartialUri}}) MERGE (t:GCPNetworkTag{id:{TagId}}) ON CREATE SET t.firstseen = timestamp(), t.tag_id = {TagId}, t.value = {TagValue} SET t.lastupdated = {gcp_update_tag} MERGE (fw)-[h:TARGET_TAG]->(t) ON CREATE SET h.firstseen = timestamp() SET h.lastupdated = {gcp_update_tag} """ for tag in fw.get('targetTags', []): tag_id = _create_gcp_network_tag_id(fw['vpc_partial_uri'], tag) neo4j_session.run( query, FwPartialUri=fw['id'], TagId=tag_id, TagValue=tag, gcp_update_tag=gcp_update_tag, )
def wait_for_operation(operation_client: discovery.Resource, operation: Dict[Text, Any]) -> None: """Waits for the completion of operation. This method retrieves operation resource and checks for its status. If the operation is not completed, then the operation is re-checked after `_WAIT_FOR_OPERATION_SLEEP_SECONDS` seconds. Args: operation_client: Client with methods for interacting with the operation APIs. The `build_service_client` method from `cloud_auth` module can be used to build the client. operation: Resource representing long running operation. Raises: Error: If the operation is not successfully completed. """ while True: request = operation_client.get(name=operation['name']) updated_operation = execute_request(request) if updated_operation.get('done'): logging.info('Operation "%s" successfully completed.', operation['name']) return if updated_operation.get('error'): logging.info('Operation "%s" failed to complete successfully.', operation['name']) raise Error( f'Operation {operation["name"]} not completed. Error Details - ' f'{updated_operation["error"]}') logging.info( 'Operation "%s" still in progress. Sleeping for ' '"%s" seconds before retrying.', operation['name'], _WAIT_FOR_OPERATION_SLEEP_SECONDS) time.sleep(_WAIT_FOR_OPERATION_SLEEP_SECONDS)
def _attach_gcp_bucket_labels(neo4j_session: neo4j.Session, bucket: Resource, gcp_update_tag: int) -> None: """ Attach GCP bucket labels to the bucket. :param neo4j_session: The neo4j session :param bucket: The GCP bucket object :param gcp_update_tag: The update tag for this sync :return: Nothing """ query = """ MERGE (l:Label:GCPBucketLabel{id: {BucketLabelId}}) ON CREATE SET l.firstseen = timestamp(), l.key = {Key} SET l.value = {Value}, l.lastupdated = {gcp_update_tag} WITH l MATCH (bucket:GCPBucket{id:{BucketId}}) MERGE (l)<-[r:LABELED]-(bucket) ON CREATE SET r.firstseen = timestamp() SET r.lastupdated = {gcp_update_tag} """ for (key, val) in bucket.get('labels', []): neo4j_session.run( query, BucketLabelId=f"GCPBucket_{key}", Key=key, Value=val, BucketId=bucket['id'], gcp_update_tag=gcp_update_tag, )
def transform_gcp_forwarding_rules(fwd_response: Resource) -> List[Dict]: """ Add additional fields to the forwarding rule object to make it easier to process in `load_gcp_forwarding_rules()`. :param fwd_response: The response object returned from compute.forwardRules.list() :return: A transformed fwd_response """ fwd_list: List[Dict] = [] prefix = fwd_response['id'] project_id = prefix.split('/')[1] for fwd in fwd_response.get('items', []): forwarding_rule: Dict[str, Any] = {} fwd_partial_uri = f"{prefix}/{fwd['name']}" forwarding_rule['id'] = fwd_partial_uri forwarding_rule['partial_uri'] = fwd_partial_uri forwarding_rule['project_id'] = project_id # Region looks like "https://www.googleapis.com/compute/v1/projects/{project}/regions/{region name}" region = fwd.get('region', None) forwarding_rule['region'] = region.split('/')[-1] if region else None forwarding_rule['ip_address'] = fwd.get('IPAddress', None) forwarding_rule['ip_protocol'] = fwd.get('IPProtocol', None) forwarding_rule['allow_global_access'] = fwd.get( 'allowGlobalAccess', None) forwarding_rule['load_balancing_scheme'] = fwd.get( 'loadBalancingScheme', None) forwarding_rule['name'] = fwd.get('name', None) forwarding_rule['port_range'] = fwd.get('portRange', None) forwarding_rule['ports'] = fwd.get('ports', None) forwarding_rule['self_link'] = fwd.get('selfLink', None) target = fwd.get('target', None) if target: forwarding_rule['target'] = _parse_compute_full_uri_to_partial_uri( target) else: forwarding_rule['target'] = None network = fwd.get('network', None) if network: forwarding_rule['network'] = network forwarding_rule[ 'network_partial_uri'] = _parse_compute_full_uri_to_partial_uri( network) subnetwork = fwd.get('subnetwork', None) if subnetwork: forwarding_rule['subnetwork'] = subnetwork forwarding_rule[ 'subnetwork_partial_uri'] = _parse_compute_full_uri_to_partial_uri( subnetwork) fwd_list.append(forwarding_rule) return fwd_list
def _attach_firewall_rules(neo4j_session: neo4j.Session, fw: Resource, gcp_update_tag: int) -> None: """ Attach the allow_rules to the Firewall object :param neo4j_session: The Neo4j session :param fw: The Firewall object :param gcp_update_tag: The timestamp :return: Nothing """ template = Template(""" MATCH (fw:GCPFirewall{id:{FwPartialUri}}) MERGE (rule:IpRule:IpPermissionInbound:GCPIpRule{id:{RuleId}}) ON CREATE SET rule.firstseen = timestamp(), rule.ruleid = {RuleId} SET rule.protocol = {Protocol}, rule.fromport = {FromPort}, rule.toport = {ToPort}, rule.lastupdated = {gcp_update_tag} MERGE (rng:IpRange{id:{Range}}) ON CREATE SET rng.firstseen = timestamp(), rng.range = {Range} SET rng.lastupdated = {gcp_update_tag} MERGE (rng)-[m:MEMBER_OF_IP_RULE]->(rule) ON CREATE SET m.firstseen = timestamp() SET m.lastupdated = {gcp_update_tag} MERGE (fw)<-[r:$fw_rule_relationship_label]-(rule) ON CREATE SET r.firstseen = timestamp() SET r.lastupdated = {gcp_update_tag} """) for list_type in 'transformed_allow_list', 'transformed_deny_list': if list_type == 'transformed_allow_list': label = "ALLOWED_BY" else: label = "DENIED_BY" for rule in fw[list_type]: # It is possible for sourceRanges to not be specified for this rule # If sourceRanges is not specified then the rule must specify sourceTags. # Since an IP range cannot have a tag applied to it, it is ok if we don't ingest this rule. for ip_range in fw.get('sourceRanges', []): neo4j_session.run( template.safe_substitute(fw_rule_relationship_label=label), FwPartialUri=fw['id'], RuleId=rule['ruleid'], Protocol=rule['protocol'], FromPort=rule.get('fromport'), ToPort=rule.get('toport'), Range=ip_range, gcp_update_tag=gcp_update_tag, )
def _attach_instance_tags(neo4j_session: neo4j.Session, instance: Resource, gcp_update_tag: int) -> None: """ Attach tags to GCP instance and to the VPCs that they are defined in. :param neo4j_session: The session :param instance: The instance object :param gcp_update_tag: The timestamp :return: Nothing """ query = """ MATCH (i:GCPInstance{id:{InstanceId}}) MERGE (t:GCPNetworkTag{id:{TagId}}) ON CREATE SET t.tag_id = {TagId}, t.value = {TagValue}, t.firstseen = timestamp() SET t.lastupdated = {gcp_update_tag} MERGE (i)-[h:TAGGED]->(t) ON CREATE SET h.firstseen = timestamp() SET h.lastupdated = {gcp_update_tag} WITH t MATCH (vpc:GCPVpc{id:{VpcPartialUri}}) MERGE (vpc)<-[d:DEFINED_IN]-(t) ON CREATE SET d.firstseen = timestamp() SET d.lastupdated = {gcp_update_tag} """ for tag in instance.get('tags', {}).get('items', []): for nic in instance.get('networkInterfaces', []): tag_id = _create_gcp_network_tag_id(nic['vpc_partial_uri'], tag) neo4j_session.run( query, InstanceId=instance['partial_uri'], TagId=tag_id, TagValue=tag, VpcPartialUri=nic['vpc_partial_uri'], gcp_update_tag=gcp_update_tag, )
def _attach_gcp_nics(neo4j_session: neo4j.Session, instance: Resource, gcp_update_tag: int) -> None: """ Attach GCP Network Interfaces to GCP Instances and GCP Subnets. Then, attach GCP Instances directly to VPCs. :param neo4j_session: The Neo4j session :param instance: The GCP instance :param gcp_update_tag: Timestamp to set the nodes :return: Nothing """ query = """ MATCH (i:GCPInstance{id:{InstanceId}}) MERGE (nic:GCPNetworkInterface:NetworkInterface{id:{NicId}}) ON CREATE SET nic.firstseen = timestamp(), nic.nic_id = {NicId} SET nic.private_ip = {NetworkIP}, nic.name = {NicName}, nic.lastupdated = {gcp_update_tag} MERGE (i)-[r:NETWORK_INTERFACE]->(nic) ON CREATE SET r.firstseen = timestamp() SET r.lastupdated = {gcp_update_tag} MERGE (subnet:GCPSubnet{id:{SubnetPartialUri}}) ON CREATE SET subnet.firstseen = timestamp(), subnet.partial_uri = {SubnetPartialUri} SET subnet.lastupdated = {gcp_update_tag} MERGE (nic)-[p:PART_OF_SUBNET]->(subnet) ON CREATE SET p.firstseen = timestamp() SET p.lastupdated = {gcp_update_tag} """ for nic in instance.get('networkInterfaces', []): # Make an ID for GCPNetworkInterface nodes because GCP doesn't define one but we need to uniquely identify them nic_id = f"{instance['partial_uri']}/networkinterfaces/{nic['name']}" neo4j_session.run( query, InstanceId=instance['partial_uri'], NicId=nic_id, NetworkIP=nic.get('networkIP'), NicName=nic['name'], gcp_update_tag=gcp_update_tag, SubnetPartialUri=nic['subnet_partial_uri'], ) _attach_gcp_nic_access_configs(neo4j_session, nic_id, nic, gcp_update_tag)
def _attach_gcp_nic_access_configs( neo4j_session: neo4j.Session, nic_id: str, nic: Resource, gcp_update_tag: int, ) -> None: """ Attach an access configuration to the GCP NIC. :param neo4j_session: The Neo4j session :param instance: The GCP instance :param gcp_update_tag: The timestamp to set updated nodes to :return: Nothing """ query = """ MATCH (nic{id:{NicId}}) MERGE (ac:GCPNicAccessConfig{id:{AccessConfigId}}) ON CREATE SET ac.firstseen = timestamp(), ac.access_config_id = {AccessConfigId} SET ac.type={Type}, ac.name = {Name}, ac.public_ip = {NatIP}, ac.set_public_ptr = {SetPublicPtr}, ac.public_ptr_domain_name = {PublicPtrDomainName}, ac.network_tier = {NetworkTier}, ac.lastupdated = {gcp_update_tag} MERGE (nic)-[r:RESOURCE]->(ac) ON CREATE SET r.firstseen = timestamp() SET r.lastupdated = {gcp_update_tag} """ for ac in nic.get('accessConfigs', []): # Make an ID for GCPNicAccessConfig nodes because GCP doesn't define one but we need to uniquely identify them access_config_id = f"{nic_id}/accessconfigs/{ac['type']}" neo4j_session.run( query, NicId=nic_id, AccessConfigId=access_config_id, Type=ac['type'], Name=ac['name'], NatIP=ac.get('natIP', None), SetPublicPtr=ac.get('setPublicPtr', None), PublicPtrDomainName=ac.get('publicPtrDomainName', None), NetworkTier=ac.get('networkTier', None), gcp_update_tag=gcp_update_tag, )
def _get_resource(self, collection: discovery.Resource, **kwargs) -> GcpResource: resp = collection.get(project=self.project, **kwargs).execute() logger.debug("Loaded %r", resp) return self.GcpResource(resp['name'], resp['selfLink'])
def _get_resource(self, collection: discovery.Resource, full_name): resource = collection.get(name=full_name).execute() logger.info('Loaded %s:\n%s', full_name, self._resource_pretty_format(resource)) return resource
def _get_resource(collection: discovery.Resource, full_name): resource = collection.get(name=full_name).execute() logger.debug("Loaded %r", resource) return resource
def _get_resource(self, collection: discovery.Resource, **kwargs) -> 'GcpResource': resp = collection.get(project=self.project, **kwargs).execute() logger.info('Loaded compute resource:\n%s', self.resource_pretty_format(resp)) return self.GcpResource(resp['name'], resp['selfLink'])