def transform_gcp_firewall(fw_response: Resource) -> List[Dict]:
"""
Adjust the firewall response objects into a format that is easy to write to Neo4j.
Also see _transform_fw_entry and _parse_port_string_to_rule().
:param fw_response: Firewall response object from the GCP API
:return: List of transformed firewall rule objects.
"""
fw_list: List[Dict] = []
prefix = fw_response['id']
for fw in fw_response.get('items', []):
fw_partial_uri = f"{prefix}/{fw['name']}"
fw['id'] = fw_partial_uri
fw['vpc_partial_uri'] = _parse_compute_full_uri_to_partial_uri(
fw['network'])
fw['transformed_allow_list'] = []
fw['transformed_deny_list'] = []
# Mark whether this FW is defined on a target service account.
# In future we will need to ingest GCP IAM objects but for now we simply mark the presence of svc accounts here.
fw['has_target_service_accounts'] = True if 'targetServiceAccounts' in fw else False
for allow_rule in fw.get('allowed', []):
transformed_allow_rules = _transform_fw_entry(allow_rule,
fw_partial_uri,
is_allow_rule=True)
fw['transformed_allow_list'].extend(transformed_allow_rules)
for deny_rule in fw.get('denied', []):
transformed_deny_rules = _transform_fw_entry(deny_rule,
fw_partial_uri,
is_allow_rule=False)
fw['transformed_deny_list'].extend(transformed_deny_rules)
fw_list.append(fw)
return fw_list
def _attach_target_tags(neo4j_session: neo4j.Session, fw: Resource,
gcp_update_tag: int) -> None:
"""
Attach target tags to the firewall object
:param neo4j_session: The neo4j session
:param fw: The firewall object
:param gcp_update_tag: The timestamp
:return: Nothing
"""
query = """
MATCH (fw:GCPFirewall{id:{FwPartialUri}})
MERGE (t:GCPNetworkTag{id:{TagId}})
ON CREATE SET t.firstseen = timestamp(),
t.tag_id = {TagId},
t.value = {TagValue}
SET t.lastupdated = {gcp_update_tag}
MERGE (fw)-[h:TARGET_TAG]->(t)
ON CREATE SET h.firstseen = timestamp()
SET h.lastupdated = {gcp_update_tag}
"""
for tag in fw.get('targetTags', []):
tag_id = _create_gcp_network_tag_id(fw['vpc_partial_uri'], tag)
neo4j_session.run(
query,
FwPartialUri=fw['id'],
TagId=tag_id,
TagValue=tag,
gcp_update_tag=gcp_update_tag,
)
def wait_for_operation(operation_client: discovery.Resource,
operation: Dict[Text, Any]) -> None:
"""Waits for the completion of operation.
This method retrieves operation resource and checks for its status. If the
operation is not completed, then the operation is re-checked after
`_WAIT_FOR_OPERATION_SLEEP_SECONDS` seconds.
Args:
operation_client: Client with methods for interacting with the operation
APIs. The `build_service_client` method from `cloud_auth` module can be
used to build the client.
operation: Resource representing long running operation.
Raises:
Error: If the operation is not successfully completed.
"""
while True:
request = operation_client.get(name=operation['name'])
updated_operation = execute_request(request)
if updated_operation.get('done'):
logging.info('Operation "%s" successfully completed.',
operation['name'])
return
if updated_operation.get('error'):
logging.info('Operation "%s" failed to complete successfully.',
operation['name'])
raise Error(
f'Operation {operation["name"]} not completed. Error Details - '
f'{updated_operation["error"]}')
logging.info(
'Operation "%s" still in progress. Sleeping for '
'"%s" seconds before retrying.', operation['name'],
_WAIT_FOR_OPERATION_SLEEP_SECONDS)
time.sleep(_WAIT_FOR_OPERATION_SLEEP_SECONDS)
def _attach_gcp_bucket_labels(neo4j_session: neo4j.Session, bucket: Resource, gcp_update_tag: int) -> None:
"""
Attach GCP bucket labels to the bucket.
:param neo4j_session: The neo4j session
:param bucket: The GCP bucket object
:param gcp_update_tag: The update tag for this sync
:return: Nothing
"""
query = """
MERGE (l:Label:GCPBucketLabel{id: {BucketLabelId}})
ON CREATE SET l.firstseen = timestamp(),
l.key = {Key}
SET l.value = {Value},
l.lastupdated = {gcp_update_tag}
WITH l
MATCH (bucket:GCPBucket{id:{BucketId}})
MERGE (l)<-[r:LABELED]-(bucket)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {gcp_update_tag}
"""
for (key, val) in bucket.get('labels', []):
neo4j_session.run(
query,
BucketLabelId=f"GCPBucket_{key}",
Key=key,
Value=val,
BucketId=bucket['id'],
gcp_update_tag=gcp_update_tag,
)
def transform_gcp_forwarding_rules(fwd_response: Resource) -> List[Dict]:
"""
Add additional fields to the forwarding rule object to make it easier to process in `load_gcp_forwarding_rules()`.
:param fwd_response: The response object returned from compute.forwardRules.list()
:return: A transformed fwd_response
"""
fwd_list: List[Dict] = []
prefix = fwd_response['id']
project_id = prefix.split('/')[1]
for fwd in fwd_response.get('items', []):
forwarding_rule: Dict[str, Any] = {}
fwd_partial_uri = f"{prefix}/{fwd['name']}"
forwarding_rule['id'] = fwd_partial_uri
forwarding_rule['partial_uri'] = fwd_partial_uri
forwarding_rule['project_id'] = project_id
# Region looks like "https://www.googleapis.com/compute/v1/projects/{project}/regions/{region name}"
region = fwd.get('region', None)
forwarding_rule['region'] = region.split('/')[-1] if region else None
forwarding_rule['ip_address'] = fwd.get('IPAddress', None)
forwarding_rule['ip_protocol'] = fwd.get('IPProtocol', None)
forwarding_rule['allow_global_access'] = fwd.get(
'allowGlobalAccess', None)
forwarding_rule['load_balancing_scheme'] = fwd.get(
'loadBalancingScheme', None)
forwarding_rule['name'] = fwd.get('name', None)
forwarding_rule['port_range'] = fwd.get('portRange', None)
forwarding_rule['ports'] = fwd.get('ports', None)
forwarding_rule['self_link'] = fwd.get('selfLink', None)
target = fwd.get('target', None)
if target:
forwarding_rule['target'] = _parse_compute_full_uri_to_partial_uri(
target)
else:
forwarding_rule['target'] = None
network = fwd.get('network', None)
if network:
forwarding_rule['network'] = network
forwarding_rule[
'network_partial_uri'] = _parse_compute_full_uri_to_partial_uri(
network)
subnetwork = fwd.get('subnetwork', None)
if subnetwork:
forwarding_rule['subnetwork'] = subnetwork
forwarding_rule[
'subnetwork_partial_uri'] = _parse_compute_full_uri_to_partial_uri(
subnetwork)
fwd_list.append(forwarding_rule)
return fwd_list
def _attach_firewall_rules(neo4j_session: neo4j.Session, fw: Resource,
gcp_update_tag: int) -> None:
"""
Attach the allow_rules to the Firewall object
:param neo4j_session: The Neo4j session
:param fw: The Firewall object
:param gcp_update_tag: The timestamp
:return: Nothing
"""
template = Template("""
MATCH (fw:GCPFirewall{id:{FwPartialUri}})
MERGE (rule:IpRule:IpPermissionInbound:GCPIpRule{id:{RuleId}})
ON CREATE SET rule.firstseen = timestamp(),
rule.ruleid = {RuleId}
SET rule.protocol = {Protocol},
rule.fromport = {FromPort},
rule.toport = {ToPort},
rule.lastupdated = {gcp_update_tag}
MERGE (rng:IpRange{id:{Range}})
ON CREATE SET rng.firstseen = timestamp(),
rng.range = {Range}
SET rng.lastupdated = {gcp_update_tag}
MERGE (rng)-[m:MEMBER_OF_IP_RULE]->(rule)
ON CREATE SET m.firstseen = timestamp()
SET m.lastupdated = {gcp_update_tag}
MERGE (fw)<-[r:$fw_rule_relationship_label]-(rule)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {gcp_update_tag}
""")
for list_type in 'transformed_allow_list', 'transformed_deny_list':
if list_type == 'transformed_allow_list':
label = "ALLOWED_BY"
else:
label = "DENIED_BY"
for rule in fw[list_type]:
# It is possible for sourceRanges to not be specified for this rule
# If sourceRanges is not specified then the rule must specify sourceTags.
# Since an IP range cannot have a tag applied to it, it is ok if we don't ingest this rule.
for ip_range in fw.get('sourceRanges', []):
neo4j_session.run(
template.safe_substitute(fw_rule_relationship_label=label),
FwPartialUri=fw['id'],
RuleId=rule['ruleid'],
Protocol=rule['protocol'],
FromPort=rule.get('fromport'),
ToPort=rule.get('toport'),
Range=ip_range,
gcp_update_tag=gcp_update_tag,
)
def _attach_instance_tags(neo4j_session: neo4j.Session, instance: Resource,
gcp_update_tag: int) -> None:
"""
Attach tags to GCP instance and to the VPCs that they are defined in.
:param neo4j_session: The session
:param instance: The instance object
:param gcp_update_tag: The timestamp
:return: Nothing
"""
query = """
MATCH (i:GCPInstance{id:{InstanceId}})
MERGE (t:GCPNetworkTag{id:{TagId}})
ON CREATE SET t.tag_id = {TagId},
t.value = {TagValue},
t.firstseen = timestamp()
SET t.lastupdated = {gcp_update_tag}
MERGE (i)-[h:TAGGED]->(t)
ON CREATE SET h.firstseen = timestamp()
SET h.lastupdated = {gcp_update_tag}
WITH t
MATCH (vpc:GCPVpc{id:{VpcPartialUri}})
MERGE (vpc)<-[d:DEFINED_IN]-(t)
ON CREATE SET d.firstseen = timestamp()
SET d.lastupdated = {gcp_update_tag}
"""
for tag in instance.get('tags', {}).get('items', []):
for nic in instance.get('networkInterfaces', []):
tag_id = _create_gcp_network_tag_id(nic['vpc_partial_uri'], tag)
neo4j_session.run(
query,
InstanceId=instance['partial_uri'],
TagId=tag_id,
TagValue=tag,
VpcPartialUri=nic['vpc_partial_uri'],
gcp_update_tag=gcp_update_tag,
)
def _attach_gcp_nics(neo4j_session: neo4j.Session, instance: Resource,
gcp_update_tag: int) -> None:
"""
Attach GCP Network Interfaces to GCP Instances and GCP Subnets.
Then, attach GCP Instances directly to VPCs.
:param neo4j_session: The Neo4j session
:param instance: The GCP instance
:param gcp_update_tag: Timestamp to set the nodes
:return: Nothing
"""
query = """
MATCH (i:GCPInstance{id:{InstanceId}})
MERGE (nic:GCPNetworkInterface:NetworkInterface{id:{NicId}})
ON CREATE SET nic.firstseen = timestamp(),
nic.nic_id = {NicId}
SET nic.private_ip = {NetworkIP},
nic.name = {NicName},
nic.lastupdated = {gcp_update_tag}
MERGE (i)-[r:NETWORK_INTERFACE]->(nic)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {gcp_update_tag}
MERGE (subnet:GCPSubnet{id:{SubnetPartialUri}})
ON CREATE SET subnet.firstseen = timestamp(),
subnet.partial_uri = {SubnetPartialUri}
SET subnet.lastupdated = {gcp_update_tag}
MERGE (nic)-[p:PART_OF_SUBNET]->(subnet)
ON CREATE SET p.firstseen = timestamp()
SET p.lastupdated = {gcp_update_tag}
"""
for nic in instance.get('networkInterfaces', []):
# Make an ID for GCPNetworkInterface nodes because GCP doesn't define one but we need to uniquely identify them
nic_id = f"{instance['partial_uri']}/networkinterfaces/{nic['name']}"
neo4j_session.run(
query,
InstanceId=instance['partial_uri'],
NicId=nic_id,
NetworkIP=nic.get('networkIP'),
NicName=nic['name'],
gcp_update_tag=gcp_update_tag,
SubnetPartialUri=nic['subnet_partial_uri'],
)
_attach_gcp_nic_access_configs(neo4j_session, nic_id, nic,
gcp_update_tag)
def _attach_gcp_nic_access_configs(
neo4j_session: neo4j.Session,
nic_id: str,
nic: Resource,
gcp_update_tag: int,
) -> None:
"""
Attach an access configuration to the GCP NIC.
:param neo4j_session: The Neo4j session
:param instance: The GCP instance
:param gcp_update_tag: The timestamp to set updated nodes to
:return: Nothing
"""
query = """
MATCH (nic{id:{NicId}})
MERGE (ac:GCPNicAccessConfig{id:{AccessConfigId}})
ON CREATE SET ac.firstseen = timestamp(),
ac.access_config_id = {AccessConfigId}
SET ac.type={Type},
ac.name = {Name},
ac.public_ip = {NatIP},
ac.set_public_ptr = {SetPublicPtr},
ac.public_ptr_domain_name = {PublicPtrDomainName},
ac.network_tier = {NetworkTier},
ac.lastupdated = {gcp_update_tag}
MERGE (nic)-[r:RESOURCE]->(ac)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {gcp_update_tag}
"""
for ac in nic.get('accessConfigs', []):
# Make an ID for GCPNicAccessConfig nodes because GCP doesn't define one but we need to uniquely identify them
access_config_id = f"{nic_id}/accessconfigs/{ac['type']}"
neo4j_session.run(
query,
NicId=nic_id,
AccessConfigId=access_config_id,
Type=ac['type'],
Name=ac['name'],
NatIP=ac.get('natIP', None),
SetPublicPtr=ac.get('setPublicPtr', None),
PublicPtrDomainName=ac.get('publicPtrDomainName', None),
NetworkTier=ac.get('networkTier', None),
gcp_update_tag=gcp_update_tag,
)
def _get_resource(self, collection: discovery.Resource,
**kwargs) -> GcpResource:
resp = collection.get(project=self.project, **kwargs).execute()
logger.debug("Loaded %r", resp)
return self.GcpResource(resp['name'], resp['selfLink'])
def _get_resource(self, collection: discovery.Resource, full_name):
resource = collection.get(name=full_name).execute()
logger.info('Loaded %s:\n%s', full_name,
self._resource_pretty_format(resource))
return resource
def _get_resource(collection: discovery.Resource, full_name):
resource = collection.get(name=full_name).execute()
logger.debug("Loaded %r", resource)
return resource
def _get_resource(self, collection: discovery.Resource,
**kwargs) -> 'GcpResource':
resp = collection.get(project=self.project, **kwargs).execute()
logger.info('Loaded compute resource:\n%s',
self.resource_pretty_format(resp))
return self.GcpResource(resp['name'], resp['selfLink'])
