def _load_db_threat_detection_policies(
neo4j_session: neo4j.Session, threat_detection_policies: List[Dict], update_tag: int,
) -> None:
"""
Ingest threat detection policy into neo4j.
"""
ingest_threat_detection_policies = """
UNWIND {threat_detection_policies_list} as tdp
MERGE (policy:AzureDatabaseThreatDetectionPolicy{id: tdp.id})
ON CREATE SET policy.firstseen = timestamp(),
policy.location = tdp.location
SET policy.name = tdp.name,
policy.location = tdp.location,
policy.kind = tdp.kind,
policy.emailadmins = tdp.email_account_admins,
policy.emailaddresses = tdp.email_addresses,
policy.retentiondays = tdp.retention_days,
policy.state = tdp.state,
policy.storageendpoint = tdp.storage_endpoint,
policy.useserverdefault = tdp.use_server_default,
policy.disabledalerts = tdp.disabled_alerts,
policy.lastupdated = {azure_update_tag}
WITH policy, tdp
MATCH (d:AzureSQLDatabase{id: tdp.database_id})
MERGE (d)-[r:CONTAINS]->(policy)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {azure_update_tag}
"""
neo4j_session.run(
ingest_threat_detection_policies,
threat_detection_policies_list=threat_detection_policies,
azure_update_tag=update_tag,
)
def load_load_balancer_v2_target_groups(
neo4j_session: neo4j.Session, load_balancer_id: str, target_groups: List[Dict], current_aws_account_id: str,
update_tag: int,
) -> None:
ingest_instances = """
MATCH (elbv2:LoadBalancerV2{id: {ID}}), (instance:EC2Instance{instanceid: {INSTANCE_ID}})
MERGE (elbv2)-[r:EXPOSE]->(instance)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {update_tag}
WITH instance
MATCH (aa:AWSAccount{id: {AWS_ACCOUNT_ID}})
MERGE (aa)-[r:RESOURCE]->(instance)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {update_tag}
"""
for target_group in target_groups:
if not target_group['TargetType'] == 'instance':
# Only working on EC2 Instances now. TODO: Add IP & Lambda EXPOSE.
continue
for instance in target_group["Targets"]:
neo4j_session.run(
ingest_instances,
ID=load_balancer_id,
INSTANCE_ID=instance,
AWS_ACCOUNT_ID=current_aws_account_id,
update_tag=update_tag,
)
def load_github_owners(neo4j_session: neo4j.Session, update_tag: int,
repo_owners: List[Dict]) -> None:
"""
Ingest the relationships for repo owners
:param neo4j_session: Neo4J session object for server communication
:param update_tag: Timestamp used to determine data freshness
:param repo_owners: list of owner to repo mappings
:return: Nothing
"""
for owner in repo_owners:
ingest_owner_template = Template("""
MERGE (user:$account_type{id: {Id}})
ON CREATE SET user.firstseen = timestamp()
SET user.username = {UserName},
user.lastupdated = {UpdateTag}
WITH user
MATCH (repo:GitHubRepository{id: {RepoId}})
MERGE (user)<-[r:OWNER]-(repo)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {UpdateTag}""")
account_type = {
'User': "******",
'Organization': "GitHubOrganization"
}
neo4j_session.run(
ingest_owner_template.safe_substitute(
account_type=account_type[owner['type']]),
Id=owner['owner_id'],
UserName=owner['owner'],
RepoId=owner['repo_id'],
UpdateTag=update_tag,
)
def load_organization_users(
neo4j_session: neo4j.Session, user_data: List[Dict], org_data: Dict,
update_tag: int,
) -> None:
query = """
MERGE (org:GitHubOrganization{id: {OrgUrl}})
ON CREATE SET org.firstseen = timestamp()
SET org.username = {OrgLogin},
org.lastupdated = {UpdateTag}
WITH org
UNWIND {UserData} as user
MERGE (u:GitHubUser{id: user.node.url})
ON CREATE SET u.firstseen = timestamp()
SET u.fullname = user.node.name,
u.username = user.node.login,
u.has_2fa_enabled = user.hasTwoFactorEnabled,
u.role = user.role,
u.is_site_admin = user.node.isSiteAdmin,
u.email = user.node.email,
u.company = user.node.company,
u.lastupdated = {UpdateTag}
MERGE (u)-[r:MEMBER_OF]->(org)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {UpdateTag}
"""
neo4j_session.run(
query,
OrgUrl=org_data['url'],
OrgLogin=org_data['login'],
UserData=user_data,
UpdateTag=update_tag,
)
def _load_kms_key_grants(neo4j_session: neo4j.Session, grants_list: List[Dict],
update_tag: int) -> None:
"""
Ingest KMS Key Grants into neo4j.
"""
ingest_grants = """
UNWIND {grants} AS grant
MERGE (g:KMSGrant{id: grant.GrantId})
ON CREATE SET g.firstseen = timestamp(), g.granteeprincipal = grant.GranteePrincipal,
g.creationdate = grant.CreationDate
SET g.name = grant.GrantName, g.lastupdated = {UpdateTag}
WITH g, grant
MATCH (kmskey:KMSKey{id: grant.KeyId})
MERGE (g)-[r:APPLIED_ON]->(kmskey)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {UpdateTag}
"""
# neo4j does not accept datetime objects and values. This loop is used to convert
# these values to string.
for grant in grants_list:
grant['CreationDate'] = str(grant['CreationDate'])
neo4j_session.run(
ingest_grants,
grants=grants_list,
UpdateTag=update_tag,
)
def _load_okta_group_to_aws_roles(
neo4j_session: neo4j.Session,
group_to_role: List[Dict],
okta_update_tag: int,
) -> None:
"""
Add the ALLOWED_BY relationship between OktaGroups and the AWSRoles they enable
:param neo4j_session: session with the Neo4j server
:param group_to_role: the mapping between OktaGroups and the AWSRoles they allow access to
:param okta_update_tag: The timestamp value to set our new Neo4j resources with
:return: Nothing
"""
ingest_statement = """
UNWIND {GROUP_TO_ROLE} as app_data
MATCH (role:AWSRole{arn: app_data.role})
MATCH (group:OktaGroup{id: app_data.groupid})
MERGE (role)<-[r:ALLOWED_BY]-(group)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {okta_update_tag}
"""
neo4j_session.run(
ingest_statement,
GROUP_TO_ROLE=group_to_role,
okta_update_tag=okta_update_tag,
)
def _load_apigateway_resources(
neo4j_session: neo4j.Session,
resources: List,
update_tag: int,
) -> None:
"""
Ingest the API Gateway Resource details into neo4j.
"""
ingest_resources = """
UNWIND {resources_list} AS res
MERGE (s:APIGatewayResource{id: res.id})
ON CREATE SET s.firstseen = timestamp()
SET s.path = res.path,
s.pathpart = res.pathPart,
s.parentid = res.parentId,
s.lastupdated ={UpdateTag}
WITH s, res
MATCH (rest_api:APIGatewayRestAPI{id: res.apiId})
MERGE (rest_api)-[r:RESOURCE]->(s)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {UpdateTag}
"""
neo4j_session.run(
ingest_resources,
resources_list=resources,
UpdateTag=update_tag,
)
def load_ec2_key_pairs(
neo4j_session: neo4j.Session,
data: List[Dict],
region: str,
current_aws_account_id: str,
update_tag: int,
) -> None:
ingest_key_pair = """
MERGE (keypair:KeyPair:EC2KeyPair{arn: {ARN}, id: {ARN}})
ON CREATE SET keypair.firstseen = timestamp()
SET keypair.keyname = {KeyName}, keypair.keyfingerprint = {KeyFingerprint}, keypair.region = {Region},
keypair.lastupdated = {update_tag}
WITH keypair
MATCH (aa:AWSAccount{id: {AWS_ACCOUNT_ID}})
MERGE (aa)-[r:RESOURCE]->(keypair)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {update_tag}
"""
for key_pair in data:
key_name = key_pair["KeyName"]
key_fingerprint = key_pair.get("KeyFingerprint")
key_pair_arn = f'arn:aws:ec2:{region}:{current_aws_account_id}:key-pair/{key_name}'
neo4j_session.run(
ingest_key_pair,
ARN=key_pair_arn,
KeyName=key_name,
KeyFingerprint=key_fingerprint,
AWS_ACCOUNT_ID=current_aws_account_id,
Region=region,
update_tag=update_tag,
)
def load_ecr_repositories(
neo4j_session: neo4j.Session,
repos: List[Dict],
region: str,
current_aws_account_id: str,
aws_update_tag: int,
) -> None:
query = """
UNWIND {Repositories} as ecr_repo
MERGE (repo:ECRRepository{id: ecr_repo.repositoryArn})
ON CREATE SET repo.firstseen = timestamp(),
repo.arn = ecr_repo.repositoryArn,
repo.name = ecr_repo.repositoryName,
repo.region = {Region},
repo.created_at = ecr_repo.createdAt
SET repo.lastupdated = {aws_update_tag},
repo.uri = ecr_repo.repositoryUri
WITH repo
MATCH (owner:AWSAccount{id: {AWS_ACCOUNT_ID}})
MERGE (owner)-[r:RESOURCE]->(repo)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {aws_update_tag}
"""
logger.debug("Loading ECR repositories for region '%s' into graph.",
region)
neo4j_session.run(
query,
Repositories=repos,
Region=region,
aws_update_tag=aws_update_tag,
AWS_ACCOUNT_ID=current_aws_account_id,
).consume() # See issue #440
def load_s3_buckets(neo4j_session: neo4j.Session, data: Dict,
current_aws_account_id: str, aws_update_tag: int) -> None:
ingest_bucket = """
MERGE (bucket:S3Bucket{id:{BucketName}})
ON CREATE SET bucket.firstseen = timestamp(), bucket.creationdate = {CreationDate}
SET bucket.name = {BucketName}, bucket.region = {BucketRegion}, bucket.arn = {Arn},
bucket.lastupdated = {aws_update_tag}
WITH bucket
MATCH (owner:AWSAccount{id: {AWS_ACCOUNT_ID}})
MERGE (owner)-[r:RESOURCE]->(bucket)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {aws_update_tag}
"""
# The owner data returned by the API maps to the aws account nickname and not the IAM user
# there doesn't seem to be a way to retreive the mapping but we can get the current context account
# so we map to that directly
for bucket in data["Buckets"]:
arn = "arn:aws:s3:::" + bucket["Name"]
neo4j_session.run(
ingest_bucket,
BucketName=bucket["Name"],
BucketRegion=bucket["Region"],
Arn=arn,
CreationDate=str(bucket["CreationDate"]),
AWS_ACCOUNT_ID=current_aws_account_id,
aws_update_tag=aws_update_tag,
)
def _load_okta_group_members(
neo4j_session: neo4j.Session,
group_id: str,
member_list: List[str],
okta_update_tag: int,
) -> None:
"""
Add group membership data into the graph
:param neo4j_session: session with the Neo4j server
:param group_id: group id to map
:param member_list: group members
:param okta_update_tag: The timestamp value to set our new Neo4j resources with
:return: Nothing
"""
ingest = """
MATCH (group:OktaGroup{id: {GROUP_ID}})
WITH group
UNWIND {MEMBER_LIST} as member_id
MATCH (user:OktaUser{id: member_id})
WITH group, user
MERGE (user)-[r:MEMBER_OF_OKTA_GROUP]->(group)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {okta_update_tag}
"""
neo4j_session.run(
ingest,
GROUP_ID=group_id,
MEMBER_LIST=member_list,
okta_update_tag=okta_update_tag,
)
def _load_s3_acls(neo4j_session: neo4j.Session, acls: Dict,
aws_account_id: str, update_tag: int) -> None:
"""
Ingest S3 ACL into neo4j.
"""
ingest_acls = """
UNWIND {acls} AS acl
MERGE (a:S3Acl{id: acl.id})
ON CREATE SET a.firstseen = timestamp(), a.owner = acl.owner, a.ownerid = acl.ownerid, a.type = acl.type,
a.displayname = acl.displayname, a.granteeid = acl.granteeid, a.uri = acl.uri, a.permission = acl.permission
SET a.lastupdated = {UpdateTag}
WITH a,acl MATCH (s3:S3Bucket{id: acl.bucket})
MERGE (a)-[r:APPLIES_TO]->(s3)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {UpdateTag}
"""
neo4j_session.run(
ingest_acls,
acls=acls,
UpdateTag=update_tag,
)
# implement the acl permission
# https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#permissions
run_analysis_job(
'aws_s3acl_analysis.json',
neo4j_session,
{'AWS_ID': aws_account_id},
)
def _load_transparent_data_encryptions(
neo4j_session: neo4j.Session, encryptions_list: List[Dict], update_tag: int,
) -> None:
"""
Ingest transparent data encryptions into neo4j.
"""
ingest_data_encryptions = """
UNWIND {transparent_data_encryptions_list} as e
MERGE (tae:AzureTransparentDataEncryption{id: e.id})
ON CREATE SET tae.firstseen = timestamp(),
tae.location = e.location
SET tae.name = e.name,
tae.status = e.status,
tae.lastupdated = {azure_update_tag}
WITH tae, e
MATCH (d:AzureSQLDatabase{id: e.database_id})
MERGE (d)-[r:CONTAINS]->(tae)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {azure_update_tag}
"""
neo4j_session.run(
ingest_data_encryptions,
transparent_data_encryptions_list=encryptions_list,
azure_update_tag=update_tag,
)
def _load_restore_points(
neo4j_session: neo4j.Session, restore_points: List[Dict], update_tag: int,
) -> None:
"""
Ingest restore points into neo4j.
"""
ingest_restore_points = """
UNWIND {restore_points_list} as rp
MERGE (point:AzureRestorePoint{id: rp.id})
ON CREATE SET point.firstseen = timestamp(),
point.location = rp.location
SET point.name = rp.name,
point.restoredate = rp.earliest_restore_date,
point.restorepointtype = rp.restore_point_type,
point.creationdate = rp.restore_point_creation_date,
point.lastupdated = {azure_update_tag}
WITH point, rp
MATCH (d:AzureSQLDatabase{id: rp.database_id})
MERGE (d)-[r:CONTAINS]->(point)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {azure_update_tag}
"""
neo4j_session.run(
ingest_restore_points,
restore_points_list=restore_points,
azure_update_tag=update_tag,
)
def load_network_interface_instance_relations(
neo4j_session: neo4j.Session,
instance_associations: List[Dict],
region: str,
aws_account_id: str,
update_tag: int,
) -> None:
"""
Creates (:EC2Instance)-[:NETWORK_INTERFACE]->(:NetworkInterface)
"""
ingest_network_interface_instance_relations = """
UNWIND {instance_associations} AS instance_association
MATCH (netinf:NetworkInterface{id: instance_association.netinf_id}),
(instance:EC2Instance{id: instance_association.instance_id})
MERGE (instance)-[r:NETWORK_INTERFACE]->(netinf)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {update_tag}
"""
logger.debug("Attaching %d EC2 instances to network interfaces in %s.",
len(instance_associations), region)
neo4j_session.run(
ingest_network_interface_instance_relations,
instance_associations=instance_associations,
update_tag=update_tag,
region=region,
aws_account_id=aws_account_id,
)
def load_snapshots(
neo4j_session: neo4j.Session, data: List[Dict], region: str, current_aws_account_id: str, update_tag: int,
) -> None:
ingest_snapshots = """
UNWIND {snapshots_list} as snapshot
MERGE (s:EBSSnapshot{id: snapshot.SnapshotId})
ON CREATE SET s.firstseen = timestamp()
SET s.lastupdated = {update_tag}, s.description = snapshot.Description, s.encrypted = snapshot.Encrypted,
s.progress = snapshot.Progress, s.starttime = snapshot.StartTime, s.state = snapshot.State,
s.statemessage = snapshot.StateMessage, s.volumeid = snapshot.VolumeId, s.volumesize = snapshot.VolumeSize,
s.outpostarn = snapshot.OutpostArn, s.dataencryptionkeyid = snapshot.DataEncryptionKeyId,
s.kmskeyid = snapshot.KmsKeyId, s.region={Region}
WITH s
MATCH (aa:AWSAccount{id: {AWS_ACCOUNT_ID}})
MERGE (aa)-[r:RESOURCE]->(s)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {update_tag}
"""
for snapshot in data:
snapshot['StartTime'] = str(snapshot['StartTime'])
neo4j_session.run(
ingest_snapshots,
snapshots_list=data,
AWS_ACCOUNT_ID=current_aws_account_id,
Region=region,
update_tag=update_tag,
)
def load_network_interface_elbv2_relations(
neo4j_session: neo4j.Session,
elb_associations_v2: List[Dict],
region: str,
aws_account_id: str,
update_tag: int,
) -> None:
"""
Creates (:LoadBalancerV2)-[:NETWORK_INTERFACE]->(:NetworkInterface)
"""
ingest_network_interface_elb2_relations = """
UNWIND {elb_associations} AS elb_association
MATCH (netinf:NetworkInterface{id: elb_association.netinf_id}),
(elb:LoadBalancerV2{id: elb_association.elb_id})
MERGE (elb)-[r:NETWORK_INTERFACE]->(netinf)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {update_tag}
"""
logger.debug("Attaching %d ELB V2s to network interfaces in %s.",
len(elb_associations_v2), region)
neo4j_session.run(
ingest_network_interface_elb2_relations,
elb_associations=elb_associations_v2,
update_tag=update_tag,
region=region,
aws_account_id=aws_account_id,
)
def load_snapshot_volume_relations(
neo4j_session: neo4j.Session, data: List[Dict], current_aws_account_id: str, update_tag: int,
) -> None:
ingest_volumes = """
UNWIND {snapshot_volumes_list} as volume
MERGE (v:EBSVolume{id: volume.VolumeId})
ON CREATE SET v.firstseen = timestamp()
SET v.lastupdated = {update_tag}
WITH v, volume
MATCH (aa:AWSAccount{id: {AWS_ACCOUNT_ID}})
MERGE (aa)-[r:RESOURCE]->(v)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {update_tag}
WITH v, volume
MATCH (s:EBSSnapshot{id: volume.SnapshotId})
MERGE (s)-[r:CREATED_FROM]->(v)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {update_tag}
"""
neo4j_session.run(
ingest_volumes,
snapshot_volumes_list=data,
AWS_ACCOUNT_ID=current_aws_account_id,
update_tag=update_tag,
)
def _load_apigateway_certificates(
neo4j_session: neo4j.Session,
certificates: List,
update_tag: int,
) -> None:
"""
Ingest the API Gateway Client Certificate details into neo4j.
"""
ingest_certificates = """
UNWIND {certificates_list} as certificate
MERGE (c:APIGatewayClientCertificate{id: certificate.clientCertificateId})
ON CREATE SET c.firstseen = timestamp(), c.createddate = certificate.createdDate
SET c.lastupdated = {UpdateTag}, c.expirationdate = certificate.expirationDate
WITH c, certificate
MATCH (stage:APIGatewayStage{id: certificate.stageArn})
MERGE (stage)-[r:HAS_CERTIFICATE]->(c)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {UpdateTag}
"""
# neo4j does not accept datetime objects and values. This loop is used to convert
# these values to string.
for certificate in certificates:
certificate['createdDate'] = str(certificate['createdDate'])
certificate['expirationDate'] = str(certificate.get('expirationDate'))
certificate['stageArn'] = "arn:aws:apigateway:::" + certificate[
'apiId'] + "/" + certificate['stageName']
neo4j_session.run(
ingest_certificates,
certificates_list=certificates,
UpdateTag=update_tag,
)
def _load_user_role(neo4j_session: neo4j.Session, user_id: str,
roles_data: List[Dict], okta_update_tag: int) -> None:
ingest = """
MATCH (user:OktaUser{id: {USER_ID}})<-[:RESOURCE]-(org:OktaOrganization)
WITH user,org
UNWIND {ROLES_DATA} as role_data
MERGE (role_node:OktaAdministrationRole{id: role_data.type})
ON CREATE SET role_node.type = role_data.type, role_node.firstseen = timestamp()
SET role_node.label = role_data.label, role_node.lastupdated = {okta_update_tag}
WITH user, role_node, org
MERGE (user)-[r:MEMBER_OF_OKTA_ROLE]->(role_node)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {okta_update_tag}
WITH role_node, org
MERGE (org)-[r2:RESOURCE]->(role_node)
ON CREATE SET r2.firstseen = timestamp()
SET r2.lastupdated = {okta_update_tag}
"""
neo4j_session.run(
ingest,
USER_ID=user_id,
ROLES_DATA=roles_data,
okta_update_tag=okta_update_tag,
)
def load_load_balancer_listeners(
neo4j_session: neo4j.Session,
load_balancer_id: str,
listener_data: List[Dict],
update_tag: int,
) -> None:
ingest_listener = """
MATCH (elb:LoadBalancer{id: {LoadBalancerId}})
WITH elb
UNWIND {Listeners} as data
MERGE (l:Endpoint:ELBListener{id: elb.id + toString(data.Listener.LoadBalancerPort) +
toString(data.Listener.Protocol)})
ON CREATE SET l.port = data.Listener.LoadBalancerPort, l.protocol = data.Listener.Protocol,
l.firstseen = timestamp()
SET l.instance_port = data.Listener.InstancePort, l.instance_protocol = data.Listener.InstanceProtocol,
l.policy_names = data.PolicyNames,
l.lastupdated = {update_tag}
WITH l, elb
MERGE (elb)-[r:ELB_LISTENER]->(l)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {update_tag}
"""
neo4j_session.run(
ingest_listener,
LoadBalancerId=load_balancer_id,
Listeners=listener_data,
update_tag=update_tag,
)
def load_dynamodb_tables(
neo4j_session: neo4j.Session, data: List[Dict], region: str, current_aws_account_id: str,
aws_update_tag: int,
) -> None:
ingest_table = """
MERGE (table:DynamoDBTable{id: {Arn}})
ON CREATE SET table.firstseen = timestamp(), table.arn = {Arn}, table.name = {TableName},
table.region = {Region}
SET table.lastupdated = {aws_update_tag}, table.rows = {Rows}, table.size = {Size},
table.provisioned_throughput_read_capacity_units = {ProvisionedThroughputReadCapacityUnits},
table.provisioned_throughput_write_capacity_units = {ProvisionedThroughputWriteCapacityUnits}
WITH table
MATCH (owner:AWSAccount{id: {AWS_ACCOUNT_ID}})
MERGE (owner)-[r:RESOURCE]->(table)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {aws_update_tag}
"""
for table in data:
neo4j_session.run(
ingest_table,
Arn=table['Table']['TableArn'],
Region=region,
ProvisionedThroughputReadCapacityUnits=table['Table']['ProvisionedThroughput']['ReadCapacityUnits'],
ProvisionedThroughputWriteCapacityUnits=table['Table']['ProvisionedThroughput']['WriteCapacityUnits'],
Size=table['Table']['TableSizeBytes'],
TableName=table['Table']['TableName'],
Rows=table['Table']['ItemCount'],
AWS_ACCOUNT_ID=current_aws_account_id,
aws_update_tag=aws_update_tag,
)
load_gsi(neo4j_session, table, region, current_aws_account_id, aws_update_tag)
def _load_lambda_event_source_mappings(
neo4j_session: neo4j.Session,
lambda_event_source_mappings: List[Dict],
update_tag: int,
) -> None:
ingest_esms = """
UNWIND {esm_list} AS esm
MERGE (e:AWSLambdaEventSourceMapping{id: esm.UUID})
ON CREATE SET e.firstseen = timestamp()
SET e.batchsize = esm.BatchSize,
e.startingposition = esm.StartingPosition,
e.startingpositiontimestamp = esm.StartingPositionTimestamp,
e.parallelizationfactor = esm.ParallelizationFactor,
e.maximumbatchingwindowinseconds = esm.MaximumBatchingWindowInSeconds,
e.eventsourcearn = esm.EventSourceArn,
e.lastmodified = esm.LastModified,
e.lastprocessingresult = esm.LastProcessingResult,
e.state = esm.State,
e.maximumrecordage = esm.MaximumRecordAgeInSeconds,
e.bisectbatchonfunctionerror = esm.BisectBatchOnFunctionError,
e.maximumretryattempts = esm.MaximumRetryAttempts,
e.tumblingwindowinseconds = esm.TumblingWindowInSeconds,
e.lastupdated = {aws_update_tag}
WITH e, esm
MATCH (lambda:AWSLambda{id: esm.FunctionArn})
MERGE (lambda)-[r:RESOURCE]->(e)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {aws_update_tag}
"""
neo4j_session.run(
ingest_esms,
esm_list=lambda_event_source_mappings,
aws_update_tag=update_tag,
)
def load_gsi(
neo4j_session: neo4j.Session, table: Dict, region: str, current_aws_account_id: str,
aws_update_tag: int,
) -> None:
ingest_gsi = """
MERGE (gsi:DynamoDBGlobalSecondaryIndex{id: {Arn}})
ON CREATE SET gsi.firstseen = timestamp(), gsi.arn = {Arn}, gsi.name = {GSIName},
gsi.region = {Region}
SET gsi.lastupdated = {aws_update_tag},
gsi.provisioned_throughput_read_capacity_units = {ProvisionedThroughputReadCapacityUnits},
gsi.provisioned_throughput_write_capacity_units = {ProvisionedThroughputWriteCapacityUnits}
WITH gsi
MATCH (table:DynamoDBTable{arn: {TableArn}})
MERGE (table)-[r:GLOBAL_SECONDARY_INDEX]->(gsi)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {aws_update_tag}
"""
for gsi in table['Table'].get('GlobalSecondaryIndexes', []):
neo4j_session.run(
ingest_gsi,
TableArn=table['Table']['TableArn'],
Arn=gsi['IndexArn'],
Region=region,
ProvisionedThroughputReadCapacityUnits=gsi['ProvisionedThroughput']['ReadCapacityUnits'],
ProvisionedThroughputWriteCapacityUnits=gsi['ProvisionedThroughput']['WriteCapacityUnits'],
GSIName=gsi['IndexName'],
AWS_ACCOUNT_ID=current_aws_account_id,
aws_update_tag=aws_update_tag,
)
def load_tags(
neo4j_session: neo4j.Session,
tag_data: Dict,
resource_type: str,
region: str,
aws_update_tag: int,
) -> None:
INGEST_TAG_TEMPLATE = Template("""
UNWIND {TagData} as tag_mapping
UNWIND tag_mapping.Tags as input_tag
MATCH (resource:$resource_label{$property:tag_mapping.resource_id})
MERGE(aws_tag:AWSTag:Tag{id:input_tag.Key + ":" + input_tag.Value})
ON CREATE SET aws_tag.firstseen = timestamp()
SET aws_tag.lastupdated = {UpdateTag},
aws_tag.key = input_tag.Key,
aws_tag.value = input_tag.Value,
aws_tag.region = {Region}
MERGE (resource)-[r:TAGGED]->(aws_tag)
SET r.lastupdated = {UpdateTag},
r.firstseen = timestamp()
""")
query = INGEST_TAG_TEMPLATE.safe_substitute(
resource_label=TAG_RESOURCE_TYPE_MAPPINGS[resource_type]['label'],
property=TAG_RESOURCE_TYPE_MAPPINGS[resource_type]['property'],
)
neo4j_session.run(
query,
TagData=tag_data,
UpdateTag=aws_update_tag,
Region=region,
)
def _attach_target_tags(neo4j_session: neo4j.Session, fw: Resource,
gcp_update_tag: int) -> None:
"""
Attach target tags to the firewall object
:param neo4j_session: The neo4j session
:param fw: The firewall object
:param gcp_update_tag: The timestamp
:return: Nothing
"""
query = """
MATCH (fw:GCPFirewall{id:{FwPartialUri}})
MERGE (t:GCPNetworkTag{id:{TagId}})
ON CREATE SET t.firstseen = timestamp(),
t.tag_id = {TagId},
t.value = {TagValue}
SET t.lastupdated = {gcp_update_tag}
MERGE (fw)-[h:TARGET_TAG]->(t)
ON CREATE SET h.firstseen = timestamp()
SET h.lastupdated = {gcp_update_tag}
"""
for tag in fw.get('targetTags', []):
tag_id = _create_gcp_network_tag_id(fw['vpc_partial_uri'], tag)
neo4j_session.run(
query,
FwPartialUri=fw['id'],
TagId=tag_id,
TagValue=tag,
gcp_update_tag=gcp_update_tag,
)
def load_github_languages(neo4j_session: neo4j.Session, update_tag: int,
repo_languages: List[Dict]) -> None:
"""
Ingest the relationships for repo languages
:param neo4j_session: Neo4J session object for server communication
:param update_tag: Timestamp used to determine data freshness
:param repo_languages: list of language to repo mappings
:return: Nothing
"""
ingest_languages = """
UNWIND {Languages} as lang
MERGE (pl:ProgrammingLanguage{id: lang.language_name})
ON CREATE SET pl.firstseen = timestamp(),
pl.name = lang.language_name
SET pl.lastupdated = {UpdateTag}
WITH pl, lang
MATCH (repo:GitHubRepository{id: lang.repo_id})
MERGE (pl)<-[r:LANGUAGE]-(repo)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {UpdateTag}"""
neo4j_session.run(
ingest_languages,
Languages=repo_languages,
UpdateTag=update_tag,
)
def _attach_ec2_security_groups(neo4j_session: neo4j.Session,
instances: List[Dict],
aws_update_tag: int) -> None:
"""
Attach an RDS instance to its EC2SecurityGroups
"""
attach_rds_to_group = """
UNWIND {Groups} as rds_sg
MATCH (rds:RDSInstance{id: rds_sg.arn})
MERGE (sg:EC2SecurityGroup{id: rds_sg.group_id})
MERGE (rds)-[m:MEMBER_OF_EC2_SECURITY_GROUP]->(sg)
ON CREATE SET m.firstseen = timestamp()
SET m.lastupdated = {aws_update_tag}
"""
groups = []
for instance in instances:
for group in instance['VpcSecurityGroups']:
groups.append({
'arn': instance['DBInstanceArn'],
'group_id': group['VpcSecurityGroupId'],
})
neo4j_session.run(
attach_rds_to_group,
Groups=groups,
aws_update_tag=aws_update_tag,
)
def load_collaborators(neo4j_session: neo4j.Session, update_tag: int,
collaborators: Dict) -> None:
query = Template("""
UNWIND {UserData} as user
MERGE (u:GitHubUser{id: user.url})
ON CREATE SET u.firstseen = timestamp()
SET u.fullname = user.name,
u.username = user.login,
u.permission = user.permission,
u.email = user.email,
u.company = user.company,
u.lastupdated = {UpdateTag}
WITH u, user
MATCH (repo:GitHubRepository{id: user.repo_url})
MERGE (repo)<-[o:$rel_label]-(u)
ON CREATE SET o.firstseen = timestamp()
SET o.lastupdated = {UpdateTag}
""")
for collab_type in collaborators.keys():
relationship_label = f"OUTSIDE_COLLAB_{collab_type}"
neo4j_session.run(
query.safe_substitute(rel_label=relationship_label),
UserData=collaborators[collab_type],
UpdateTag=update_tag,
)
def _load_replication_links(
neo4j_session: neo4j.Session, replication_links: List[Dict], update_tag: int,
) -> None:
"""
Ingest replication links into neo4j.
"""
ingest_replication_links = """
UNWIND {replication_links_list} as replication_link
MERGE (rl:AzureReplicationLink{id: replication_link.id})
ON CREATE SET rl.firstseen = timestamp(),
rl.location = replication_link.location
SET rl.name = replication_link.name,
rl.partnerdatabase = replication_link.partner_database,
rl.partnerlocation = replication_link.partner_location,
rl.partnerrole = replication_link.partner_role,
rl.partnerserver = replication_link.partner_server,
rl.mode = replication_link.replication_mode,
rl.state = replication_link.replication_state,
rl.percentcomplete = replication_link.percent_complete,
rl.role = replication_link.role,
rl.starttime = replication_link.start_time,
rl.terminationallowed = replication_link.is_termination_allowed,
rl.lastupdated = {azure_update_tag}
WITH rl, replication_link
MATCH (d:AzureSQLDatabase{id: replication_link.database_id})
MERGE (d)-[r:CONTAINS]->(rl)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {azure_update_tag}
"""
neo4j_session.run(
ingest_replication_links,
replication_links_list=replication_links,
azure_update_tag=update_tag,
)