def _load_transparent_data_encryptions(
neo4j_session: neo4j.Session,
encryptions_list: List[Dict],
update_tag: int,
) -> None:
"""
Ingest transparent data encryptions into neo4j.
"""
ingest_data_encryptions = """
UNWIND {transparent_data_encryptions_list} as e
MERGE (tae:AzureTransparentDataEncryption{id: e.id})
ON CREATE SET tae.firstseen = timestamp(),
tae.location = e.location
SET tae.name = e.name,
tae.status = e.status,
tae.lastupdated = {azure_update_tag}
WITH tae, e
MATCH (d:AzureSQLDatabase{id: e.database_id})
MERGE (d)-[r:CONTAINS]->(tae)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {azure_update_tag}
"""
neo4j_session.run(
ingest_data_encryptions,
transparent_data_encryptions_list=encryptions_list,
azure_update_tag=update_tag,
)
def load_policy_statements(
neo4j_session: neo4j.Session,
policy_id: str,
policy_name: str,
statements: Any,
aws_update_tag: int,
) -> None:
injest_policy_statement = """
MATCH (policy:AWSPolicy{id: {PolicyId}})
WITH policy
UNWIND {Statements} as statement_data
MERGE (statement:AWSPolicyStatement{id: statement_data.id})
SET
statement.effect = statement_data.Effect,
statement.action = statement_data.Action,
statement.notaction = statement_data.NotAction,
statement.resource = statement_data.Resource,
statement.notresource = statement_data.NotResource,
statement.condition = statement_data.Condition,
statement.sid = statement_data.Sid,
statement.lastupdated = {aws_update_tag}
MERGE (policy)-[r:STATEMENT]->(statement)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {aws_update_tag}
"""
neo4j_session.run(
injest_policy_statement,
PolicyId=policy_id,
PolicyName=policy_name,
Statements=statements,
aws_update_tag=aws_update_tag,
).consume()
def load_gsuite_members(neo4j_session: neo4j.Session, group: Dict,
members: List[Dict], gsuite_update_tag: int) -> None:
ingestion_qry = """
UNWIND {MemberData} as member
MATCH (user:GSuiteUser {id: member.id}),(group:GSuiteGroup {id: {GroupID} })
MERGE (user)-[r:MEMBER_GSUITE_GROUP]->(group)
ON CREATE SET
r.firstseen = {UpdateTag}
ON MATCH SET
r.lastupdated = {UpdateTag}
"""
neo4j_session.run(
ingestion_qry,
MemberData=members,
GroupID=group.get("id"),
UpdateTag=gsuite_update_tag,
)
membership_qry = """
UNWIND {MemberData} as member
MATCH(group_1: GSuiteGroup{id: member.id}), (group_2:GSuiteGroup {id: {GroupID}})
MERGE (group_1)-[r:MEMBER_GSUITE_GROUP]->(group_2)
ON CREATE SET
r.firstseen = {UpdateTag}
ON MATCH SET
r.lastupdated = {UpdateTag}
"""
neo4j_session.run(membership_qry,
MemberData=members,
GroupID=group.get("id"),
UpdateTag=gsuite_update_tag)
def load_groups(
neo4j_session: neo4j.Session,
groups: List[Dict],
current_aws_account_id: str,
aws_update_tag: int,
) -> None:
ingest_group = """
MERGE (gnode:AWSGroup{arn: {ARN}})
ON CREATE SET gnode.groupid = {GROUP_ID}, gnode.firstseen = timestamp(), gnode.createdate = {CREATE_DATE}
SET gnode:AWSPrincipal, gnode.name = {GROUP_NAME}, gnode.path = {PATH},gnode.lastupdated = {aws_update_tag}
WITH gnode
MATCH (aa:AWSAccount{id: {AWS_ACCOUNT_ID}})
MERGE (aa)-[r:RESOURCE]->(gnode)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {aws_update_tag}
"""
for group in groups:
neo4j_session.run(
ingest_group,
ARN=group["Arn"],
GROUP_ID=group["GroupId"],
CREATE_DATE=str(group["CreateDate"]),
GROUP_NAME=group["GroupName"],
PATH=group["Path"],
AWS_ACCOUNT_ID=current_aws_account_id,
aws_update_tag=aws_update_tag,
)
def load_user_access_keys(neo4j_session: neo4j.Session, user_access_keys: Dict,
aws_update_tag: int) -> None:
# TODO change the node label to reflect that this is a user access key, not an account access key
ingest_account_key = """
MATCH (user:AWSUser{name: {UserName}})
WITH user
MERGE (key:AccountAccessKey{accesskeyid: {AccessKeyId}})
ON CREATE SET key.firstseen = timestamp(), key.createdate = {CreateDate}
SET key.status = {Status}, key.lastupdated = {aws_update_tag}
WITH user,key
MERGE (user)-[r:AWS_ACCESS_KEY]->(key)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {aws_update_tag}
"""
for username, access_keys in user_access_keys.items():
for key in access_keys["AccessKeyMetadata"]:
if key.get('AccessKeyId'):
neo4j_session.run(
ingest_account_key,
UserName=username,
AccessKeyId=key['AccessKeyId'],
CreateDate=str(key['CreateDate']),
Status=key['Status'],
aws_update_tag=aws_update_tag,
)
def load_tags(
neo4j_session: neo4j.Session, tag_data: Dict, resource_type: str, region: str,
aws_update_tag: int,
) -> None:
INGEST_TAG_TEMPLATE = Template("""
UNWIND {TagData} as tag_mapping
UNWIND tag_mapping.Tags as input_tag
MATCH (resource:$resource_label{$property:tag_mapping.resource_id})
MERGE(aws_tag:AWSTag:Tag{id:input_tag.Key + ":" + input_tag.Value})
ON CREATE SET aws_tag.firstseen = timestamp()
SET aws_tag.lastupdated = {UpdateTag},
aws_tag.key = input_tag.Key,
aws_tag.value = input_tag.Value,
aws_tag.region = {Region}
MERGE (resource)-[r:TAGGED]->(aws_tag)
SET r.lastupdated = {UpdateTag},
r.firstseen = timestamp()
""")
query = INGEST_TAG_TEMPLATE.safe_substitute(
resource_label=TAG_RESOURCE_TYPE_MAPPINGS[resource_type]['label'],
property=TAG_RESOURCE_TYPE_MAPPINGS[resource_type]['property'],
)
neo4j_session.run(
query,
TagData=tag_data,
UpdateTag=aws_update_tag,
Region=region,
)
def load_s3_buckets(neo4j_session: neo4j.Session, data: Dict,
current_aws_account_id: str, aws_update_tag: int) -> None:
ingest_bucket = """
MERGE (bucket:S3Bucket{id:{BucketName}})
ON CREATE SET bucket.firstseen = timestamp(), bucket.creationdate = {CreationDate}
SET bucket.name = {BucketName}, bucket.region = {BucketRegion}, bucket.arn = {Arn},
bucket.lastupdated = {aws_update_tag}
WITH bucket
MATCH (owner:AWSAccount{id: {AWS_ACCOUNT_ID}})
MERGE (owner)-[r:RESOURCE]->(bucket)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {aws_update_tag}
"""
# The owner data returned by the API maps to the aws account nickname and not the IAM user
# there doesn't seem to be a way to retreive the mapping but we can get the current context account
# so we map to that directly
for bucket in data["Buckets"]:
arn = "arn:aws:s3:::" + bucket["Name"]
neo4j_session.run(
ingest_bucket,
BucketName=bucket["Name"],
BucketRegion=bucket["Region"],
Arn=arn,
CreationDate=str(bucket["CreationDate"]),
AWS_ACCOUNT_ID=current_aws_account_id,
aws_update_tag=aws_update_tag,
)
def _load_cosmosdb_virtual_network_rules(
neo4j_session: neo4j.Session,
database_account: Dict,
azure_update_tag: int,
) -> None:
"""
Ingest the details of the Virtual Network Rules of the database account.
"""
if 'virtual_network_rules' in database_account and len(
database_account['virtual_network_rules']) > 0:
database_account_id = database_account['id']
virtual_network_rules = database_account['virtual_network_rules']
ingest_virtual_network_rules = """
UNWIND {virtual_network_rules_list} AS vnr
MERGE (rules:AzureCosmosDBVirtualNetworkRule{id: vnr.id})
ON CREATE SET rules.firstseen = timestamp()
SET rules.lastupdated = {azure_update_tag},
rules.ignoremissingvnetserviceendpoint = vnr.ignore_missing_v_net_service_endpoint
WITH rules
MATCH (d:AzureCosmosDBAccount{id: {DatabaseAccountId}})
MERGE (d)-[r:CONFIGURED_WITH]->(rules)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {azure_update_tag}
"""
neo4j_session.run(
ingest_virtual_network_rules,
virtual_network_rules_list=virtual_network_rules,
DatabaseAccountId=database_account_id,
azure_update_tag=azure_update_tag,
)
def _load_cassandra_keyspaces(neo4j_session: neo4j.Session,
cassandra_keyspaces: List[Dict],
update_tag: int) -> None:
"""
Ingest Cassandra keyspaces into neo4j.
"""
ingest_cassandra_keyspaces = """
UNWIND {cassandra_keyspaces_list} AS keyspace
MERGE (ck:AzureCosmosDBCassandraKeyspace{id: keyspace.id})
ON CREATE SET ck.firstseen = timestamp(), ck.type = keyspace.type,
ck.location = keyspace.location
SET ck.name = keyspace.name,
ck.lastupdated = {azure_update_tag},
ck.throughput = keyspace.options.throughput,
ck.maxthroughput = keyspace.options.autoscale_setting.max_throughput
WITH ck, keyspace
MATCH (d:AzureCosmosDBAccount{id: keyspace.database_account_id})
MERGE (d)-[r:CONTAINS]->(ck)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {azure_update_tag}
"""
neo4j_session.run(
ingest_cassandra_keyspaces,
cassandra_keyspaces_list=cassandra_keyspaces,
azure_update_tag=update_tag,
)
def _load_collections(neo4j_session: neo4j.Session, collections: List[Dict],
update_tag: int) -> None:
"""
Ingest MongoDB Collections into neo4j.
"""
ingest_collections = """
UNWIND {mongodb_collections_list} AS collection
MERGE (col:AzureCosmosDBMongoDBCollection{id: collection.id})
ON CREATE SET col.firstseen = timestamp(), col.type = collection.type,
col.location = collection.location
SET col.name = collection.name,
col.lastupdated = {azure_update_tag},
col.throughput = collection.options.throughput,
col.maxthroughput = collection.options.autoscale_setting.max_throughput,
col.collectionname = collection.resource.id,
col.analyticalttl = collection.resource.analytical_storage_ttl
WITH col, collection
MATCH (mdb:AzureCosmosDBMongoDBDatabase{id: collection.database_id})
MERGE (mdb)-[r:CONTAINS]->(col)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {azure_update_tag}
"""
neo4j_session.run(
ingest_collections,
mongodb_collections_list=collections,
azure_update_tag=update_tag,
)
def _load_cosmosdb_failover_policies(
neo4j_session: neo4j.Session,
database_account: Dict,
azure_update_tag: int,
) -> None:
"""
Ingest the details of the Failover Policies of the database account.
"""
if 'failover_policies' in database_account and len(
database_account['failover_policies']) > 0:
database_account_id = database_account['id']
failover_policies = database_account['failover_policies']
ingest_failover_policies = """
UNWIND {failover_policies_list} AS fp
MERGE (fpolicy:AzureCosmosDBAccountFailoverPolicy{id: fp.id})
ON CREATE SET fpolicy.firstseen = timestamp()
SET fpolicy.lastupdated = {azure_update_tag},
fpolicy.locationname = fp.location_name,
fpolicy.failoverpriority = fp.failover_priority
WITH fpolicy
MATCH (d:AzureCosmosDBAccount{id: {DatabaseAccountId}})
MERGE (d)-[r:CONTAINS]->(fpolicy)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {azure_update_tag}
"""
neo4j_session.run(
ingest_failover_policies,
failover_policies_list=failover_policies,
DatabaseAccountId=database_account_id,
azure_update_tag=azure_update_tag,
)
def _load_cassandra_tables(neo4j_session: neo4j.Session,
cassandra_tables: List[Dict],
update_tag: int) -> None:
"""
Ingest Cassandra Tables into neo4j.
"""
ingest_cassandra_tables = """
UNWIND {cassandra_tables_list} AS table
MERGE (ct:AzureCosmosDBCassandraTable{id: table.id})
ON CREATE SET ct.firstseen = timestamp(), ct.type = table.type,
ct.location = table.location
SET ct.name = table.name,
ct.lastupdated = {azure_update_tag},
ct.throughput = table.options.throughput,
ct.maxthroughput = table.options.autoscale_setting.max_throughput,
ct.container = table.resource.id,
ct.defaultttl = table.resource.default_ttl,
ct.analyticalttl = table.resource.analytical_storage_ttl
WITH ct, table
MATCH (ck:AzureCosmosDBCassandraKeyspace{id: table.keyspace_id})
MERGE (ck)-[r:CONTAINS]->(ct)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {azure_update_tag}
"""
neo4j_session.run(
ingest_cassandra_tables,
cassandra_tables_list=cassandra_tables,
azure_update_tag=update_tag,
)
def _load_user_role(neo4j_session: neo4j.Session, user_id: str,
roles_data: List[Dict], okta_update_tag: int) -> None:
ingest = """
MATCH (user:OktaUser{id: {USER_ID}})<-[:RESOURCE]-(org:OktaOrganization)
WITH user,org
UNWIND {ROLES_DATA} as role_data
MERGE (role_node:OktaAdministrationRole{id: role_data.type})
ON CREATE SET role_node.type = role_data.type, role_node.firstseen = timestamp()
SET role_node.label = role_data.label, role_node.lastupdated = {okta_update_tag}
WITH user, role_node, org
MERGE (user)-[r:MEMBER_OF_OKTA_ROLE]->(role_node)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {okta_update_tag}
WITH role_node, org
MERGE (org)-[r2:RESOURCE]->(role_node)
ON CREATE SET r2.firstseen = timestamp()
SET r2.lastupdated = {okta_update_tag}
"""
neo4j_session.run(
ingest,
USER_ID=user_id,
ROLES_DATA=roles_data,
okta_update_tag=okta_update_tag,
)
def _link_ip_to_A_record(neo4j_session: neo4j.Session, update_tag: int,
ip_list: List[str], parent_record: str) -> None:
"""
Link A record to to its IP
:param neo4j_session: Neo4j session object
:param update_tag: Update tag to set the node with and childs
:param ip_list: List of IP to link
:param parent_record: parent record to set DNS_POINTS_TO relationship to
"""
ingest = """
MATCH (parent:DNSRecord{id: {ParentId}})
WITH parent
UNWIND {IP_LIST} as current_ip
MERGE (ip_node:Ip{id: current_ip})
ON CREATE SET ip_node.firstseen = timestamp(), ip_node.ip = current_ip
SET ip_node.lastupdated = {update_tag}
WITH parent, ip_node
MERGE (parent)-[r:DNS_POINTS_TO]->(ip_node)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {update_tag}
"""
neo4j_session.run(
ingest,
ParentId=parent_record,
IP_LIST=ip_list,
update_tag=update_tag,
)
def _load_kms_key_grants(neo4j_session: neo4j.Session, grants_list: List[Dict],
update_tag: int) -> None:
"""
Ingest KMS Key Grants into neo4j.
"""
ingest_grants = """
UNWIND {grants} AS grant
MERGE (g:KMSGrant{id: grant.GrantId})
ON CREATE SET g.firstseen = timestamp(), g.granteeprincipal = grant.GranteePrincipal,
g.creationdate = grant.CreationDate
SET g.name = grant.GrantName, g.lastupdated = {UpdateTag}
WITH g, grant
MATCH (kmskey:KMSKey{id: grant.KeyId})
MERGE (g)-[r:APPLIED_ON]->(kmskey)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {UpdateTag}
"""
# neo4j does not accept datetime objects and values. This loop is used to convert
# these values to string.
for grant in grants_list:
grant['CreationDate'] = str(grant['CreationDate'])
neo4j_session.run(
ingest_grants,
grants=grants_list,
UpdateTag=update_tag,
)
def _load_mongodb_databases(neo4j_session: neo4j.Session,
mongodb_databases: List[Dict],
update_tag: int) -> None:
"""
Ingest MongoDB databases into neo4j.
"""
ingest_mongodb_databases = """
UNWIND {mongodb_databases_list} AS database
MERGE (mdb:AzureCosmosDBMongoDBDatabase{id: database.id})
ON CREATE SET mdb.firstseen = timestamp(), mdb.type = database.type,
mdb.location = database.location
SET mdb.name = database.name,
mdb.throughput = database.options.throughput,
mdb.maxthroughput = database.options.autoscale_setting.max_throughput,
mdb.lastupdated = {azure_update_tag}
WITH mdb, database
MATCH (d:AzureCosmosDBAccount{id: database.database_account_id})
MERGE (d)-[r:CONTAINS]->(mdb)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {azure_update_tag}
"""
neo4j_session.run(
ingest_mongodb_databases,
mongodb_databases_list=mongodb_databases,
azure_update_tag=update_tag,
)
def merge_module_sync_metadata(
neo4j_session: neo4j.Session,
group_type: str,
group_id: Union[str, int],
synced_type: str,
update_tag: int,
stat_handler: ScopedStatsClient,
) -> None:
'''
This creates `ModuleSyncMetadata` nodes when called from each of the individual modules or sub-modules.
The 'types' used here should be actual node labels. For example, if we did sync a particular AWSAccount's S3Buckets,
the `grouptype` is 'AWSAccount', the `groupid` is the particular account's `id`, and the `syncedtype` is 'S3Bucket'.
:param neo4j_session: Neo4j session object
:param group_type: The parent module's type
:param group_id: The parent module's id
:param synced_type: The sub-module's type
:param update_tag: Timestamp used to determine data freshness
'''
template = Template("""
MERGE (n:ModuleSyncMetadata{id:'${group_type}_${group_id}_${synced_type}'})
ON CREATE SET
n:SyncMetadata, n.firstseen=timestamp()
SET n.syncedtype='${synced_type}',
n.grouptype='${group_type}',
n.groupid={group_id},
n.lastupdated={UPDATE_TAG}
""")
neo4j_session.run(
template.safe_substitute(group_type=group_type, group_id=group_id, synced_type=synced_type),
group_id=group_id,
UPDATE_TAG=update_tag,
)
stat_handler.incr(f'{group_type}_{group_id}_{synced_type}_lastupdated', update_tag)
def _load_table_resources(neo4j_session: neo4j.Session,
table_resources: List[Dict],
update_tag: int) -> None:
"""
Ingest Table resources into neo4j.
"""
ingest_tables = """
UNWIND {table_resources_list} AS table
MERGE (tr:AzureCosmosDBTableResource{id: table.id})
ON CREATE SET tr.firstseen = timestamp(), tr.type = table.type,
tr.location = table.location
SET tr.name = table.name,
tr.lastupdated = {azure_update_tag},
tr.throughput = table.options.throughput,
tr.maxthroughput = table.options.autoscale_setting.max_throughput
WITH tr, table
MATCH (d:AzureCosmosDBAccount{id: table.database_account_id})
MERGE (d)-[r:CONTAINS]->(tr)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {azure_update_tag}
"""
neo4j_session.run(
ingest_tables,
table_resources_list=table_resources,
azure_update_tag=update_tag,
)
def _load_s3_acls(neo4j_session: neo4j.Session, acls: Dict,
aws_account_id: str, update_tag: int) -> None:
"""
Ingest S3 ACL into neo4j.
"""
ingest_acls = """
UNWIND {acls} AS acl
MERGE (a:S3Acl{id: acl.id})
ON CREATE SET a.firstseen = timestamp(), a.owner = acl.owner, a.ownerid = acl.ownerid, a.type = acl.type,
a.displayname = acl.displayname, a.granteeid = acl.granteeid, a.uri = acl.uri, a.permission = acl.permission
SET a.lastupdated = {UpdateTag}
WITH a,acl MATCH (s3:S3Bucket{id: acl.bucket})
MERGE (a)-[r:APPLIES_TO]->(s3)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {UpdateTag}
"""
neo4j_session.run(
ingest_acls,
acls=acls,
UpdateTag=update_tag,
)
# implement the acl permission
# https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#permissions
run_analysis_job(
'aws_s3acl_analysis.json',
neo4j_session,
{'AWS_ID': aws_account_id},
)
def _load_sql_containers(neo4j_session: neo4j.Session, containers: List[Dict],
update_tag: int) -> None:
"""
Ingest SQL Container details into neo4j.
"""
ingest_containers = """
UNWIND {sql_containers_list} AS container
MERGE (c:AzureCosmosDBSqlContainer{id: container.id})
ON CREATE SET c.firstseen = timestamp(), c.type = container.type,
c.location = container.location
SET c.name = container.name,
c.lastupdated = {azure_update_tag},
c.throughput = container.options.throughput,
c.maxthroughput = container.options.autoscale_setting.max_throughput,
c.container = container.resource.id,
c.defaultttl = container.resource.default_ttl,
c.analyticalttl = container.resource.analytical_storage_ttl,
c.isautomaticindexingpolicy = container.resource.indexing_policy.automatic,
c.indexingmode = container.resource.indexing_policy.indexing_mode,
c.conflictresolutionpolicymode = container.resource.conflict_resolution_policy.mode
WITH c, container
MATCH (sdb:AzureCosmosDBSqlDatabase{id: container.database_id})
MERGE (sdb)-[r:CONTAINS]->(c)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {azure_update_tag}
"""
neo4j_session.run(
ingest_containers,
sql_containers_list=containers,
azure_update_tag=update_tag,
)
def load_users(
neo4j_session: neo4j.Session,
users: List[Dict],
current_aws_account_id: str,
aws_update_tag: int,
) -> None:
ingest_user = """
MERGE (unode:AWSUser{arn: {ARN}})
ON CREATE SET unode:AWSPrincipal, unode.userid = {USERID}, unode.firstseen = timestamp(),
unode.createdate = {CREATE_DATE}
SET unode.name = {USERNAME}, unode.path = {PATH}, unode.passwordlastused = {PASSWORD_LASTUSED},
unode.lastupdated = {aws_update_tag}
WITH unode
MATCH (aa:AWSAccount{id: {AWS_ACCOUNT_ID}})
MERGE (aa)-[r:RESOURCE]->(unode)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {aws_update_tag}
"""
for user in users:
neo4j_session.run(
ingest_user,
ARN=user["Arn"],
USERID=user["UserId"],
CREATE_DATE=str(user["CreateDate"]),
USERNAME=user["UserName"],
PATH=user["Path"],
PASSWORD_LASTUSED=str(user.get("PasswordLastUsed", "")),
AWS_ACCOUNT_ID=current_aws_account_id,
aws_update_tag=aws_update_tag,
)
def load_disks(neo4j_session: neo4j.Session, subscription_id: str,
disk_list: List[Dict], update_tag: int) -> None:
ingest_disks = """
UNWIND {disks} AS disk
MERGE (d:AzureDisk{id: disk.id})
ON CREATE SET d.firstseen = timestamp(),
d.type = disk.type, d.location = disk.location,
d.resourcegroup = disk.resource_group
SET d.lastupdated = {update_tag}, d.name = disk.name,
d.createoption = disk.creation_data.create_option, d.disksizegb = disk.disk_size_gb,
d.encryption = disk.encryption_settings_collection.enabled, d.maxshares = disk.max_shares,
d.network_access_policy = disk.network_access_policy,
d.ostype = disk.os_type, d.tier = disk.tier,
d.sku = disk.sku.name, d.zones = disk.zones
WITH d
MATCH (owner:AzureSubscription{id: {SUBSCRIPTION_ID}})
MERGE (owner)-[r:RESOURCE]->(d)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {update_tag}"""
neo4j_session.run(
ingest_disks,
disks=disk_list,
SUBSCRIPTION_ID=subscription_id,
update_tag=update_tag,
)
def load_group_memberships(neo4j_session: neo4j.Session,
group_memberships: Dict,
aws_update_tag: int) -> None:
ingest_membership = """
MATCH (group:AWSGroup{arn: {GroupArn}})
WITH group
MATCH (user:AWSUser{arn: {PrincipalArn}})
MERGE (user)-[r:MEMBER_AWS_GROUP]->(group)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {aws_update_tag}
WITH user, group
MATCH (group)-[:POLICY]->(policy:AWSPolicy)
MERGE (user)-[r2:POLICY]->(policy)
SET r2.lastupdated = {aws_update_tag}
"""
for group_arn, membership_data in group_memberships.items():
for info in membership_data.get("Users", []):
principal_arn = info["Arn"]
neo4j_session.run(
ingest_membership,
GroupArn=group_arn,
PrincipalArn=principal_arn,
aws_update_tag=aws_update_tag,
)
def load_snapshots(neo4j_session: neo4j.Session, subscription_id: str,
snapshots: List[Dict], update_tag: int) -> None:
ingest_snapshots = """
UNWIND {snapshots} as snapshot
MERGE (s:AzureSnapshot{id: snapshot.id})
ON CREATE SET s.firstseen = timestamp(),
s.resourcegroup = snapshot.resource_group,
s.type = snapshot.type, s.location = snapshot.location
SET s.lastupdated = {update_tag}, s.name = snapshot.name,
s.createoption = snapshot.creation_data.create_option, s.disksizegb = snapshot.disk_size_gb,
s.encryption = snapshot.encryption_settings_collection.enabled, s.incremental = snapshot.incremental,
s.network_access_policy = snapshot.network_access_policy, s.ostype = snapshot.os_type,
s.tier = snapshot.tier, s.sku = snapshot.sku.name
WITH s
MATCH (owner:AzureSubscription{id: {SUBSCRIPTION_ID}})
MERGE (owner)-[r:RESOURCE]->(s)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {update_tag}"""
neo4j_session.run(
ingest_snapshots,
snapshots=snapshots,
SUBSCRIPTION_ID=subscription_id,
update_tag=update_tag,
)
def load_policy(
neo4j_session: neo4j.Session,
policy_id: str,
policy_name: str,
policy_type: str,
principal_arn: str,
aws_update_tag: int,
) -> None:
injest_policy = """
MERGE (policy:AWSPolicy{id: {PolicyId}})
ON CREATE SET
policy.firstseen = timestamp(),
policy.type = {PolicyType},
policy.name = {PolicyName}
SET
policy.lastupdated = {aws_update_tag}
WITH policy
MATCH (principal:AWSPrincipal{arn: {PrincipalArn}})
MERGE (policy) <-[r:POLICY]-(principal)
SET r.lastupdated = {aws_update_tag}
"""
neo4j_session.run(
injest_policy,
PolicyId=policy_id,
PolicyName=policy_name,
PolicyType=policy_type,
PrincipalArn=principal_arn,
aws_update_tag=aws_update_tag,
).consume()
def load_vms(neo4j_session: neo4j.Session, subscription_id: str,
vm_list: List[Dict], update_tag: int) -> None:
ingest_vm = """
UNWIND {vms} AS vm
MERGE (v:AzureVirtualMachine{id: vm.id})
ON CREATE SET v.firstseen = timestamp(),
v.type = vm.type, v.location = vm.location,
v.resourcegroup = vm.resource_group
SET v.lastupdated = {update_tag}, v.name = vm.name,
v.plan = vm.plan.product, v.size = vm.hardware_profile.vm_size,
v.license_type=vm.license_type, v.computer_name=vm.os_profile.computer_ame,
v.identity_type=vm.identity.type, v.zones=vm.zones,
v.ultra_ssd_enabled=vm.additional_capabilities.ultra_ssd_enabled,
v.priority=vm.priority, v.eviction_policy=vm.eviction_policy
WITH v
MATCH (owner:AzureSubscription{id: {SUBSCRIPTION_ID}})
MERGE (owner)-[r:RESOURCE]->(v)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {update_tag}
"""
neo4j_session.run(
ingest_vm,
vms=vm_list,
SUBSCRIPTION_ID=subscription_id,
update_tag=update_tag,
)
for vm in vm_list:
if vm.get('storage_profile', {}).get('data_disks'):
load_vm_data_disks(neo4j_session, vm['id'],
vm['storage_profile']['data_disks'], update_tag)
def load_ecr_repositories(
neo4j_session: neo4j.Session, repos: List[Dict], region: str, current_aws_account_id: str,
aws_update_tag: int,
) -> None:
query = """
UNWIND {Repositories} as ecr_repo
MERGE (repo:ECRRepository{id: ecr_repo.repositoryArn})
ON CREATE SET repo.firstseen = timestamp(),
repo.arn = ecr_repo.repositoryArn,
repo.name = ecr_repo.repositoryName,
repo.region = {Region},
repo.created_at = ecr_repo.createdAt
SET repo.lastupdated = {aws_update_tag},
repo.uri = ecr_repo.repositoryUri
WITH repo
MATCH (owner:AWSAccount{id: {AWS_ACCOUNT_ID}})
MERGE (owner)-[r:RESOURCE]->(repo)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {aws_update_tag}
"""
logger.info(f"Loading {len(repos)} ECR repositories for region {region} into graph.")
neo4j_session.run(
query,
Repositories=repos,
Region=region,
aws_update_tag=aws_update_tag,
AWS_ACCOUNT_ID=current_aws_account_id,
).consume() # See issue #440
def load_vm_data_disks(neo4j_session: neo4j.Session, vm_id: str,
data_disks: List[Dict], update_tag: int) -> None:
ingest_data_disk = """
UNWIND {disks} AS disk
MERGE (d:AzureDataDisk{id: disk.managed_disk.id})
ON CREATE SET d.firstseen = timestamp(), d.lun = disk.lun
SET d.lastupdated = {update_tag}, d.name = disk.name,
d.vhd = disk.vhd.uri, d.image = disk.image.uri,
d.size = disk.disk_size_gb, d.caching = disk.caching,
d.createoption = disk.create_option, d.write_accelerator_enabled=disk.write_accelerator_enabled,
d.managed_disk_storage_type=disk.managed_disk.storage_account_type
WITH d
MATCH (owner:AzureVirtualMachine{id: {VM_ID}})
MERGE (owner)-[r:ATTACHED_TO]->(d)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {update_tag}
"""
# for disk in data_disks:
neo4j_session.run(
ingest_data_disk,
disks=data_disks,
VM_ID=vm_id,
update_tag=update_tag,
)
def load_azure_subscriptions(
neo4j_session: neo4j.Session, tenant_id: str, subscriptions: List[Dict], update_tag: int,
) -> None:
query = """
MERGE (at:AzureTenant{id: {TENANT_ID}})
ON CREATE SET at.firstseen = timestamp()
SET at.lastupdated = {update_tag}
WITH at
MERGE (as:AzureSubscription{id: {SUBSCRIPTION_ID}})
ON CREATE SET as.firstseen = timestamp(), as.path = {SUBSCRIPTION_PATH}
SET as.lastupdated = {update_tag}, as.name = {SUBSCRIPTION_NAME}, as.state = {SUBSCRIPTION_STATE}
WITH as, at
MERGE (at)-[r:RESOURCE]->(as)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {update_tag};
"""
for sub in subscriptions:
neo4j_session.run(
query,
TENANT_ID=tenant_id,
SUBSCRIPTION_ID=sub['subscriptionId'],
SUBSCRIPTION_PATH=sub['id'],
SUBSCRIPTION_NAME=sub['displayName'],
SUBSCRIPTION_STATE=sub['state'],
update_tag=update_tag,
)
def _load_restore_points(
neo4j_session: neo4j.Session,
restore_points: List[Dict],
update_tag: int,
) -> None:
"""
Ingest restore points into neo4j.
"""
ingest_restore_points = """
UNWIND {restore_points_list} as rp
MERGE (point:AzureRestorePoint{id: rp.id})
ON CREATE SET point.firstseen = timestamp(),
point.location = rp.location
SET point.name = rp.name,
point.restoredate = rp.earliest_restore_date,
point.restorepointtype = rp.restore_point_type,
point.creationdate = rp.restore_point_creation_date,
point.lastupdated = {azure_update_tag}
WITH point, rp
MATCH (d:AzureSQLDatabase{id: rp.database_id})
MERGE (d)-[r:CONTAINS]->(point)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {azure_update_tag}
"""
neo4j_session.run(
ingest_restore_points,
restore_points_list=restore_points,
azure_update_tag=update_tag,
)
