def _load_transparent_data_encryptions( neo4j_session: neo4j.Session, encryptions_list: List[Dict], update_tag: int, ) -> None: """ Ingest transparent data encryptions into neo4j. """ ingest_data_encryptions = """ UNWIND {transparent_data_encryptions_list} as e MERGE (tae:AzureTransparentDataEncryption{id: e.id}) ON CREATE SET tae.firstseen = timestamp(), tae.location = e.location SET tae.name = e.name, tae.status = e.status, tae.lastupdated = {azure_update_tag} WITH tae, e MATCH (d:AzureSQLDatabase{id: e.database_id}) MERGE (d)-[r:CONTAINS]->(tae) ON CREATE SET r.firstseen = timestamp() SET r.lastupdated = {azure_update_tag} """ neo4j_session.run( ingest_data_encryptions, transparent_data_encryptions_list=encryptions_list, azure_update_tag=update_tag, )
def load_policy_statements( neo4j_session: neo4j.Session, policy_id: str, policy_name: str, statements: Any, aws_update_tag: int, ) -> None: injest_policy_statement = """ MATCH (policy:AWSPolicy{id: {PolicyId}}) WITH policy UNWIND {Statements} as statement_data MERGE (statement:AWSPolicyStatement{id: statement_data.id}) SET statement.effect = statement_data.Effect, statement.action = statement_data.Action, statement.notaction = statement_data.NotAction, statement.resource = statement_data.Resource, statement.notresource = statement_data.NotResource, statement.condition = statement_data.Condition, statement.sid = statement_data.Sid, statement.lastupdated = {aws_update_tag} MERGE (policy)-[r:STATEMENT]->(statement) ON CREATE SET r.firstseen = timestamp() SET r.lastupdated = {aws_update_tag} """ neo4j_session.run( injest_policy_statement, PolicyId=policy_id, PolicyName=policy_name, Statements=statements, aws_update_tag=aws_update_tag, ).consume()
def load_gsuite_members(neo4j_session: neo4j.Session, group: Dict, members: List[Dict], gsuite_update_tag: int) -> None: ingestion_qry = """ UNWIND {MemberData} as member MATCH (user:GSuiteUser {id: member.id}),(group:GSuiteGroup {id: {GroupID} }) MERGE (user)-[r:MEMBER_GSUITE_GROUP]->(group) ON CREATE SET r.firstseen = {UpdateTag} ON MATCH SET r.lastupdated = {UpdateTag} """ neo4j_session.run( ingestion_qry, MemberData=members, GroupID=group.get("id"), UpdateTag=gsuite_update_tag, ) membership_qry = """ UNWIND {MemberData} as member MATCH(group_1: GSuiteGroup{id: member.id}), (group_2:GSuiteGroup {id: {GroupID}}) MERGE (group_1)-[r:MEMBER_GSUITE_GROUP]->(group_2) ON CREATE SET r.firstseen = {UpdateTag} ON MATCH SET r.lastupdated = {UpdateTag} """ neo4j_session.run(membership_qry, MemberData=members, GroupID=group.get("id"), UpdateTag=gsuite_update_tag)
def load_groups( neo4j_session: neo4j.Session, groups: List[Dict], current_aws_account_id: str, aws_update_tag: int, ) -> None: ingest_group = """ MERGE (gnode:AWSGroup{arn: {ARN}}) ON CREATE SET gnode.groupid = {GROUP_ID}, gnode.firstseen = timestamp(), gnode.createdate = {CREATE_DATE} SET gnode:AWSPrincipal, gnode.name = {GROUP_NAME}, gnode.path = {PATH},gnode.lastupdated = {aws_update_tag} WITH gnode MATCH (aa:AWSAccount{id: {AWS_ACCOUNT_ID}}) MERGE (aa)-[r:RESOURCE]->(gnode) ON CREATE SET r.firstseen = timestamp() SET r.lastupdated = {aws_update_tag} """ for group in groups: neo4j_session.run( ingest_group, ARN=group["Arn"], GROUP_ID=group["GroupId"], CREATE_DATE=str(group["CreateDate"]), GROUP_NAME=group["GroupName"], PATH=group["Path"], AWS_ACCOUNT_ID=current_aws_account_id, aws_update_tag=aws_update_tag, )
def load_user_access_keys(neo4j_session: neo4j.Session, user_access_keys: Dict, aws_update_tag: int) -> None: # TODO change the node label to reflect that this is a user access key, not an account access key ingest_account_key = """ MATCH (user:AWSUser{name: {UserName}}) WITH user MERGE (key:AccountAccessKey{accesskeyid: {AccessKeyId}}) ON CREATE SET key.firstseen = timestamp(), key.createdate = {CreateDate} SET key.status = {Status}, key.lastupdated = {aws_update_tag} WITH user,key MERGE (user)-[r:AWS_ACCESS_KEY]->(key) ON CREATE SET r.firstseen = timestamp() SET r.lastupdated = {aws_update_tag} """ for username, access_keys in user_access_keys.items(): for key in access_keys["AccessKeyMetadata"]: if key.get('AccessKeyId'): neo4j_session.run( ingest_account_key, UserName=username, AccessKeyId=key['AccessKeyId'], CreateDate=str(key['CreateDate']), Status=key['Status'], aws_update_tag=aws_update_tag, )
def load_tags( neo4j_session: neo4j.Session, tag_data: Dict, resource_type: str, region: str, aws_update_tag: int, ) -> None: INGEST_TAG_TEMPLATE = Template(""" UNWIND {TagData} as tag_mapping UNWIND tag_mapping.Tags as input_tag MATCH (resource:$resource_label{$property:tag_mapping.resource_id}) MERGE(aws_tag:AWSTag:Tag{id:input_tag.Key + ":" + input_tag.Value}) ON CREATE SET aws_tag.firstseen = timestamp() SET aws_tag.lastupdated = {UpdateTag}, aws_tag.key = input_tag.Key, aws_tag.value = input_tag.Value, aws_tag.region = {Region} MERGE (resource)-[r:TAGGED]->(aws_tag) SET r.lastupdated = {UpdateTag}, r.firstseen = timestamp() """) query = INGEST_TAG_TEMPLATE.safe_substitute( resource_label=TAG_RESOURCE_TYPE_MAPPINGS[resource_type]['label'], property=TAG_RESOURCE_TYPE_MAPPINGS[resource_type]['property'], ) neo4j_session.run( query, TagData=tag_data, UpdateTag=aws_update_tag, Region=region, )
def load_s3_buckets(neo4j_session: neo4j.Session, data: Dict, current_aws_account_id: str, aws_update_tag: int) -> None: ingest_bucket = """ MERGE (bucket:S3Bucket{id:{BucketName}}) ON CREATE SET bucket.firstseen = timestamp(), bucket.creationdate = {CreationDate} SET bucket.name = {BucketName}, bucket.region = {BucketRegion}, bucket.arn = {Arn}, bucket.lastupdated = {aws_update_tag} WITH bucket MATCH (owner:AWSAccount{id: {AWS_ACCOUNT_ID}}) MERGE (owner)-[r:RESOURCE]->(bucket) ON CREATE SET r.firstseen = timestamp() SET r.lastupdated = {aws_update_tag} """ # The owner data returned by the API maps to the aws account nickname and not the IAM user # there doesn't seem to be a way to retreive the mapping but we can get the current context account # so we map to that directly for bucket in data["Buckets"]: arn = "arn:aws:s3:::" + bucket["Name"] neo4j_session.run( ingest_bucket, BucketName=bucket["Name"], BucketRegion=bucket["Region"], Arn=arn, CreationDate=str(bucket["CreationDate"]), AWS_ACCOUNT_ID=current_aws_account_id, aws_update_tag=aws_update_tag, )
def _load_cosmosdb_virtual_network_rules( neo4j_session: neo4j.Session, database_account: Dict, azure_update_tag: int, ) -> None: """ Ingest the details of the Virtual Network Rules of the database account. """ if 'virtual_network_rules' in database_account and len( database_account['virtual_network_rules']) > 0: database_account_id = database_account['id'] virtual_network_rules = database_account['virtual_network_rules'] ingest_virtual_network_rules = """ UNWIND {virtual_network_rules_list} AS vnr MERGE (rules:AzureCosmosDBVirtualNetworkRule{id: vnr.id}) ON CREATE SET rules.firstseen = timestamp() SET rules.lastupdated = {azure_update_tag}, rules.ignoremissingvnetserviceendpoint = vnr.ignore_missing_v_net_service_endpoint WITH rules MATCH (d:AzureCosmosDBAccount{id: {DatabaseAccountId}}) MERGE (d)-[r:CONFIGURED_WITH]->(rules) ON CREATE SET r.firstseen = timestamp() SET r.lastupdated = {azure_update_tag} """ neo4j_session.run( ingest_virtual_network_rules, virtual_network_rules_list=virtual_network_rules, DatabaseAccountId=database_account_id, azure_update_tag=azure_update_tag, )
def _load_cassandra_keyspaces(neo4j_session: neo4j.Session, cassandra_keyspaces: List[Dict], update_tag: int) -> None: """ Ingest Cassandra keyspaces into neo4j. """ ingest_cassandra_keyspaces = """ UNWIND {cassandra_keyspaces_list} AS keyspace MERGE (ck:AzureCosmosDBCassandraKeyspace{id: keyspace.id}) ON CREATE SET ck.firstseen = timestamp(), ck.type = keyspace.type, ck.location = keyspace.location SET ck.name = keyspace.name, ck.lastupdated = {azure_update_tag}, ck.throughput = keyspace.options.throughput, ck.maxthroughput = keyspace.options.autoscale_setting.max_throughput WITH ck, keyspace MATCH (d:AzureCosmosDBAccount{id: keyspace.database_account_id}) MERGE (d)-[r:CONTAINS]->(ck) ON CREATE SET r.firstseen = timestamp() SET r.lastupdated = {azure_update_tag} """ neo4j_session.run( ingest_cassandra_keyspaces, cassandra_keyspaces_list=cassandra_keyspaces, azure_update_tag=update_tag, )
def _load_collections(neo4j_session: neo4j.Session, collections: List[Dict], update_tag: int) -> None: """ Ingest MongoDB Collections into neo4j. """ ingest_collections = """ UNWIND {mongodb_collections_list} AS collection MERGE (col:AzureCosmosDBMongoDBCollection{id: collection.id}) ON CREATE SET col.firstseen = timestamp(), col.type = collection.type, col.location = collection.location SET col.name = collection.name, col.lastupdated = {azure_update_tag}, col.throughput = collection.options.throughput, col.maxthroughput = collection.options.autoscale_setting.max_throughput, col.collectionname = collection.resource.id, col.analyticalttl = collection.resource.analytical_storage_ttl WITH col, collection MATCH (mdb:AzureCosmosDBMongoDBDatabase{id: collection.database_id}) MERGE (mdb)-[r:CONTAINS]->(col) ON CREATE SET r.firstseen = timestamp() SET r.lastupdated = {azure_update_tag} """ neo4j_session.run( ingest_collections, mongodb_collections_list=collections, azure_update_tag=update_tag, )
def _load_cosmosdb_failover_policies( neo4j_session: neo4j.Session, database_account: Dict, azure_update_tag: int, ) -> None: """ Ingest the details of the Failover Policies of the database account. """ if 'failover_policies' in database_account and len( database_account['failover_policies']) > 0: database_account_id = database_account['id'] failover_policies = database_account['failover_policies'] ingest_failover_policies = """ UNWIND {failover_policies_list} AS fp MERGE (fpolicy:AzureCosmosDBAccountFailoverPolicy{id: fp.id}) ON CREATE SET fpolicy.firstseen = timestamp() SET fpolicy.lastupdated = {azure_update_tag}, fpolicy.locationname = fp.location_name, fpolicy.failoverpriority = fp.failover_priority WITH fpolicy MATCH (d:AzureCosmosDBAccount{id: {DatabaseAccountId}}) MERGE (d)-[r:CONTAINS]->(fpolicy) ON CREATE SET r.firstseen = timestamp() SET r.lastupdated = {azure_update_tag} """ neo4j_session.run( ingest_failover_policies, failover_policies_list=failover_policies, DatabaseAccountId=database_account_id, azure_update_tag=azure_update_tag, )
def _load_cassandra_tables(neo4j_session: neo4j.Session, cassandra_tables: List[Dict], update_tag: int) -> None: """ Ingest Cassandra Tables into neo4j. """ ingest_cassandra_tables = """ UNWIND {cassandra_tables_list} AS table MERGE (ct:AzureCosmosDBCassandraTable{id: table.id}) ON CREATE SET ct.firstseen = timestamp(), ct.type = table.type, ct.location = table.location SET ct.name = table.name, ct.lastupdated = {azure_update_tag}, ct.throughput = table.options.throughput, ct.maxthroughput = table.options.autoscale_setting.max_throughput, ct.container = table.resource.id, ct.defaultttl = table.resource.default_ttl, ct.analyticalttl = table.resource.analytical_storage_ttl WITH ct, table MATCH (ck:AzureCosmosDBCassandraKeyspace{id: table.keyspace_id}) MERGE (ck)-[r:CONTAINS]->(ct) ON CREATE SET r.firstseen = timestamp() SET r.lastupdated = {azure_update_tag} """ neo4j_session.run( ingest_cassandra_tables, cassandra_tables_list=cassandra_tables, azure_update_tag=update_tag, )
def _load_user_role(neo4j_session: neo4j.Session, user_id: str, roles_data: List[Dict], okta_update_tag: int) -> None: ingest = """ MATCH (user:OktaUser{id: {USER_ID}})<-[:RESOURCE]-(org:OktaOrganization) WITH user,org UNWIND {ROLES_DATA} as role_data MERGE (role_node:OktaAdministrationRole{id: role_data.type}) ON CREATE SET role_node.type = role_data.type, role_node.firstseen = timestamp() SET role_node.label = role_data.label, role_node.lastupdated = {okta_update_tag} WITH user, role_node, org MERGE (user)-[r:MEMBER_OF_OKTA_ROLE]->(role_node) ON CREATE SET r.firstseen = timestamp() SET r.lastupdated = {okta_update_tag} WITH role_node, org MERGE (org)-[r2:RESOURCE]->(role_node) ON CREATE SET r2.firstseen = timestamp() SET r2.lastupdated = {okta_update_tag} """ neo4j_session.run( ingest, USER_ID=user_id, ROLES_DATA=roles_data, okta_update_tag=okta_update_tag, )
def _link_ip_to_A_record(neo4j_session: neo4j.Session, update_tag: int, ip_list: List[str], parent_record: str) -> None: """ Link A record to to its IP :param neo4j_session: Neo4j session object :param update_tag: Update tag to set the node with and childs :param ip_list: List of IP to link :param parent_record: parent record to set DNS_POINTS_TO relationship to """ ingest = """ MATCH (parent:DNSRecord{id: {ParentId}}) WITH parent UNWIND {IP_LIST} as current_ip MERGE (ip_node:Ip{id: current_ip}) ON CREATE SET ip_node.firstseen = timestamp(), ip_node.ip = current_ip SET ip_node.lastupdated = {update_tag} WITH parent, ip_node MERGE (parent)-[r:DNS_POINTS_TO]->(ip_node) ON CREATE SET r.firstseen = timestamp() SET r.lastupdated = {update_tag} """ neo4j_session.run( ingest, ParentId=parent_record, IP_LIST=ip_list, update_tag=update_tag, )
def _load_kms_key_grants(neo4j_session: neo4j.Session, grants_list: List[Dict], update_tag: int) -> None: """ Ingest KMS Key Grants into neo4j. """ ingest_grants = """ UNWIND {grants} AS grant MERGE (g:KMSGrant{id: grant.GrantId}) ON CREATE SET g.firstseen = timestamp(), g.granteeprincipal = grant.GranteePrincipal, g.creationdate = grant.CreationDate SET g.name = grant.GrantName, g.lastupdated = {UpdateTag} WITH g, grant MATCH (kmskey:KMSKey{id: grant.KeyId}) MERGE (g)-[r:APPLIED_ON]->(kmskey) ON CREATE SET r.firstseen = timestamp() SET r.lastupdated = {UpdateTag} """ # neo4j does not accept datetime objects and values. This loop is used to convert # these values to string. for grant in grants_list: grant['CreationDate'] = str(grant['CreationDate']) neo4j_session.run( ingest_grants, grants=grants_list, UpdateTag=update_tag, )
def _load_mongodb_databases(neo4j_session: neo4j.Session, mongodb_databases: List[Dict], update_tag: int) -> None: """ Ingest MongoDB databases into neo4j. """ ingest_mongodb_databases = """ UNWIND {mongodb_databases_list} AS database MERGE (mdb:AzureCosmosDBMongoDBDatabase{id: database.id}) ON CREATE SET mdb.firstseen = timestamp(), mdb.type = database.type, mdb.location = database.location SET mdb.name = database.name, mdb.throughput = database.options.throughput, mdb.maxthroughput = database.options.autoscale_setting.max_throughput, mdb.lastupdated = {azure_update_tag} WITH mdb, database MATCH (d:AzureCosmosDBAccount{id: database.database_account_id}) MERGE (d)-[r:CONTAINS]->(mdb) ON CREATE SET r.firstseen = timestamp() SET r.lastupdated = {azure_update_tag} """ neo4j_session.run( ingest_mongodb_databases, mongodb_databases_list=mongodb_databases, azure_update_tag=update_tag, )
def merge_module_sync_metadata( neo4j_session: neo4j.Session, group_type: str, group_id: Union[str, int], synced_type: str, update_tag: int, stat_handler: ScopedStatsClient, ) -> None: ''' This creates `ModuleSyncMetadata` nodes when called from each of the individual modules or sub-modules. The 'types' used here should be actual node labels. For example, if we did sync a particular AWSAccount's S3Buckets, the `grouptype` is 'AWSAccount', the `groupid` is the particular account's `id`, and the `syncedtype` is 'S3Bucket'. :param neo4j_session: Neo4j session object :param group_type: The parent module's type :param group_id: The parent module's id :param synced_type: The sub-module's type :param update_tag: Timestamp used to determine data freshness ''' template = Template(""" MERGE (n:ModuleSyncMetadata{id:'${group_type}_${group_id}_${synced_type}'}) ON CREATE SET n:SyncMetadata, n.firstseen=timestamp() SET n.syncedtype='${synced_type}', n.grouptype='${group_type}', n.groupid={group_id}, n.lastupdated={UPDATE_TAG} """) neo4j_session.run( template.safe_substitute(group_type=group_type, group_id=group_id, synced_type=synced_type), group_id=group_id, UPDATE_TAG=update_tag, ) stat_handler.incr(f'{group_type}_{group_id}_{synced_type}_lastupdated', update_tag)
def _load_table_resources(neo4j_session: neo4j.Session, table_resources: List[Dict], update_tag: int) -> None: """ Ingest Table resources into neo4j. """ ingest_tables = """ UNWIND {table_resources_list} AS table MERGE (tr:AzureCosmosDBTableResource{id: table.id}) ON CREATE SET tr.firstseen = timestamp(), tr.type = table.type, tr.location = table.location SET tr.name = table.name, tr.lastupdated = {azure_update_tag}, tr.throughput = table.options.throughput, tr.maxthroughput = table.options.autoscale_setting.max_throughput WITH tr, table MATCH (d:AzureCosmosDBAccount{id: table.database_account_id}) MERGE (d)-[r:CONTAINS]->(tr) ON CREATE SET r.firstseen = timestamp() SET r.lastupdated = {azure_update_tag} """ neo4j_session.run( ingest_tables, table_resources_list=table_resources, azure_update_tag=update_tag, )
def _load_s3_acls(neo4j_session: neo4j.Session, acls: Dict, aws_account_id: str, update_tag: int) -> None: """ Ingest S3 ACL into neo4j. """ ingest_acls = """ UNWIND {acls} AS acl MERGE (a:S3Acl{id: acl.id}) ON CREATE SET a.firstseen = timestamp(), a.owner = acl.owner, a.ownerid = acl.ownerid, a.type = acl.type, a.displayname = acl.displayname, a.granteeid = acl.granteeid, a.uri = acl.uri, a.permission = acl.permission SET a.lastupdated = {UpdateTag} WITH a,acl MATCH (s3:S3Bucket{id: acl.bucket}) MERGE (a)-[r:APPLIES_TO]->(s3) ON CREATE SET r.firstseen = timestamp() SET r.lastupdated = {UpdateTag} """ neo4j_session.run( ingest_acls, acls=acls, UpdateTag=update_tag, ) # implement the acl permission # https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#permissions run_analysis_job( 'aws_s3acl_analysis.json', neo4j_session, {'AWS_ID': aws_account_id}, )
def _load_sql_containers(neo4j_session: neo4j.Session, containers: List[Dict], update_tag: int) -> None: """ Ingest SQL Container details into neo4j. """ ingest_containers = """ UNWIND {sql_containers_list} AS container MERGE (c:AzureCosmosDBSqlContainer{id: container.id}) ON CREATE SET c.firstseen = timestamp(), c.type = container.type, c.location = container.location SET c.name = container.name, c.lastupdated = {azure_update_tag}, c.throughput = container.options.throughput, c.maxthroughput = container.options.autoscale_setting.max_throughput, c.container = container.resource.id, c.defaultttl = container.resource.default_ttl, c.analyticalttl = container.resource.analytical_storage_ttl, c.isautomaticindexingpolicy = container.resource.indexing_policy.automatic, c.indexingmode = container.resource.indexing_policy.indexing_mode, c.conflictresolutionpolicymode = container.resource.conflict_resolution_policy.mode WITH c, container MATCH (sdb:AzureCosmosDBSqlDatabase{id: container.database_id}) MERGE (sdb)-[r:CONTAINS]->(c) ON CREATE SET r.firstseen = timestamp() SET r.lastupdated = {azure_update_tag} """ neo4j_session.run( ingest_containers, sql_containers_list=containers, azure_update_tag=update_tag, )
def load_users( neo4j_session: neo4j.Session, users: List[Dict], current_aws_account_id: str, aws_update_tag: int, ) -> None: ingest_user = """ MERGE (unode:AWSUser{arn: {ARN}}) ON CREATE SET unode:AWSPrincipal, unode.userid = {USERID}, unode.firstseen = timestamp(), unode.createdate = {CREATE_DATE} SET unode.name = {USERNAME}, unode.path = {PATH}, unode.passwordlastused = {PASSWORD_LASTUSED}, unode.lastupdated = {aws_update_tag} WITH unode MATCH (aa:AWSAccount{id: {AWS_ACCOUNT_ID}}) MERGE (aa)-[r:RESOURCE]->(unode) ON CREATE SET r.firstseen = timestamp() SET r.lastupdated = {aws_update_tag} """ for user in users: neo4j_session.run( ingest_user, ARN=user["Arn"], USERID=user["UserId"], CREATE_DATE=str(user["CreateDate"]), USERNAME=user["UserName"], PATH=user["Path"], PASSWORD_LASTUSED=str(user.get("PasswordLastUsed", "")), AWS_ACCOUNT_ID=current_aws_account_id, aws_update_tag=aws_update_tag, )
def load_disks(neo4j_session: neo4j.Session, subscription_id: str, disk_list: List[Dict], update_tag: int) -> None: ingest_disks = """ UNWIND {disks} AS disk MERGE (d:AzureDisk{id: disk.id}) ON CREATE SET d.firstseen = timestamp(), d.type = disk.type, d.location = disk.location, d.resourcegroup = disk.resource_group SET d.lastupdated = {update_tag}, d.name = disk.name, d.createoption = disk.creation_data.create_option, d.disksizegb = disk.disk_size_gb, d.encryption = disk.encryption_settings_collection.enabled, d.maxshares = disk.max_shares, d.network_access_policy = disk.network_access_policy, d.ostype = disk.os_type, d.tier = disk.tier, d.sku = disk.sku.name, d.zones = disk.zones WITH d MATCH (owner:AzureSubscription{id: {SUBSCRIPTION_ID}}) MERGE (owner)-[r:RESOURCE]->(d) ON CREATE SET r.firstseen = timestamp() SET r.lastupdated = {update_tag}""" neo4j_session.run( ingest_disks, disks=disk_list, SUBSCRIPTION_ID=subscription_id, update_tag=update_tag, )
def load_group_memberships(neo4j_session: neo4j.Session, group_memberships: Dict, aws_update_tag: int) -> None: ingest_membership = """ MATCH (group:AWSGroup{arn: {GroupArn}}) WITH group MATCH (user:AWSUser{arn: {PrincipalArn}}) MERGE (user)-[r:MEMBER_AWS_GROUP]->(group) ON CREATE SET r.firstseen = timestamp() SET r.lastupdated = {aws_update_tag} WITH user, group MATCH (group)-[:POLICY]->(policy:AWSPolicy) MERGE (user)-[r2:POLICY]->(policy) SET r2.lastupdated = {aws_update_tag} """ for group_arn, membership_data in group_memberships.items(): for info in membership_data.get("Users", []): principal_arn = info["Arn"] neo4j_session.run( ingest_membership, GroupArn=group_arn, PrincipalArn=principal_arn, aws_update_tag=aws_update_tag, )
def load_snapshots(neo4j_session: neo4j.Session, subscription_id: str, snapshots: List[Dict], update_tag: int) -> None: ingest_snapshots = """ UNWIND {snapshots} as snapshot MERGE (s:AzureSnapshot{id: snapshot.id}) ON CREATE SET s.firstseen = timestamp(), s.resourcegroup = snapshot.resource_group, s.type = snapshot.type, s.location = snapshot.location SET s.lastupdated = {update_tag}, s.name = snapshot.name, s.createoption = snapshot.creation_data.create_option, s.disksizegb = snapshot.disk_size_gb, s.encryption = snapshot.encryption_settings_collection.enabled, s.incremental = snapshot.incremental, s.network_access_policy = snapshot.network_access_policy, s.ostype = snapshot.os_type, s.tier = snapshot.tier, s.sku = snapshot.sku.name WITH s MATCH (owner:AzureSubscription{id: {SUBSCRIPTION_ID}}) MERGE (owner)-[r:RESOURCE]->(s) ON CREATE SET r.firstseen = timestamp() SET r.lastupdated = {update_tag}""" neo4j_session.run( ingest_snapshots, snapshots=snapshots, SUBSCRIPTION_ID=subscription_id, update_tag=update_tag, )
def load_policy( neo4j_session: neo4j.Session, policy_id: str, policy_name: str, policy_type: str, principal_arn: str, aws_update_tag: int, ) -> None: injest_policy = """ MERGE (policy:AWSPolicy{id: {PolicyId}}) ON CREATE SET policy.firstseen = timestamp(), policy.type = {PolicyType}, policy.name = {PolicyName} SET policy.lastupdated = {aws_update_tag} WITH policy MATCH (principal:AWSPrincipal{arn: {PrincipalArn}}) MERGE (policy) <-[r:POLICY]-(principal) SET r.lastupdated = {aws_update_tag} """ neo4j_session.run( injest_policy, PolicyId=policy_id, PolicyName=policy_name, PolicyType=policy_type, PrincipalArn=principal_arn, aws_update_tag=aws_update_tag, ).consume()
def load_vms(neo4j_session: neo4j.Session, subscription_id: str, vm_list: List[Dict], update_tag: int) -> None: ingest_vm = """ UNWIND {vms} AS vm MERGE (v:AzureVirtualMachine{id: vm.id}) ON CREATE SET v.firstseen = timestamp(), v.type = vm.type, v.location = vm.location, v.resourcegroup = vm.resource_group SET v.lastupdated = {update_tag}, v.name = vm.name, v.plan = vm.plan.product, v.size = vm.hardware_profile.vm_size, v.license_type=vm.license_type, v.computer_name=vm.os_profile.computer_ame, v.identity_type=vm.identity.type, v.zones=vm.zones, v.ultra_ssd_enabled=vm.additional_capabilities.ultra_ssd_enabled, v.priority=vm.priority, v.eviction_policy=vm.eviction_policy WITH v MATCH (owner:AzureSubscription{id: {SUBSCRIPTION_ID}}) MERGE (owner)-[r:RESOURCE]->(v) ON CREATE SET r.firstseen = timestamp() SET r.lastupdated = {update_tag} """ neo4j_session.run( ingest_vm, vms=vm_list, SUBSCRIPTION_ID=subscription_id, update_tag=update_tag, ) for vm in vm_list: if vm.get('storage_profile', {}).get('data_disks'): load_vm_data_disks(neo4j_session, vm['id'], vm['storage_profile']['data_disks'], update_tag)
def load_ecr_repositories( neo4j_session: neo4j.Session, repos: List[Dict], region: str, current_aws_account_id: str, aws_update_tag: int, ) -> None: query = """ UNWIND {Repositories} as ecr_repo MERGE (repo:ECRRepository{id: ecr_repo.repositoryArn}) ON CREATE SET repo.firstseen = timestamp(), repo.arn = ecr_repo.repositoryArn, repo.name = ecr_repo.repositoryName, repo.region = {Region}, repo.created_at = ecr_repo.createdAt SET repo.lastupdated = {aws_update_tag}, repo.uri = ecr_repo.repositoryUri WITH repo MATCH (owner:AWSAccount{id: {AWS_ACCOUNT_ID}}) MERGE (owner)-[r:RESOURCE]->(repo) ON CREATE SET r.firstseen = timestamp() SET r.lastupdated = {aws_update_tag} """ logger.info(f"Loading {len(repos)} ECR repositories for region {region} into graph.") neo4j_session.run( query, Repositories=repos, Region=region, aws_update_tag=aws_update_tag, AWS_ACCOUNT_ID=current_aws_account_id, ).consume() # See issue #440
def load_vm_data_disks(neo4j_session: neo4j.Session, vm_id: str, data_disks: List[Dict], update_tag: int) -> None: ingest_data_disk = """ UNWIND {disks} AS disk MERGE (d:AzureDataDisk{id: disk.managed_disk.id}) ON CREATE SET d.firstseen = timestamp(), d.lun = disk.lun SET d.lastupdated = {update_tag}, d.name = disk.name, d.vhd = disk.vhd.uri, d.image = disk.image.uri, d.size = disk.disk_size_gb, d.caching = disk.caching, d.createoption = disk.create_option, d.write_accelerator_enabled=disk.write_accelerator_enabled, d.managed_disk_storage_type=disk.managed_disk.storage_account_type WITH d MATCH (owner:AzureVirtualMachine{id: {VM_ID}}) MERGE (owner)-[r:ATTACHED_TO]->(d) ON CREATE SET r.firstseen = timestamp() SET r.lastupdated = {update_tag} """ # for disk in data_disks: neo4j_session.run( ingest_data_disk, disks=data_disks, VM_ID=vm_id, update_tag=update_tag, )
def load_azure_subscriptions( neo4j_session: neo4j.Session, tenant_id: str, subscriptions: List[Dict], update_tag: int, ) -> None: query = """ MERGE (at:AzureTenant{id: {TENANT_ID}}) ON CREATE SET at.firstseen = timestamp() SET at.lastupdated = {update_tag} WITH at MERGE (as:AzureSubscription{id: {SUBSCRIPTION_ID}}) ON CREATE SET as.firstseen = timestamp(), as.path = {SUBSCRIPTION_PATH} SET as.lastupdated = {update_tag}, as.name = {SUBSCRIPTION_NAME}, as.state = {SUBSCRIPTION_STATE} WITH as, at MERGE (at)-[r:RESOURCE]->(as) ON CREATE SET r.firstseen = timestamp() SET r.lastupdated = {update_tag}; """ for sub in subscriptions: neo4j_session.run( query, TENANT_ID=tenant_id, SUBSCRIPTION_ID=sub['subscriptionId'], SUBSCRIPTION_PATH=sub['id'], SUBSCRIPTION_NAME=sub['displayName'], SUBSCRIPTION_STATE=sub['state'], update_tag=update_tag, )
def _load_restore_points( neo4j_session: neo4j.Session, restore_points: List[Dict], update_tag: int, ) -> None: """ Ingest restore points into neo4j. """ ingest_restore_points = """ UNWIND {restore_points_list} as rp MERGE (point:AzureRestorePoint{id: rp.id}) ON CREATE SET point.firstseen = timestamp(), point.location = rp.location SET point.name = rp.name, point.restoredate = rp.earliest_restore_date, point.restorepointtype = rp.restore_point_type, point.creationdate = rp.restore_point_creation_date, point.lastupdated = {azure_update_tag} WITH point, rp MATCH (d:AzureSQLDatabase{id: rp.database_id}) MERGE (d)-[r:CONTAINS]->(point) ON CREATE SET r.firstseen = timestamp() SET r.lastupdated = {azure_update_tag} """ neo4j_session.run( ingest_restore_points, restore_points_list=restore_points, azure_update_tag=update_tag, )