def _load_ec2_instance_ebs_tx(
tx: neo4j.Transaction,
ebs_data: List[Dict[str, Any]],
update_tag: int,
current_aws_account_id: str,
) -> None:
query = """
UNWIND {ebs_mappings_list} as em
MERGE (vol:EBSVolume{id: em.Ebs.VolumeId})
ON CREATE SET vol.firstseen = timestamp()
SET vol.lastupdated = {update_tag},
vol.deleteontermination = em.Ebs.DeleteOnTermination,
vol.snapshotid = vol.SnapshotId
WITH vol, em
MATCH (aa:AWSAccount{id: {AWS_ACCOUNT_ID}})
MERGE (aa)-[r:RESOURCE]->(vol)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {update_tag}
WITH vol, em
MATCH (instance:EC2Instance{instanceid: em.InstanceId})
MERGE (vol)-[r:ATTACHED_TO]->(instance)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {update_tag}
"""
tx.run(
query,
ebs_mappings_list=ebs_data,
update_tag=update_tag,
AWS_ACCOUNT_ID=current_aws_account_id,
)
def _load_ec2_security_groups_tx(
tx: neo4j.Transaction,
group_id: str,
group: Dict[str, Any],
instanceid: str,
region: str,
current_aws_account_id: str,
update_tag: int,
) -> None:
query = """
MERGE (group:EC2SecurityGroup{id: {GroupId}})
ON CREATE SET group.firstseen = timestamp(), group.groupid = {GroupId}
SET group.name = {GroupName}, group.region = {Region}, group.lastupdated = {update_tag}
WITH group
MATCH (aa:AWSAccount{id: {AWS_ACCOUNT_ID}})
MERGE (aa)-[r:RESOURCE]->(group)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {update_tag}
WITH group
MATCH (instance:EC2Instance{instanceid: {InstanceId}})
MERGE (instance)-[r:MEMBER_OF_EC2_SECURITY_GROUP]->(group)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {update_tag}
"""
tx.run(
query,
GroupId=group_id,
GroupName=group.get("GroupName"),
InstanceId=instanceid,
Region=region,
AWS_ACCOUNT_ID=current_aws_account_id,
update_tag=update_tag,
)
def _load_ecr_repo_img_tx(
tx: neo4j.Transaction, repo_images_list: List[Dict], aws_update_tag: int,
region: str,
) -> None:
query = """
UNWIND {RepoList} as repo_img
MERGE (ri:ECRRepositoryImage{id: repo_img.repo_uri + COALESCE(":" + repo_img.imageTag, '')})
ON CREATE SET ri.firstseen = timestamp()
SET ri.lastupdated = {aws_update_tag},
ri.tag = repo_img.imageTag,
ri.uri = repo_img.repo_uri + COALESCE(":" + repo_img.imageTag, '')
WITH ri, repo_img
MERGE (img:ECRImage{id: repo_img.imageDigest})
ON CREATE SET img.firstseen = timestamp(),
img.digest = repo_img.imageDigest
SET img.lastupdated = {aws_update_tag},
img.region = {Region}
WITH ri, img, repo_img
MERGE (ri)-[r1:IMAGE]->(img)
ON CREATE SET r1.firstseen = timestamp()
SET r1.lastupdated = {aws_update_tag}
WITH ri, repo_img
MATCH (repo:ECRRepository{uri: repo_img.repo_uri})
MERGE (repo)-[r2:REPO_IMAGE]->(ri)
ON CREATE SET r2.firstseen = timestamp()
SET r2.lastupdated = {aws_update_tag}
"""
tx.run(query, RepoList=repo_images_list, Region=region, aws_update_tag=aws_update_tag)
def _load_policy_tx(
tx: neo4j.Transaction,
policy_id: str,
policy_name: str,
policy_type: str,
principal_arn: str,
aws_update_tag: int,
) -> None:
ingest_policy = """
MERGE (policy:AWSPolicy{id: {PolicyId}})
ON CREATE SET
policy.firstseen = timestamp(),
policy.type = {PolicyType},
policy.name = {PolicyName}
SET policy.lastupdated = {aws_update_tag}
WITH policy
MATCH (principal:AWSPrincipal{arn: {PrincipalArn}})
MERGE (policy) <-[r:POLICY]-(principal)
SET r.lastupdated = {aws_update_tag}
"""
tx.run(
ingest_policy,
PolicyId=policy_id,
PolicyName=policy_name,
PolicyType=policy_type,
PrincipalArn=principal_arn,
aws_update_tag=aws_update_tag,
)
def _load_ec2_reservation_tx(
tx: neo4j.Transaction,
reservation_id: str,
reservation: Dict[str, Any],
current_aws_account_id: str,
region: str,
update_tag: int,
) -> None:
query = """
MERGE (reservation:EC2Reservation{reservationid: {ReservationId}})
ON CREATE SET reservation.firstseen = timestamp()
SET reservation.ownerid = {OwnerId},
reservation.requesterid = {RequesterId},
reservation.region = {Region},
reservation.lastupdated = {update_tag}
WITH reservation
MATCH (awsAccount:AWSAccount{id: {AWS_ACCOUNT_ID}})
MERGE (awsAccount)-[r:RESOURCE]->(reservation)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {update_tag}
"""
tx.run(
query,
ReservationId=reservation_id,
OwnerId=reservation.get("OwnerId"),
RequesterId=reservation.get("RequesterId"),
AWS_ACCOUNT_ID=current_aws_account_id,
Region=region,
update_tag=update_tag,
)
def _load_ec2_keypairs_tx(
tx: neo4j.Transaction,
key_pair_arn: str,
key_name: str,
region: str,
instanceid: str,
current_aws_account_id: str,
update_tag: int,
) -> None:
query = """
MERGE (keypair:KeyPair:EC2KeyPair{arn: {KeyPairARN}, id: {KeyPairARN}})
ON CREATE SET keypair.firstseen = timestamp()
SET keypair.keyname = {KeyName}, keypair.region = {Region}, keypair.lastupdated = {update_tag}
WITH keypair
MATCH (aa:AWSAccount{id: {AWS_ACCOUNT_ID}})
MERGE (aa)-[r:RESOURCE]->(keypair)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {update_tag}
with keypair
MATCH (instance:EC2Instance{instanceid: {InstanceId}})
MERGE (instance)<-[r:SSH_LOGIN_TO]-(keypair)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {update_tag}
"""
tx.run(
query,
KeyPairARN=key_pair_arn,
KeyName=key_name,
Region=region,
InstanceId=instanceid,
AWS_ACCOUNT_ID=current_aws_account_id,
update_tag=update_tag,
)
def process_ace_list(ace_list: list, objectid: str, objecttype: str,
tx: neo4j.Transaction) -> None:
for entry in ace_list:
principal = entry['PrincipalSID']
principaltype = entry['PrincipalType']
right = entry['RightName']
acetype = entry['AceType']
if objectid == principal:
continue
rights = []
if acetype in ACETYPE_MAP:
rights.append(ACETYPE_MAP[acetype])
elif right == "ExtendedRight":
rights.append(acetype)
if right in RIGHTS_MAP:
rights.append(RIGHTS_MAP[right])
for right in rights:
query = build_add_edge_query(
principaltype, objecttype, right,
'{isacl: true, isinherited: prop.isinherited}')
props = dict(
source=principal,
target=objectid,
isinherited=entry['IsInherited'],
)
tx.run(query, props=props)
def parse_computer(tx: neo4j.Transaction, computer: dict):
"""Parse a computer object.
Arguments:
session {neo4j.Transaction} -- Neo4j transaction
computer {dict} -- Single computer object.
"""
identifier = computer['ObjectIdentifier']
property_query = 'UNWIND $props AS prop MERGE (n:Base {objectid: prop.source}) ON MATCH SET n:Computer ON CREATE SET n:Computer SET n += prop.map'
props = {'map': computer['Properties'], 'source': identifier}
tx.run(property_query, props=props)
if 'PrimaryGroupSid' in computer and computer['PrimaryGroupSid']:
query = build_add_edge_query('Computer', 'Group', 'MemberOf',
'{isacl:false}')
tx.run(query,
props=dict(source=identifier,
target=computer['PrimaryGroupSid']))
if 'AllowedToDelegate' in computer and computer['AllowedToDelegate']:
query = build_add_edge_query('Computer', 'Group', 'MemberOf',
'{isacl:false}')
for entry in computer['AllowedToDelegate']:
tx.run(query, props=dict(source=identifier, target=entry))
options = [('AllowedToAct', 'AllowedToAct'), ('LocalAdmins', 'AdminTo'),
('RemoteDesktopUsers', 'CanRDP'), ('DcomUsers', 'ExecuteDCOM'),
('PSRemoteUsers', 'CanPSRemote')]
for option, edge_name in options:
if option in computer and computer[option]:
targets = computer[option]
for target in targets:
query = build_add_edge_query(target['MemberType'], 'Computer',
edge_name,
'{isacl:false, fromgpo: false}')
tx.run(query,
props=dict(source=target['MemberId'],
target=identifier))
if 'Sessions' in computer and computer['Sessions']:
query = build_add_edge_query('Computer', 'User', 'HasSession',
'{isacl:false}')
for entry in computer['Sessions']:
tx.run(query,
props=dict(source=entry['UserId'], target=identifier))
if 'Aces' in computer and computer['Aces'] is not None:
process_ace_list(computer['Aces'], identifier, "Computer", tx)
def parse_gpo(tx: neo4j.Transaction, gpo: dict):
"""Parses a single GPO.
Arguments:
tx {neo4j.Transaction} -- Neo4j transaction
gpo {dict} -- Single gpo object.
"""
identifier = gpo['ObjectIdentifier']
query = 'UNWIND $props AS prop MERGE (n:Base {objectid: prop.source}) ON MATCH SET n:GPO ON CREATE SET n:GPO SET n += prop.map'
props = {'map': gpo['Properties'], 'source': identifier}
tx.run(query, props=props)
if "Aces" in gpo and gpo["Aces"] is not None:
process_ace_list(gpo['Aces'], identifier, "GPO", tx)
def _run_noniterative(self,
tx: neo4j.Transaction) -> neo4j.StatementResult:
"""
Non-iterative statement execution.
"""
result: neo4j.StatementResult = tx.run(self.query, self.parameters)
# Handle stats
summary: neo4j.BoltStatementResultSummary = result.summary()
stat_handler.incr('constraints_added',
summary.counters.constraints_added)
stat_handler.incr('constraints_removed',
summary.counters.constraints_removed)
stat_handler.incr('indexes_added', summary.counters.indexes_added)
stat_handler.incr('indexes_removed', summary.counters.indexes_removed)
stat_handler.incr('labels_added', summary.counters.labels_added)
stat_handler.incr('labels_removed', summary.counters.labels_removed)
stat_handler.incr('nodes_created', summary.counters.nodes_created)
stat_handler.incr('nodes_deleted', summary.counters.nodes_deleted)
stat_handler.incr('properties_set', summary.counters.properties_set)
stat_handler.incr('relationships_created',
summary.counters.relationships_created)
stat_handler.incr('relationships_deleted',
summary.counters.relationships_deleted)
return result
def parse_ou(tx: neo4j.Transaction, ou: dict):
"""Parses a single ou.
Arguments:
tx {neo4j.Transaction} -- Neo4j session
ou {dict} -- Single ou object.
"""
identifier = ou['ObjectIdentifier'].upper()
property_query = 'UNWIND $props AS prop MERGE (n:Base {objectid: prop.source}) ON MATCH SET n:OU ON CREATE SET n:OU SET n += prop.map'
props = {'map': ou['Properties'], 'source': identifier}
tx.run(property_query, props=props)
if 'Aces' in ou and ou['Aces'] is not None:
process_ace_list(ou['Aces'], identifier, "OU", tx)
options = [
('Users', 'User', 'Contains'),
('Computers', 'Computer', 'Contains'),
('ChildOus', 'OU', 'Contains'),
]
for option, member_type, edge_name in options:
if option in ou and ou[option]:
targets = ou[option]
for target in targets:
query = build_add_edge_query('OU', member_type, edge_name,
'{isacl: false}')
tx.run(query, props=dict(source=identifier, target=target))
if 'Links' in ou and ou['Links']:
query = build_add_edge_query(
'GPO', 'OU', 'GpLink', '{isacl: false, enforced: prop.enforced}')
for gpo in ou['Links']:
tx.run(query,
props=dict(source=identifier,
target=gpo['Guid'].upper(),
enforced=gpo['IsEnforced']))
options = [
('LocalAdmins', 'AdminTo'),
('PSRemoteUsers', 'CanPSRemote'),
('DcomUsers', 'ExecuteDCOM'),
('RemoteDesktopUsers', 'CanRDP'),
]
for option, edge_name in options:
if option in ou and ou[option]:
targets = ou[option]
for target in targets:
query = build_add_edge_query(target['MemberType'], 'Computer',
edge_name,
'{isacl: false, fromgpo: true}')
for computer in ou['Computers']:
tx.run(query,
props=dict(target=computer,
source=target['MemberId']))
def get_infector(txn: Transaction, chat_id: int) -> str:
'''Get chatID of the infector'''
query = """
MATCH (infector:Person)-[]->(:Person { chatID: $chat_id })
RETURN infector.chatID
"""
result: str = txn.run(query, chat_id=chat_id).single().value()
return result
def get_referrer(txn: Transaction, chat_id: int) -> str:
'''Get referrer of a user.'''
query = """
MATCH (user:Person { chatID: $chat_id })
RETURN user.referrer
"""
result: str = txn.run(query, chat_id=chat_id).single().value()
return result
def _load_ec2_instance_net_if_tx(
tx: neo4j.Transaction,
instance_data: Dict[str, Any],
update_tag: int,
) -> None:
query = """
MATCH (instance:EC2Instance{instanceid: {InstanceId}})
UNWIND {Interfaces} as interface
MERGE (nic:NetworkInterface{id: interface.NetworkInterfaceId})
ON CREATE SET nic.firstseen = timestamp()
SET nic.status = interface.Status,
nic.mac_address = interface.MacAddress,
nic.description = interface.Description,
nic.private_dns_name = interface.PrivateDnsName,
nic.private_ip_address = interface.PrivateIpAddress,
nic.lastupdated = {update_tag}
MERGE (instance)-[r:NETWORK_INTERFACE]->(nic)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {update_tag}
WITH nic, interface
WHERE interface.SubnetId IS NOT NULL
MERGE (subnet:EC2Subnet{subnetid: interface.SubnetId})
ON CREATE SET subnet.firstseen = timestamp()
SET subnet.lastupdated = {update_tag}
MERGE (nic)-[r:PART_OF_SUBNET]->(subnet)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {update_tag}
WITH nic, interface
UNWIND interface.Groups as group
MATCH (ec2group:EC2SecurityGroup{groupid: group.GroupId})
MERGE (nic)-[r:MEMBER_OF_EC2_SECURITY_GROUP]->(ec2group)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {update_tag}
"""
tx.run(
query,
Interfaces=instance_data['NetworkInterfaces'],
InstanceId=instance_data['InstanceId'],
update_tag=update_tag,
)
def _load_ec2_subnet_tx(tx: neo4j.Transaction, instanceid: str, subnet_id: str,
region: str, update_tag: int) -> None:
query = """
MATCH (instance:EC2Instance{id: {InstanceId}})
MERGE (subnet:EC2Subnet{subnetid: {SubnetId}})
ON CREATE SET subnet.firstseen = timestamp()
SET subnet.region = {Region},
subnet.lastupdated = {update_tag}
MERGE (instance)-[r:PART_OF_SUBNET]->(subnet)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {update_tag}
"""
tx.run(
query,
InstanceId=instanceid,
SubnetId=subnet_id,
Region=region,
update_tag=update_tag,
)
def parse_user(tx: neo4j.Transaction, user: dict):
"""Parse a user object.
Arguments:
tx {neo4j.Transaction} -- Neo4j session
user {dict} -- Single user object from the bloodhound json.
"""
identifier = user['ObjectIdentifier']
property_query = 'UNWIND $props AS prop MERGE (n:Base {objectid: prop.source}) ON MATCH SET n:User ON CREATE SET n:User SET n += prop.map'
props = {'map': user['Properties'], 'source': identifier}
tx.run(property_query, props=props)
if 'PrimaryGroupSid' in user and user['PrimaryGroupSid']:
query = build_add_edge_query('User', 'Group', 'MemberOf',
'{isacl: false}')
tx.run(query,
props=dict(source=identifier, target=user['PrimaryGroupSid']))
if 'AllowedToDelegate' in user and user['AllowedToDelegate']:
query = build_add_edge_query('User', 'Computer', 'AllowedToDelegate',
'{isacl: false}')
for entry in user['AllowedToDelegate']:
tx.run(query, props=dict(source=identifier, target=entry))
# TODO add HasSIDHistory objects
if 'Aces' in user and user['Aces'] is not None:
process_ace_list(user['Aces'], identifier, "User", tx)
def create_node(thx: Transaction, chat_id: int) -> str:
'''Create a patient zero in the DB.'''
query = """
CREATE (newUser:Person {
chatID: $chat_id,
referrer: apoc.create.uuid(),
infectedFromDate: date()
})
RETURN newUser.referrer
"""
result: str = thx.run(query, chat_id=chat_id).single().value()
return result
def parse_group(tx: neo4j.Transaction, group: dict):
"""Parse a group object.
Arguments:
tx {neo4j.Transaction} -- Neo4j Transaction
group {dict} -- Single group object from the bloodhound json.
"""
properties = group['Properties']
identifier = group['ObjectIdentifier']
members = group['Members']
property_query = 'UNWIND $props AS prop MERGE (n:Base {objectid: prop.source}) ON MATCH SET n:Group ON CREATE SET n:Group SET n += prop.map'
props = {'map': properties, 'source': identifier}
tx.run(property_query, props=props)
if 'Aces' in group and group['Aces'] is not None:
process_ace_list(group['Aces'], identifier, "Group", tx)
for member in members:
query = build_add_edge_query(member['MemberType'], 'Group', 'MemberOf',
'{isacl: false}')
tx.run(query, props=dict(source=member['MemberId'], target=identifier))
def link_attribute(tx: Transaction, origin_uuid: int,
attributes: Dict) -> bool:
"""
After creating a node describing the current BDObject (e.g BDFunction, BDBasicBlock, BDInstruction etc)
we link the node to the various attributes describing it.
:param tx: a neo4j tx Transaction object used to communicate with DB.
:param origin_uuid: uuid of the node describing the current BDObject.
:param attributes: a dict containing attributes in the form of {attr_name: attr_features}
:return: success or failure (if one attr fails to link, the function fails)
"""
for attr_name, attr_features in attributes.items():
attr_name: str
attr_features: dict
# ident_features is used to identify an existing attribute node in order to speed up the merge process
attribute_uuid = attr_features.pop('uuid', None)
if attribute_uuid:
ident_features = {'uuid': attribute_uuid}
else:
ident_features = attr_features
cypher_statement = f'CALL apoc.merge.node({{attr_name}}, ' \
f'{{ident_features}}, ' \
f'{{attr_features}} ' \
f') ' \
f'yield node '
cypher_statement += f'MATCH (origin {{uuid: $origin_uuid}}) '
cypher_statement += f'CREATE (origin)-[r:Attribute]->(node) ' \
f'RETURN r'
r = tx.run(cypher_statement,
attr_name=[attr_name],
attr_features=attr_features,
ident_features=ident_features,
origin_uuid=origin_uuid).single()
if r:
continue
else:
log.log_debug(
f'Failed to link attribute {attr_name} to node with uuid {origin_uuid}'
)
return False
log.log_debug(
f'successfully linked all attributes to node with uuid {origin_uuid}'
)
return True
def _load_tags_tx(
tx: neo4j.Transaction,
tag_data: Dict,
resource_type: str,
region: str,
current_aws_account_id: str,
aws_update_tag: int,
) -> None:
INGEST_TAG_TEMPLATE = Template("""
UNWIND {TagData} as tag_mapping
UNWIND tag_mapping.Tags as input_tag
MATCH
(a:AWSAccount{id:{Account}})-[res:RESOURCE]->(resource:$resource_label{$property:tag_mapping.resource_id})
MERGE
(aws_tag:AWSTag:Tag{id:input_tag.Key + ":" + input_tag.Value})
ON CREATE SET aws_tag.firstseen = timestamp()
SET aws_tag.lastupdated = {UpdateTag},
aws_tag.key = input_tag.Key,
aws_tag.value = input_tag.Value,
aws_tag.region = {Region}
MERGE (resource)-[r:TAGGED]->(aws_tag)
SET r.lastupdated = {UpdateTag},
r.firstseen = timestamp()
""")
query = INGEST_TAG_TEMPLATE.safe_substitute(
resource_label=TAG_RESOURCE_TYPE_MAPPINGS[resource_type]['label'],
property=TAG_RESOURCE_TYPE_MAPPINGS[resource_type]['property'],
)
tx.run(
query,
TagData=tag_data,
UpdateTag=aws_update_tag,
Region=region,
Account=current_aws_account_id,
)
def query(self, query: str, params: dict = None, transaction: Transaction = None):
"""Execute a query on the Graph database.
Args:
query (str): The query to execute.
Returns:
obj: The query result.
"""
if transaction:
results = transaction.run(query, params)
else:
db = self.connect()
results = db.run(query, params)
return results
def create_node_from(thx: Transaction, chat_id: int,
referrer: str) -> BoltStatementResult:
'''Create an infected in the DB.'''
query = """
MATCH (infector:Person { referrer: $referrer })
CREATE (newUser:Person {
chatID: $chat_id,
referrer: apoc.create.uuid(),
infectedFromDate: date()
}), (infector)-[:INFECTED]->(newUser)
RETURN newUser.referrer
"""
result: BoltStatementResult = thx.run(query,
chat_id=chat_id,
referrer=referrer)
return result
def eco_transitions_query(tx: Transaction,
model_id: str) -> Iterable[TransitionRule]:
"""Query the graph for all possible ecological transitions.
Args:
tx: Database transaction object.
model_id: Name of the specific model version for which to extract the
ecological transitions from the database.
Returns:
Iterable over the complete set of transition rules for the model.
"""
results = tx.run(
"MATCH (lct1:LandCoverType)<-[:SOURCE]-(t:SuccessionTrajectory) "
"-[:TARGET]->(lct2:LandCoverType) "
"WHERE lct1.model_ID=$model_ID AND lct1.code<>lct2.code "
"WITH lct1, lct2, t MATCH (e:EnvironCondition)-[:CAUSES]->(t) "
"RETURN lct1.code as start_state, e.succession as succession_pathway, "
" e.aspect as aspect, e.pine as pine_seeds, e.oak as oak_seeds, "
" e.deciduous as deciduous_seeds, e.water as water, "
" lct2.code as target_state, e.delta_t as transition_time;",
model_ID=model_id)
return [convert_record_to_transition_rule(record) for record in results]
def _execute_statement(self,
stmt: str,
tx: Transaction,
params: bool = None,
expect_result: bool = False) -> Transaction:
"""
Executes statement against Neo4j. If execution fails, it rollsback and raise exception.
If 'expect_result' flag is True, it confirms if result object is not null.
:param stmt:
:param tx:
:param count:
:param expect_result: By having this True, it will validate if result object is not None.
:return:
"""
try:
if LOGGER.isEnabledFor(logging.DEBUG):
LOGGER.debug('Executing statement: {} with params {}'.format(
stmt, params))
result = tx.run(str(stmt).encode('utf-8', 'ignore'),
parameters=params)
if expect_result and not result.single():
raise RuntimeError(
'Failed to executed statement: {}'.format(stmt))
self._count += 1
if self._count > 1 and self._count % self._transaction_size == 0:
tx.commit()
LOGGER.info('Committed {} statements so far'.format(
self._count))
return self._session.begin_transaction()
if self._count > 1 and self._count % self._progress_report_frequency == 0:
LOGGER.info('Processed {} statements so far'.format(
self._count))
return tx
except Exception as e:
LOGGER.exception('Failed to execute Cypher query')
if not tx.closed():
tx.rollback()
raise e
def get_sparse_vector(self, tx: Transaction,
current_id: str) -> Dict[int, float]:
params = {"id": current_id}
result = tx.run(self.sparse_vector_query, params)
return dict(result.values())
def add_constraints(tx: neo4j.Transaction):
"""Adds bloodhound contraints to neo4j
Arguments:
tx {neo4j.Transaction} -- Neo4j transaction.
"""
tx.run("CREATE CONSTRAINT ON (c:User) ASSERT c.name IS UNIQUE")
tx.run("CREATE CONSTRAINT ON (c:Computer) ASSERT c.name IS UNIQUE")
tx.run("CREATE CONSTRAINT ON (c:Group) ASSERT c.name IS UNIQUE")
tx.run("CREATE CONSTRAINT ON (c:Domain) ASSERT c.name IS UNIQUE")
tx.run("CREATE CONSTRAINT ON (c:OU) ASSERT c.guid IS UNIQUE")
tx.run("CREATE CONSTRAINT ON (c:GPO) ASSERT c.name IS UNIQUE")
def _run(self, tx: neo4j.Transaction) -> neo4j.StatementResult:
"""
Non-iterative statement execution.
"""
return tx.run(self.query, self.parameters)
def parse_domain(tx: neo4j.Transaction, domain: dict):
"""Parse a domain object.
Arguments:
tx {neo4j.Transaction} -- Neo4j Transaction
domain {dict} -- Single domain object from the bloodhound json.
"""
identifier = domain['ObjectIdentifier']
property_query = 'UNWIND $props AS prop MERGE (n:Base {objectid: prop.source}) ON MATCH SET n:Domain ON CREATE SET n:Domain SET n += prop.map'
props = {'map': domain['Properties'], 'source': identifier}
tx.run(property_query, props=props)
if 'Aces' in domain and domain['Aces'] is not None:
process_ace_list(domain['Aces'], identifier, 'Domain', tx)
trust_map = {
0: 'ParentChild',
1: 'CrossLink',
2: 'Forest',
3: 'External',
4: 'Unknown'
}
if 'Trusts' in domain and domain['Trusts'] is not None:
query = build_add_edge_query(
'Domain', 'Domain', 'TrustedBy',
'{sidfiltering: prop.sidfiltering, trusttype: prop.trusttype, transitive: prop.transitive, isacl: false}'
)
for trust in domain['Trusts']:
trust_type = trust['TrustType']
direction = trust['TrustDirection']
props = {}
if direction in [1, 3]:
props = dict(
source=identifier,
target=trust['TargetDomainSid'],
trusttype=trust_map[trust_type],
transitive=trust['IsTransitive'],
sidfiltering=trust['SidFilteringEnabled'],
)
elif direction in [2, 4]:
props = dict(
target=identifier,
source=trust['TargetDomainSid'],
trusttype=trust_map[trust_type],
transitive=trust['IsTransitive'],
sidfiltering=trust['SidFilteringEnabled'],
)
else:
logging.error(
"Could not determine direction of trust... direction: %s",
direction)
continue
tx.run(query, props=props)
options = [
('Users', 'User', 'Contains'),
('Computers', 'Computer', 'Contains'),
('ChildOus', 'OU', 'Contains'),
]
for option, member_type, edge_name in options:
if option in domain and domain[option]:
targets = domain[option]
for target in targets:
query = build_add_edge_query('OU', member_type, edge_name,
'{isacl: false}')
tx.run(query, props=dict(source=identifier, target=target))
if 'Links' in domain and domain['Links']:
query = build_add_edge_query(
'GPO', 'OU', 'GpLink', '{isacl: false, enforced: prop.enforced}')
for gpo in domain['Links']:
tx.run(query,
props=dict(source=identifier,
target=gpo['Guid'].upper(),
enforced=gpo['IsEnforced']))
options = [
('LocalAdmins', 'AdminTo'),
('PSRemoteUsers', 'CanPSRemote'),
('DcomUsers', 'ExecuteDCOM'),
('RemoteDesktopUsers', 'CanRDP'),
]
for option, edge_name in options:
if option in domain and domain[option]:
targets = domain[option]
for target in targets:
query = build_add_edge_query(target['MemberType'], 'Computer',
edge_name,
'{isacl: false, fromgpo: true}')
for computer in domain['Computers']:
tx.run(query,
props=dict(target=computer,
source=target['MemberId']))
def add_constraints(tx: neo4j.Transaction):
"""Adds bloodhound contraints to neo4j
Arguments:
tx {neo4j.Transaction} -- Neo4j transaction.
"""
tx.run(
'CREATE CONSTRAINT base_objectid_unique ON (b:Base) ASSERT b.objectid IS UNIQUE'
)
tx.run(
'CREATE CONSTRAINT computer_objectid_unique ON (c:Computer) ASSERT c.objectid IS UNIQUE'
)
tx.run(
'CREATE CONSTRAINT domain_objectid_unique ON (d:Domain) ASSERT d.objectid IS UNIQUE'
)
tx.run(
'CREATE CONSTRAINT group_objectid_unique ON (g:Group) ASSERT g.objectid IS UNIQUE'
)
tx.run(
'CREATE CONSTRAINT user_objectid_unique ON (u:User) ASSERT u.objectid IS UNIQUE'
)
tx.run("CREATE CONSTRAINT ON (c:User) ASSERT c.name IS UNIQUE")
tx.run("CREATE CONSTRAINT ON (c:Computer) ASSERT c.name IS UNIQUE")
tx.run("CREATE CONSTRAINT ON (c:Group) ASSERT c.name IS UNIQUE")
tx.run("CREATE CONSTRAINT ON (c:Domain) ASSERT c.name IS UNIQUE")
tx.run("CREATE CONSTRAINT ON (c:OU) ASSERT c.guid IS UNIQUE")
tx.run("CREATE CONSTRAINT ON (c:GPO) ASSERT c.name IS UNIQUE")
def _load_ec2_instance_tx(
tx: neo4j.Transaction,
instanceid: str,
instance: Dict[str, Any],
reservation_id: str,
monitoring_state: str,
launch_time: datetime,
launch_time_unix: str,
instance_state: str,
current_aws_account_id: str,
region: str,
update_tag: int,
) -> None:
query = """
MERGE (instance:Instance:EC2Instance{id: {InstanceId}})
ON CREATE SET instance.firstseen = timestamp()
SET instance.instanceid = {InstanceId},
instance.publicdnsname = {PublicDnsName},
instance.privateipaddress = {PrivateIpAddress},
instance.publicipaddress = {PublicIpAddress},
instance.imageid = {ImageId},
instance.instancetype = {InstanceType},
instance.monitoringstate = {MonitoringState},
instance.state = {State},
instance.launchtime = {LaunchTime},
instance.launchtimeunix = {LaunchTimeUnix},
instance.region = {Region},
instance.lastupdated = {update_tag},
instance.iaminstanceprofile = {IamInstanceProfile},
instance.availabilityzone = {AvailabilityZone},
instance.tenancy = {Tenancy},
instance.hostresourcegrouparn = {HostResourceGroupArn},
instance.platform = {Platform},
instance.architecture = {Architecture},
instance.ebsoptimized = {EbsOptimized},
instance.bootmode = {BootMode},
instance.instancelifecycle = {InstanceLifecycle},
instance.hibernationoptions = {HibernationOptions}
WITH instance
MATCH (rez:EC2Reservation{reservationid: {ReservationId}})
MERGE (instance)-[r:MEMBER_OF_EC2_RESERVATION]->(rez)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {update_tag}
WITH instance
MATCH (aa:AWSAccount{id: {AWS_ACCOUNT_ID}})
MERGE (aa)-[r:RESOURCE]->(instance)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {update_tag}
"""
tx.run(
query,
InstanceId=instanceid,
PublicDnsName=instance.get("PublicDnsName"),
PublicIpAddress=instance.get("PublicIpAddress"),
PrivateIpAddress=instance.get("PrivateIpAddress"),
ImageId=instance.get("ImageId"),
InstanceType=instance.get("InstanceType"),
IamInstanceProfile=instance.get("IamInstanceProfile", {}).get("Arn"),
ReservationId=reservation_id,
MonitoringState=monitoring_state,
LaunchTime=str(launch_time),
LaunchTimeUnix=launch_time_unix,
State=instance_state,
AvailabilityZone=instance.get("Placement", {}).get("AvailabilityZone"),
Tenancy=instance.get("Placement", {}).get("Tenancy"),
HostResourceGroupArn=instance.get("Placement",
{}).get("HostResourceGroupArn"),
Platform=instance.get("Platform"),
Architecture=instance.get("Architecture"),
EbsOptimized=instance.get("EbsOptimized"),
BootMode=instance.get("BootMode"),
InstanceLifecycle=instance.get("InstanceLifecycle"),
HibernationOptions=instance.get("HibernationOptions",
{}).get("Configured"),
AWS_ACCOUNT_ID=current_aws_account_id,
Region=region,
update_tag=update_tag,
)
