async def create_source(self, report):
def add_fact(fact_list, trait, value):
fact_list.append(Fact(trait, value, collected_by='pathfinder'))
return fact_list[-1:][0]
if not report:
return None
facts = []
relationships = []
for host in report.hosts.values():
ip_fact = add_fact(facts, 'scan.host.ip', host.ip)
if host.hostname:
relationships.append(
Relationship(
ip_fact,
'has_hostname',
add_fact(facts, 'scan.host.hostname', host.hostname),
)
)
for num, port in host.ports.items():
port_fact = add_fact(facts, 'scan.host.port', num)
for cve_ in port.cves:
cve_fact = add_fact(facts, 'scan.found.cve', cve_)
relationships.append(
Relationship(ip_fact, 'has_vulnerability', cve_fact)
)
relationships.append(
Relationship(port_fact, 'has_vulnerability', cve_fact)
)
source = Source(report.id, report.name, facts, relationships)
source.access = BaseWorld.Access.RED
await self.data_svc.store(source)
return source
def test_retrieve_relationship(self, loop, knowledge_svc):
dummy = Fact(trait='ttest',
value='tdemo',
score=1,
collected_by='thin_air',
technique_id='T1234')
dummy2 = Fact(trait='ttest2',
value='tdemo2',
score=1,
collected_by='thin_air',
technique_id='T1234')
loop.run_until_complete(
knowledge_svc.add_relationship(
Relationship(source=dummy, edge='tpotato', target=dummy2)))
loop.run_until_complete(
knowledge_svc.add_relationship(
Relationship(source=dummy2, edge='tpotato', target=dummy)))
relationships = loop.run_until_complete(
knowledge_svc.get_relationships(dict(edge='tpotato')))
assert len(relationships) == 2
specific = loop.run_until_complete(
knowledge_svc.get_relationships(dict(source=dummy)))
assert len(specific) == 1
readable = specific[0].display
assert readable['edge'] == 'tpotato'
assert readable['target'].trait == 'ttest2'
async def test_no_duplicate_relationship(self, knowledge_svc):
dummy = Fact(trait='test',
value='demo',
score=1,
collected_by=['thin_air'],
technique_id='T1234')
await knowledge_svc.add_relationship(
Relationship(source=dummy, edge='potato', target=dummy))
await knowledge_svc.add_relationship(
Relationship(source=dummy, edge='potato', target=dummy))
relationships = await knowledge_svc.get_relationships(
dict(edge='potato'))
assert len(relationships) == 1
def test_no_duplicate_relationship(self, loop, knowledge_svc):
dummy = Fact(trait='test',
value='demo',
score=1,
collected_by='thin_air',
technique_id='T1234')
loop.run_until_complete(
knowledge_svc.add_relationship(
Relationship(source=dummy, edge='potato', target=dummy)))
loop.run_until_complete(
knowledge_svc.add_relationship(
Relationship(source=dummy, edge='potato', target=dummy)))
relationships = loop.run_until_complete(
knowledge_svc.get_relationships(dict(edge='potato')))
assert len(relationships) == 1
def test_link_knowledge_svc_synchronization(self, loop, executor, ability,
knowledge_svc):
test_executor = executor(name='psh', platform='windows')
test_ability = ability(ability_id='123', executors=[test_executor])
fact = Fact(trait='remote.host.fqdn', value='dc')
fact2 = Fact(trait='domain.user.name', value='Bob')
relationship = Relationship(source=fact,
edge='has_admin',
target=fact2)
test_link = Link(command='echo "this was a triumph"',
paw='123456',
ability=test_ability,
id=111111,
executor=test_executor)
loop.run_until_complete(
test_link._create_relationships([relationship], None))
checkable = [(x.trait, x.value) for x in test_link.facts]
assert (fact.trait, fact.value) in checkable
assert (fact2.trait, fact2.value) in checkable
knowledge_base_f = loop.run_until_complete(
knowledge_svc.get_facts(dict(source=test_link.id)))
assert len(knowledge_base_f) == 2
assert test_link.id in knowledge_base_f[0].links
knowledge_base_r = loop.run_until_complete(
knowledge_svc.get_relationships(dict(edge='has_admin')))
assert len(knowledge_base_r) == 1
async def _store_results(self, link, facts, operation=None):
facts_covered = []
for relationship in self.model:
matches = []
for fact in facts:
if fact.trait in relationship:
matches.append(fact)
facts_covered.append(fact)
else:
await link._save_fact(operation=operation,
fact=fact,
score=link.score,
relationship=None)
for pair in itertools.combinations(matches, r=2):
if pair[0].trait != pair[1].trait:
await link._create_relationships([
Relationship(
source=pair[0], edge='has', target=pair[1])
],
operation=operation)
# make sure we always record all the facts, even if there isn't a model set, or it would slip through otherwise
e_facts = link.facts
if operation:
e_facts = await operation.all_facts()
for f in [x for x in facts if x not in facts_covered]:
await self._save_fact(link, e_facts, f, operation)
async def add_relationships(self, request: web.Request):
knowledge_svc_handle = self._api_manager.knowledge_svc
relationship_data = await self._api_manager.extract_data(request)
try:
origin_target = WILDCARD_STRING
new_relationship = Relationship.load(relationship_data)
if 'origin' in relationship_data:
origin_target = relationship_data['origin']
else:
new_relationship.origin = origin_target
shorthand = new_relationship.shorthand
new_relationship.source.relationships = [shorthand]
new_relationship.source.source = origin_target
new_relationship.source.origin_type = OriginType.USER
if 'target' in relationship_data:
new_relationship.target.source = origin_target
new_relationship.target.origin_type = OriginType.USER
new_relationship.target.relationships = [shorthand]
await knowledge_svc_handle.add_fact(new_relationship.target)
await knowledge_svc_handle.add_fact(new_relationship.source)
await knowledge_svc_handle.add_relationship(new_relationship)
store = await knowledge_svc_handle.get_relationships(
criteria=dict(source=new_relationship.source,
edge=new_relationship.edge if 'edge' in
relationship_data else None,
target=new_relationship.target if 'target' in
relationship_data else None,
origin=origin_target))
resp = await self._api_manager.verify_relationship_integrity(store)
return web.json_response(dict(added=resp))
except Exception as e:
error_msg = f'Encountered issue saving relationship {relationship_data} - {e}'
self.log.warning(error_msg)
raise JsonHttpBadRequest(error_msg)
def parse(self, blob):
relationships = []
loaded = json.loads(blob)
# Do not parse facts if the result is an array (multiple results returned).
# This prevents facts from being parsed when results are directly returned from elasticat,
# allowing them to be parsed and added to pseudo-links created for the results. This
# restriction is present because a fact can not exist on more than one link in an operation at
# a time.
if isinstance(loaded, dict):
event = loaded
for mp in self.mappers:
try:
match = self.parse_options[mp.target.split('.').pop()](event)
if match:
guid = self.parse_process_guid(event)
relationships.append(Relationship(source=Fact(mp.source, guid),
edge=mp.edge,
target=Fact(mp.target, match)))
except Exception as e:
self.logger.debug('Problem with mapper: %s - %s ' % (mp, e), exc_info=True)
relationships.extend(self.parse_elasticsearch_results(event))
return relationships
async def _create_ability(self,
ability_id,
tactic=None,
technique_name=None,
technique_id=None,
name=None,
test=None,
description=None,
executor=None,
platform=None,
cleanup=None,
payloads=None,
parsers=None,
requirements=None,
privilege=None,
timeout=60,
access=None,
buckets=None,
repeatable=False,
code=None,
language=None,
build_target=None,
variations=None):
ps = []
for module in parsers:
pcs = [(ParserConfig(**m)) for m in parsers[module]]
ps.append(Parser(module=module, parserconfigs=pcs))
rs = []
for requirement in requirements:
for module in requirement:
relation = [
Relationship(source=r['source'],
edge=r.get('edge'),
target=r.get('target'))
for r in requirement[module]
]
rs.append(Requirement(module=module, relationships=relation))
ability = Ability(ability_id=ability_id,
name=name,
test=test,
tactic=tactic,
technique_id=technique_id,
technique=technique_name,
code=code,
language=language,
executor=executor,
platform=platform,
description=description,
build_target=build_target,
cleanup=cleanup,
payloads=payloads,
parsers=ps,
requirements=rs,
privilege=privilege,
timeout=timeout,
repeatable=repeatable,
variations=variations,
buckets=buckets)
ability.access = access
return await self.store(ability)
def test_save_discover_seeded_fact_not_in_command(self, event_loop,
ability, executor,
operation,
knowledge_svc):
test_executor = executor(name='psh', platform='windows')
test_ability = ability(ability_id='123', executors=[test_executor])
fact1 = Fact(trait='remote.host.fqdn', value='dc')
fact2 = Fact(trait='domain.user.name', value='Bob')
relationship = Relationship(source=fact1,
edge='has_user',
target=fact2)
link = Link(command='net user',
paw='123456',
ability=test_ability,
id='111111',
executor=test_executor)
operation = operation(name='test-op',
agents=[],
adversary=Adversary(name='sample',
adversary_id='XYZ',
atomic_ordering=[],
description='test'),
source=Source(id='test-source',
facts=[fact1, fact2]))
event_loop.run_until_complete(operation._init_source())
event_loop.run_until_complete(
link.save_fact(operation, fact2, 1, relationship))
assert fact2.origin_type == OriginType.SEEDED
assert '123456' in fact2.collected_by
def test_remove_relationship(self, loop, knowledge_svc):
dummy = Fact(trait='rtest', value='rdemo', score=1, collected_by='thin_air', technique_id='T1234')
loop.run_until_complete(knowledge_svc.add_relationship(Relationship(source=dummy, edge='rpotato', target=dummy),
constraints=dict(test_field='test_value')))
loop.run_until_complete(knowledge_svc.delete_relationship(dict(edge='rpotato')))
relationships = loop.run_until_complete(knowledge_svc.get_relationships(dict(edge='rpotato')))
assert len(relationships) == 0
assert len(knowledge_svc._KnowledgeService__loaded_knowledge_module.fact_ram['constraints']) == 0
async def _build_relationships(self, link, facts):
for relationship in self.model:
matches = []
for fact in facts:
if fact.trait in relationship:
matches.append(fact)
for pair in itertools.combinations(matches, r=2):
if pair[0].trait != pair[1].trait:
link.relationships.append(Relationship(source=pair[0], edge='has', target=pair[1]))
def test_update_relationship(self, loop, knowledge_svc):
dummy = Fact(trait='utest', value='udemo', score=1, collected_by='thin_air', technique_id='T1234')
dummy2 = Fact(trait='utest2', value='udemo2', score=1, collected_by='thin_air', technique_id='T4321')
loop.run_until_complete(knowledge_svc.add_relationship(Relationship(source=dummy, edge='upotato', target=dummy)))
loop.run_until_complete(knowledge_svc.update_relationship(criteria=dict(edge='upotato'),
updates=dict(source=dummy2, edge='ubacon')))
relationships = loop.run_until_complete(knowledge_svc.get_relationships(dict(edge='ubacon')))
assert len(relationships) == 1
assert relationships[0].source == dummy2
def parse(self, blob):
IPs = []
for ip in re.findall(r'\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b', blob):
if self._is_valid_ip(ip):
for mp in self.mappers:
IPs.append(Relationship(source=Fact(trait=mp.source, value=ip),
edge=mp.edge,
target=Fact(mp.target, '')))
return IPs
def parse(self, blob):
relationships = []
for match in self.line(blob):
values = match.split(':')
for mp in self.mappers:
relationships.append(
Relationship(source=Fact(mp.source, values[0]),
edge=mp.edge,
target=Fact(mp.target, values[1])))
return relationships
def parse(self, blob):
relationships = []
vm_names = self._get_vm_names(blob)
for name in vm_names:
for mp in self.mappers:
relationships.append(
Relationship(source=(mp.source, name),
edge=mp.edge,
target=(mp.target, None)))
return relationships
def parse(self, blob):
relationships = []
for match in self.filename(blob):
for mp in self.mappers:
source = self.set_value(mp.source, match, self.used_facts)
target = self.set_value(mp.target, match, self.used_facts)
relationships.append(
Relationship(source=(mp.source, source),
edge=mp.edge,
target=(mp.target, target)))
return relationships
def parse(self, blob):
relationships = []
for ssh_cmd in re.findall(r'ssh.* (\w.+@\w.+)', blob):
for mp in self.mappers:
source = self.set_value(mp.source, ssh_cmd, self.used_facts)
target = self.set_value(mp.target, ssh_cmd, self.used_facts)
relationships.append(
Relationship(source=Fact(mp.source, source),
edge=mp.edge,
target=Fact(mp.target, target)))
return relationships
def parse(self, blob):
relationships = []
fqdn, ip = self.nslookup_parser(blob)
for mp in self.mappers:
source = self.set_value(mp.source, fqdn, self.used_facts)
target = self.set_value(mp.target, ip, self.used_facts)
relationships.append(
Relationship(source=(mp.source, source),
edge=mp.edge,
target=(mp.target, target)))
return relationships
def parse(self, blob):
relationships = []
for match in self.line(blob.strip()):
for mp in self.mappers:
strings = match.split('>')
source = strings[0].strip()
target = strings[1].strip()
relationships.append(
Relationship(source=Fact(mp.source, source),
edge=mp.edge,
target=Fact(mp.target, target)))
return relationships
def parse(self, blob):
relationships = []
events = [event for event in blob.split('\r\n\r\n') if event != '']
for event in events:
for mp in self.mappers:
match = self.parse_options[mp.target.split('.').pop()](event)
if match:
guid = [f.value for f in self.used_facts if f.trait == mp.source].pop()
relationships.append(Relationship(source=(mp.source, guid),
edge=mp.edge,
target=(mp.target, match.group(1))))
return relationships
def parse(self, blob):
relationships = []
for match in self.load_json(blob):
for mp in self.mappers:
source = self.set_value(mp.source, match['pid'],
self.used_facts)
target = self.set_value(mp.target, match['port'],
self.used_facts)
relationships.append(
Relationship(source=Fact(mp.source, source),
edge=mp.edge,
target=Fact(mp.target, target)))
return relationships
def parse(self, blob):
relationships = []
for match in self.line(blob):
if 'The command completed successfully.' in match:
for mp in self.mappers:
relationships.append(
Relationship(source=(mp.source, self._get_remote_host(mp.source, self.used_facts)),
edge=mp.edge,
target=(mp.target, None))
)
# we can only have one resulting relationship in this parser type. return immediately
return relationships
return relationships
def parse(self, blob):
relationships = []
all_facts = self.used_facts
for mp in self.mappers:
matches = self.parse_options[mp.target.split('.').pop()](blob)
for match in matches:
src_fact_value = [f.value for f in all_facts if f.trait == mp.source].pop()
r = Relationship(source=Fact(mp.source, src_fact_value),
edge=mp.edge,
target=Fact(mp.target, match))
relationships.append(r)
all_facts.append(r.target)
return relationships
def parse(self, blob):
relationships = []
for match in self.line(blob):
port = self._locate_port(match)
if port:
for mp in self.mappers:
source = self.set_value(mp.source, port, self.used_facts)
target = self.set_value(mp.target, port, self.used_facts)
relationships.append(
Relationship(source=(mp.source, source),
edge=mp.edge,
target=(mp.target, target)))
return relationships
def parse_elasticsearch_results(cls, event):
elasticsearch_id = event['_id']
relationships = []
for k, v in cls.flatten_dict(event["_source"]).items():
relationships.append(
Relationship(
source=Fact(trait='elasticsearch.result.id',
value=elasticsearch_id),
target=Fact(
trait=cls._sanitize_fact_traits(k),
value=v if isinstance(v, str) else json.dumps(v)),
edge='has_property'))
return relationships
def parse(self, blob):
relationships = []
for match in self.line(blob):
if match.startswith(' Packets'):
if '(0%' in match:
for mp in self.mappers:
source = self.set_value(mp.source, match,
self.used_facts)
relationships.append(
Relationship(source=Fact(mp.source, source),
edge=mp.edge,
target=Fact(mp.target, None)))
return relationships
def parse(self, blob):
relationships = []
try:
parse_data = self.nbt_parser(blob)
for match in parse_data:
for mp in self.mappers:
relationships.append(
Relationship(source=Fact(mp.source, match),
edge=mp.edge,
target=Fact(mp.target, None)))
except Exception:
pass
return relationships
def test_source_existing_relationships(event_loop):
test_fact_1 = Fact(trait='test_1', value='1')
test_fact_2 = Fact(trait='test_2', value='2')
test_relationship = Relationship(source=test_fact_1,
edge='test_edge',
target=test_fact_2)
test_source = Source(id='123',
name='test',
facts=[test_fact_1, test_fact_2],
adjustments=[],
relationships=[test_relationship])
event_loop.run_until_complete(
BaseService.get_service('data_svc').store(test_source))
return test_source
def parse(self, blob):
relationships = []
for match in self.line(blob):
if self.ABILITY_SUCCESS_FLAG in match:
for mp in self.mappers:
relationships.append(
Relationship(source=(mp.source,
self._get_remote_host(
mp.source, self.used_facts)),
edge=mp.edge,
target=(mp.target, None)))
# we can only have one resulting relationship in this parser type. return immediately
return relationships
return relationships
