async def run_abilities_on_agent(self, blue_agent, red_agent_pid,
original_pid, op_type):
facts = [
Fact(trait='host.process.id', value=original_pid),
Fact(trait='sysmon.time.range', value=self.search_time_range)
]
links = []
relationships = []
for ability_id in self.abilities:
if ability_id == self.child_process_ability_id:
depth = self.get_config(prop='child_process_recursion_depth',
name='response')
if not depth:
depth = 5
ability_facts, ability_links, ability_relationships = \
await self.find_child_processes(blue_agent, ability_id, original_pid, relationships, op_type, depth)
else:
ability_facts, ability_links, ability_relationships = \
await self.run_ability_on_agent(blue_agent, red_agent_pid, ability_id, facts, original_pid,
relationships, op_type)
if ability_id == self.collect_guid_ability_id:
for link in ability_links:
await self.add_link_to_process_tree(link,
top_level=True)
links.extend(ability_links)
facts.extend(ability_facts)
relationships.extend(ability_relationships)
return facts, links
def test_no_duplicate_fact(self, loop, knowledge_svc):
loop.run_until_complete(knowledge_svc.add_fact(Fact(trait='test', value='demo', score=1,
collected_by='thin_air', technique_id='T1234')))
loop.run_until_complete(knowledge_svc.add_fact(Fact(trait='test', value='demo', score=1,
collected_by='thin_air', technique_id='T1234')))
facts = loop.run_until_complete(knowledge_svc.get_facts(dict(trait='test')))
assert len(facts) == 1
async def test_trait_with_multiple_variations_of_parts(
self, setup_planning_test, planning_svc):
_, agent, operation, ability = setup_planning_test
encoded_command = BaseWorld.encode_string('#{a} #{a.b} #{a.b.c}')
link = Link.load(
dict(command=encoded_command,
paw=agent.paw,
ability=ability,
executor=next(ability.executors),
status=0))
input_facts = [
Fact(trait='a', value='1'),
Fact(trait='a.b', value='2'),
Fact(trait='a.b.c', value='3'),
Fact(trait='server', value='5')
]
new_links = await planning_svc.add_test_variants([link],
agent,
facts=input_facts)
assert len(new_links) == 2
found_commands = set(x.command for x in new_links)
assert len(found_commands) == 2 # the original and the replaced
assert encoded_command in found_commands
assert BaseWorld.encode_string('1 2 3') in found_commands
async def test_global_variables_not_replaced_with_facts(
self, setup_planning_test, planning_svc):
_, agent, operation, ability = setup_planning_test
encoded_command = BaseWorld.encode_string(
'#{server} #{origin_link_id}')
link = Link.load(
dict(command=encoded_command,
paw=agent.paw,
ability=ability,
executor=next(ability.executors),
status=0))
input_facts = [
Fact(trait='server', value='bad.server'),
Fact(trait='origin_link_id', value='bad.origin_link_id')
]
planning_svc.add_global_variable_owner(Agent) # handles #{server}
planning_svc.add_global_variable_owner(
Link) # handles #{origin_link_id}
new_links = await planning_svc.add_test_variants([link],
agent,
facts=input_facts)
assert len(new_links) == 1
assert new_links[0].raw_command == f'{agent.server} {link.id}'
def test_link_knowledge_svc_synchronization(self, loop, executor, ability,
knowledge_svc):
test_executor = executor(name='psh', platform='windows')
test_ability = ability(ability_id='123', executors=[test_executor])
fact = Fact(trait='remote.host.fqdn', value='dc')
fact2 = Fact(trait='domain.user.name', value='Bob')
relationship = Relationship(source=fact,
edge='has_admin',
target=fact2)
test_link = Link(command='echo "this was a triumph"',
paw='123456',
ability=test_ability,
id=111111,
executor=test_executor)
loop.run_until_complete(
test_link._create_relationships([relationship], None))
checkable = [(x.trait, x.value) for x in test_link.facts]
assert (fact.trait, fact.value) in checkable
assert (fact2.trait, fact2.value) in checkable
knowledge_base_f = loop.run_until_complete(
knowledge_svc.get_facts(dict(source=test_link.id)))
assert len(knowledge_base_f) == 2
assert test_link.id in knowledge_base_f[0].links
knowledge_base_r = loop.run_until_complete(
knowledge_svc.get_relationships(dict(edge='has_admin')))
assert len(knowledge_base_r) == 1
async def find_child_processes(self, blue_agent, ability_id, original_pid, relationships, op_type, depth=5):
process_tree_links = []
ability_facts = []
ability_relationships = []
parent_guids = [await self._get_original_guid(original_pid, relationships)]
child_guids = []
count = 1
while parent_guids and count <= depth:
for pguid in parent_guids:
facts = [Fact(trait='host.process.guid', value=pguid),
Fact(trait='sysmon.time.range', value=self.search_time_range)]
links = await self.rest_svc.task_agent_with_ability(paw=blue_agent.paw, ability_id=ability_id,
obfuscator='plain-text', facts=facts)
await self.save_to_operation(links, op_type)
await self.wait_for_link_completion(links, blue_agent)
for link in links:
ability_facts.extend(link.facts)
ability_relationships.extend(link.relationships)
link.pin = int(original_pid)
child_guids.extend(await self.process_child_process_links(links))
process_tree_links.extend(links)
parent_guids = child_guids
child_guids = []
count += 1
return ability_facts, process_tree_links, ability_relationships
def test_retrieve_relationship(self, loop, knowledge_svc):
dummy = Fact(trait='ttest',
value='tdemo',
score=1,
collected_by='thin_air',
technique_id='T1234')
dummy2 = Fact(trait='ttest2',
value='tdemo2',
score=1,
collected_by='thin_air',
technique_id='T1234')
loop.run_until_complete(
knowledge_svc.add_relationship(
Relationship(source=dummy, edge='tpotato', target=dummy2)))
loop.run_until_complete(
knowledge_svc.add_relationship(
Relationship(source=dummy2, edge='tpotato', target=dummy)))
relationships = loop.run_until_complete(
knowledge_svc.get_relationships(dict(edge='tpotato')))
assert len(relationships) == 2
specific = loop.run_until_complete(
knowledge_svc.get_relationships(dict(source=dummy)))
assert len(specific) == 1
readable = specific[0].display
assert readable['edge'] == 'tpotato'
assert readable['target'].trait == 'ttest2'
async def test_trim_links(self, setup_planning_test, planning_svc):
"""
This test covers both remove_links_with_unset_variables and remove_links_missing_requirements.
It uses a fact set that causes add_test_variants to create three links. One of which is the original
that has not been populated with facts, this one gets pruned off by remove_links_with_unset_variables.
Of the remaining two links that are populated, one is pruned off by a requirement that requires that
the character 0 is in the link's command. The tests show that only one link is returned by trim_links
and that the returned link is the one that is populated and adheres to the requirement.
"""
ability, agent, operation, _ = setup_planning_test
link = Link.load(dict(command=BaseWorld.encode_string(test_string), paw=agent.paw, ability=ability,
executor=next(ability.executors), status=0))
facts = [
Fact(trait='1_2_3', value='0'),
Fact(trait='1_2_3', value='4'),
Fact(trait='a.b.c', value='1'),
Fact(trait='a.b.d', value='2'),
Fact(trait='a.b.e', value='3'),
]
operation.all_facts = async_wrapper(facts)
operation.planner = MagicMock()
planning_svc.load_module = async_wrapper(RequirementFake())
link.ability.requirements = [Requirement('fake_requirement', [{'fake': 'relationship'}])]
trimmed_links = await planning_svc.trim_links(operation, [link], agent)
assert len(trimmed_links) == 1
assert BaseWorld.decode_bytes(trimmed_links[0].display['command']) == target_string
def parse(self, blob):
relationships = []
loaded = json.loads(blob)
# Do not parse facts if the result is an array (multiple results returned).
# This prevents facts from being parsed when results are directly returned from elasticat,
# allowing them to be parsed and added to pseudo-links created for the results. This
# restriction is present because a fact can not exist on more than one link in an operation at
# a time.
if isinstance(loaded, dict):
event = loaded
for mp in self.mappers:
try:
match = self.parse_options[mp.target.split('.').pop()](event)
if match:
guid = self.parse_process_guid(event)
relationships.append(Relationship(source=Fact(mp.source, guid),
edge=mp.edge,
target=Fact(mp.target, match)))
except Exception as e:
self.logger.debug('Problem with mapper: %s - %s ' % (mp, e), exc_info=True)
relationships.extend(self.parse_elasticsearch_results(event))
return relationships
def test_get_links(self, loop, setup_planning_test, planning_svc, data_svc):
# PART A: Don't fill in facts for "cability" so only "tability"
# is returned in "links"
tability, agent, operation, cability = setup_planning_test
operation.adversary.atomic_ordering = ["123", "321"]
links = loop.run_until_complete(planning_svc.get_links
(operation=operation, buckets=None,
agent=agent))
assert links[0].ability.ability_id == tability.ability_id
# PART B: Fill in facts to allow "cability" to be returned in "links"
# in addition to "tability"
operation.add_link(Link.load(
dict(command='', paw=agent.paw, ability=tability, status=0)))
operation.chain[0].facts.append(Fact(trait='1_2_3', value='0'))
operation.chain[0].facts.append(Fact(trait='a.b.c', value='1'))
operation.chain[0].facts.append(Fact(trait='a.b.d', value='2'))
operation.chain[0].facts.append(Fact(trait='a.b.e', value='3'))
links = loop.run_until_complete(planning_svc.get_links
(operation=operation, buckets=None,
agent=agent))
assert links[0].ability.ability_id == cability.ability_id
assert links[1].ability.ability_id == tability.ability_id
assert base64.b64decode(links[0].command).decode('utf-8') == target_string
def test_save_discover_seeded_fact_not_in_command(self, event_loop,
ability, executor,
operation,
knowledge_svc):
test_executor = executor(name='psh', platform='windows')
test_ability = ability(ability_id='123', executors=[test_executor])
fact1 = Fact(trait='remote.host.fqdn', value='dc')
fact2 = Fact(trait='domain.user.name', value='Bob')
relationship = Relationship(source=fact1,
edge='has_user',
target=fact2)
link = Link(command='net user',
paw='123456',
ability=test_ability,
id='111111',
executor=test_executor)
operation = operation(name='test-op',
agents=[],
adversary=Adversary(name='sample',
adversary_id='XYZ',
atomic_ordering=[],
description='test'),
source=Source(id='test-source',
facts=[fact1, fact2]))
event_loop.run_until_complete(operation._init_source())
event_loop.run_until_complete(
link.save_fact(operation, fact2, 1, relationship))
assert fact2.origin_type == OriginType.SEEDED
assert '123456' in fact2.collected_by
def test_update_relationship(self, loop, knowledge_svc):
dummy = Fact(trait='utest', value='udemo', score=1, collected_by='thin_air', technique_id='T1234')
dummy2 = Fact(trait='utest2', value='udemo2', score=1, collected_by='thin_air', technique_id='T4321')
loop.run_until_complete(knowledge_svc.add_relationship(Relationship(source=dummy, edge='upotato', target=dummy)))
loop.run_until_complete(knowledge_svc.update_relationship(criteria=dict(edge='upotato'),
updates=dict(source=dummy2, edge='ubacon')))
relationships = loop.run_until_complete(knowledge_svc.get_relationships(dict(edge='ubacon')))
assert len(relationships) == 1
assert relationships[0].source == dummy2
def test_link_neq(self, ability):
test_ability = ability(ability_id='123')
fact_a = Fact(trait='host.user.name', value='a')
fact_b = Fact(trait='host.user.name', value='b')
test_link_a = Link(command='net user a', paw='123456', ability=test_ability, id=111111)
test_link_a.used = [fact_a]
test_link_b = Link(command='net user b', paw='123456', ability=test_ability, id=222222)
test_link_b.used = [fact_b]
assert test_link_a != test_link_b
def parse(self, blob):
IPs = []
for ip in re.findall(r'\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b', blob):
if self._is_valid_ip(ip):
for mp in self.mappers:
IPs.append(Relationship(source=Fact(trait=mp.source, value=ip),
edge=mp.edge,
target=Fact(mp.target, '')))
return IPs
def test_remove_fact(self, loop, knowledge_svc):
loop.run_until_complete(knowledge_svc.add_fact(Fact(trait='rtest', value='rdemo', score=1,
collected_by='thin_air', technique_id='T1234'),
constraints=dict(test_field='test_value')))
loop.run_until_complete(knowledge_svc.add_fact(Fact(trait='ktest', value='rdemo', score=1,
collected_by='thin_air', technique_id='T1234')))
loop.run_until_complete(knowledge_svc.delete_fact(dict(trait='rtest')))
facts = loop.run_until_complete(knowledge_svc.get_facts(dict(value='rdemo')))
assert len(facts) == 1
assert len(knowledge_svc._KnowledgeService__loaded_knowledge_module.fact_ram['constraints']) == 0
def test_goals_satisfied(self):
test_goal1 = Goal(target='target', value='value', count=1)
test_goal2 = Goal(target='target2', value='value2', count=1)
test_facta = Fact(trait='target', value='value')
test_factb = Fact(trait='target2', value='value2')
multi = Objective(id='123',
name='test',
goals=[test_goal1, test_goal2])
assert multi.completed([test_facta]) is False
assert multi.completed([test_facta, test_factb]) is True
def test_retrieve_fact(self, loop, knowledge_svc):
loop.run_until_complete(knowledge_svc.add_fact(Fact(trait='ttestA', value='tdemoB', score=24,
collected_by='thin_airA', technique_id='T1234')))
loop.run_until_complete(knowledge_svc.add_fact(Fact(trait='ttestB', value='tdemoA', score=42,
collected_by='thin_airB', technique_id='T4321')))
facts = loop.run_until_complete(knowledge_svc.get_facts(dict(trait='ttestB')))
assert len(facts) == 1
readable = facts[0].display
assert readable['value'] == 'tdemoA'
assert readable['score'] == 42
def parse(self, blob):
relationships = []
for match in self.line(blob):
values = match.split(':')
for mp in self.mappers:
relationships.append(
Relationship(source=Fact(mp.source, values[0]),
edge=mp.edge,
target=Fact(mp.target, values[1])))
return relationships
def test_link_fact_coverage(self, loop, setup_planning_test, planning_svc):
_, agent, operation, ability = setup_planning_test
link = Link.load(dict(command=BaseWorld.encode_string(test_string), paw=agent.paw, ability=ability, status=0))
f1 = Fact(trait='a.b.c', value='1')
f2 = Fact(trait='a.b.d', value='2')
f3 = Fact(trait='a.b.e', value='3')
gen = loop.run_until_complete(planning_svc.add_test_variants([link], agent, facts=[f1, f2, f3]))
assert len(gen) == 2
assert BaseWorld.decode_bytes(gen[1].display['command']) == target_string
def parse(self, blob):
relationships = []
for match in self._get_network_names(blob):
for mp in self.mappers:
source = self.set_value(mp.source, match, self.used_facts)
target = self.set_value(mp.target, match, self.used_facts)
relationships.append(
Relationship(source=Fact(mp.source, source),
edge=mp.edge,
target=Fact(mp.target, target)))
return relationships
def parse(self, blob):
relationships = []
fqdn, ip = self.nslookup_parser(blob)
for mp in self.mappers:
source = self.set_value(mp.source, fqdn, self.used_facts)
target = self.set_value(mp.target, ip, self.used_facts)
relationships.append(
Relationship(source=Fact(mp.source, source),
edge=mp.edge,
target=Fact(mp.target, target)))
return relationships
def parse(self, blob):
relationships = []
vm_names = self._get_vm_names(blob)
for name in vm_names:
for mp in self.mappers:
relationships.append(
Relationship(source=Fact(mp.source, name),
edge=mp.edge,
target=Fact(mp.target, None))
)
return relationships
def parse(self, blob):
relationships = []
for ssh_cmd in re.findall(r'ssh.* (\w.+@\w.+)', blob):
for mp in self.mappers:
source = self.set_value(mp.source, ssh_cmd, self.used_facts)
target = self.set_value(mp.target, ssh_cmd, self.used_facts)
relationships.append(
Relationship(source=Fact(mp.source, source),
edge=mp.edge,
target=Fact(mp.target, target)))
return relationships
def test_fact_origin(self, loop, knowledge_svc, ability, executor):
texecutor = executor(name='sh',
platform='darwin',
command='mkdir test',
cleanup='rm -rf test')
tability = ability(ability_id='123',
executors=[texecutor],
repeatable=True,
buckets=['test'])
link = Link.load(
dict(command='',
paw='n1234',
ability=tability,
executor=next(tability.executors),
status=0,
id='ganymede'))
type1_fact = Fact(trait='t1',
value='d1',
score=1,
collected_by=['thin_air'],
technique_id='T1234',
links=[link.id],
origin_type=OriginType.LEARNED)
type2_fact = Fact(trait='t2',
value='d2',
score=1,
collected_by=['thin_air'],
technique_id='T1234',
links=[link.id],
origin_type=OriginType.LEARNED)
type3_fact = Fact(
trait='t3',
value='d3',
score=1,
collected_by=['tiny_lightning_bolts_running_through_sand'],
technique_id='T1234',
origin_type=OriginType.SEEDED,
source="Europa")
loop.run_until_complete(knowledge_svc.add_fact(type1_fact))
loop.run_until_complete(knowledge_svc.add_fact(type2_fact))
loop.run_until_complete(knowledge_svc.add_fact(type3_fact))
origin_1, type_1 = loop.run_until_complete(
knowledge_svc.get_fact_origin(type1_fact))
origin_2, type_2 = loop.run_until_complete(
knowledge_svc.get_fact_origin(type2_fact.trait))
origin_3, type_3 = loop.run_until_complete(
knowledge_svc.get_fact_origin(type3_fact.trait))
assert origin_1 == link.id
assert origin_2 == link.id
assert origin_3 == 'Europa'
assert type_1 == OriginType.LEARNED
assert type_2 == OriginType.LEARNED
assert type_3 == OriginType.SEEDED
def parse(self, blob):
relationships = []
for match in self.line(blob.strip()):
for mp in self.mappers:
strings = match.split('>')
source = strings[0].strip()
target = strings[1].strip()
relationships.append(
Relationship(source=Fact(mp.source, source),
edge=mp.edge,
target=Fact(mp.target, target)))
return relationships
def parse(self, blob):
relationships = []
for match in self.load_json(blob):
for mp in self.mappers:
source = self.set_value(mp.source, match['pid'],
self.used_facts)
target = self.set_value(mp.target, match['port'],
self.used_facts)
relationships.append(
Relationship(source=Fact(mp.source, source),
edge=mp.edge,
target=Fact(mp.target, target)))
return relationships
def parse(self, blob):
relationships = []
all_facts = self.used_facts
for mp in self.mappers:
matches = self.parse_options[mp.target.split('.').pop()](blob)
for match in matches:
src_fact_value = [f.value for f in all_facts if f.trait == mp.source].pop()
r = Relationship(source=Fact(mp.source, src_fact_value),
edge=mp.edge,
target=Fact(mp.target, match))
relationships.append(r)
all_facts.append(r.target)
return relationships
def test_build_relationships(self, loop, setup_learning_service, learning_svc):
_, link = setup_learning_service
learning_svc.model.add(frozenset({'host.user.name', 'target.org.name'}))
learning_svc.model.add(frozenset({'host.file.extension', 'host.user.name', 'domain.user.name'}))
facts = [
Fact(trait='target.org.name', value='something'),
Fact(trait='host.user.name', value='admin'),
Fact(trait='host.user.name', value='root'),
Fact(trait='domain.user.name', value='user'),
Fact(trait='not.really.here', value='should never be found')
]
loop.run_until_complete(learning_svc._build_relationships(link, facts))
assert len(link.relationships) == 4
def parse_elasticsearch_results(cls, event):
elasticsearch_id = event['_id']
relationships = []
for k, v in cls.flatten_dict(event["_source"]).items():
relationships.append(
Relationship(
source=Fact(trait='elasticsearch.result.id',
value=elasticsearch_id),
target=Fact(
trait=cls._sanitize_fact_traits(k),
value=v if isinstance(v, str) else json.dumps(v)),
edge='has_property'))
return relationships
def parse(self, blob):
relationships = []
try:
parse_data = self.nbt_parser(blob)
for match in parse_data:
for mp in self.mappers:
relationships.append(
Relationship(source=Fact(mp.source, match),
edge=mp.edge,
target=Fact(mp.target, None)))
except Exception:
pass
return relationships
