async def run_abilities_on_agent(self, blue_agent, red_agent_pid, original_pid, op_type): facts = [ Fact(trait='host.process.id', value=original_pid), Fact(trait='sysmon.time.range', value=self.search_time_range) ] links = [] relationships = [] for ability_id in self.abilities: if ability_id == self.child_process_ability_id: depth = self.get_config(prop='child_process_recursion_depth', name='response') if not depth: depth = 5 ability_facts, ability_links, ability_relationships = \ await self.find_child_processes(blue_agent, ability_id, original_pid, relationships, op_type, depth) else: ability_facts, ability_links, ability_relationships = \ await self.run_ability_on_agent(blue_agent, red_agent_pid, ability_id, facts, original_pid, relationships, op_type) if ability_id == self.collect_guid_ability_id: for link in ability_links: await self.add_link_to_process_tree(link, top_level=True) links.extend(ability_links) facts.extend(ability_facts) relationships.extend(ability_relationships) return facts, links
def test_no_duplicate_fact(self, loop, knowledge_svc): loop.run_until_complete(knowledge_svc.add_fact(Fact(trait='test', value='demo', score=1, collected_by='thin_air', technique_id='T1234'))) loop.run_until_complete(knowledge_svc.add_fact(Fact(trait='test', value='demo', score=1, collected_by='thin_air', technique_id='T1234'))) facts = loop.run_until_complete(knowledge_svc.get_facts(dict(trait='test'))) assert len(facts) == 1
async def test_trait_with_multiple_variations_of_parts( self, setup_planning_test, planning_svc): _, agent, operation, ability = setup_planning_test encoded_command = BaseWorld.encode_string('#{a} #{a.b} #{a.b.c}') link = Link.load( dict(command=encoded_command, paw=agent.paw, ability=ability, executor=next(ability.executors), status=0)) input_facts = [ Fact(trait='a', value='1'), Fact(trait='a.b', value='2'), Fact(trait='a.b.c', value='3'), Fact(trait='server', value='5') ] new_links = await planning_svc.add_test_variants([link], agent, facts=input_facts) assert len(new_links) == 2 found_commands = set(x.command for x in new_links) assert len(found_commands) == 2 # the original and the replaced assert encoded_command in found_commands assert BaseWorld.encode_string('1 2 3') in found_commands
async def test_global_variables_not_replaced_with_facts( self, setup_planning_test, planning_svc): _, agent, operation, ability = setup_planning_test encoded_command = BaseWorld.encode_string( '#{server} #{origin_link_id}') link = Link.load( dict(command=encoded_command, paw=agent.paw, ability=ability, executor=next(ability.executors), status=0)) input_facts = [ Fact(trait='server', value='bad.server'), Fact(trait='origin_link_id', value='bad.origin_link_id') ] planning_svc.add_global_variable_owner(Agent) # handles #{server} planning_svc.add_global_variable_owner( Link) # handles #{origin_link_id} new_links = await planning_svc.add_test_variants([link], agent, facts=input_facts) assert len(new_links) == 1 assert new_links[0].raw_command == f'{agent.server} {link.id}'
def test_link_knowledge_svc_synchronization(self, loop, executor, ability, knowledge_svc): test_executor = executor(name='psh', platform='windows') test_ability = ability(ability_id='123', executors=[test_executor]) fact = Fact(trait='remote.host.fqdn', value='dc') fact2 = Fact(trait='domain.user.name', value='Bob') relationship = Relationship(source=fact, edge='has_admin', target=fact2) test_link = Link(command='echo "this was a triumph"', paw='123456', ability=test_ability, id=111111, executor=test_executor) loop.run_until_complete( test_link._create_relationships([relationship], None)) checkable = [(x.trait, x.value) for x in test_link.facts] assert (fact.trait, fact.value) in checkable assert (fact2.trait, fact2.value) in checkable knowledge_base_f = loop.run_until_complete( knowledge_svc.get_facts(dict(source=test_link.id))) assert len(knowledge_base_f) == 2 assert test_link.id in knowledge_base_f[0].links knowledge_base_r = loop.run_until_complete( knowledge_svc.get_relationships(dict(edge='has_admin'))) assert len(knowledge_base_r) == 1
async def find_child_processes(self, blue_agent, ability_id, original_pid, relationships, op_type, depth=5): process_tree_links = [] ability_facts = [] ability_relationships = [] parent_guids = [await self._get_original_guid(original_pid, relationships)] child_guids = [] count = 1 while parent_guids and count <= depth: for pguid in parent_guids: facts = [Fact(trait='host.process.guid', value=pguid), Fact(trait='sysmon.time.range', value=self.search_time_range)] links = await self.rest_svc.task_agent_with_ability(paw=blue_agent.paw, ability_id=ability_id, obfuscator='plain-text', facts=facts) await self.save_to_operation(links, op_type) await self.wait_for_link_completion(links, blue_agent) for link in links: ability_facts.extend(link.facts) ability_relationships.extend(link.relationships) link.pin = int(original_pid) child_guids.extend(await self.process_child_process_links(links)) process_tree_links.extend(links) parent_guids = child_guids child_guids = [] count += 1 return ability_facts, process_tree_links, ability_relationships
def test_retrieve_relationship(self, loop, knowledge_svc): dummy = Fact(trait='ttest', value='tdemo', score=1, collected_by='thin_air', technique_id='T1234') dummy2 = Fact(trait='ttest2', value='tdemo2', score=1, collected_by='thin_air', technique_id='T1234') loop.run_until_complete( knowledge_svc.add_relationship( Relationship(source=dummy, edge='tpotato', target=dummy2))) loop.run_until_complete( knowledge_svc.add_relationship( Relationship(source=dummy2, edge='tpotato', target=dummy))) relationships = loop.run_until_complete( knowledge_svc.get_relationships(dict(edge='tpotato'))) assert len(relationships) == 2 specific = loop.run_until_complete( knowledge_svc.get_relationships(dict(source=dummy))) assert len(specific) == 1 readable = specific[0].display assert readable['edge'] == 'tpotato' assert readable['target'].trait == 'ttest2'
async def test_trim_links(self, setup_planning_test, planning_svc): """ This test covers both remove_links_with_unset_variables and remove_links_missing_requirements. It uses a fact set that causes add_test_variants to create three links. One of which is the original that has not been populated with facts, this one gets pruned off by remove_links_with_unset_variables. Of the remaining two links that are populated, one is pruned off by a requirement that requires that the character 0 is in the link's command. The tests show that only one link is returned by trim_links and that the returned link is the one that is populated and adheres to the requirement. """ ability, agent, operation, _ = setup_planning_test link = Link.load(dict(command=BaseWorld.encode_string(test_string), paw=agent.paw, ability=ability, executor=next(ability.executors), status=0)) facts = [ Fact(trait='1_2_3', value='0'), Fact(trait='1_2_3', value='4'), Fact(trait='a.b.c', value='1'), Fact(trait='a.b.d', value='2'), Fact(trait='a.b.e', value='3'), ] operation.all_facts = async_wrapper(facts) operation.planner = MagicMock() planning_svc.load_module = async_wrapper(RequirementFake()) link.ability.requirements = [Requirement('fake_requirement', [{'fake': 'relationship'}])] trimmed_links = await planning_svc.trim_links(operation, [link], agent) assert len(trimmed_links) == 1 assert BaseWorld.decode_bytes(trimmed_links[0].display['command']) == target_string
def parse(self, blob): relationships = [] loaded = json.loads(blob) # Do not parse facts if the result is an array (multiple results returned). # This prevents facts from being parsed when results are directly returned from elasticat, # allowing them to be parsed and added to pseudo-links created for the results. This # restriction is present because a fact can not exist on more than one link in an operation at # a time. if isinstance(loaded, dict): event = loaded for mp in self.mappers: try: match = self.parse_options[mp.target.split('.').pop()](event) if match: guid = self.parse_process_guid(event) relationships.append(Relationship(source=Fact(mp.source, guid), edge=mp.edge, target=Fact(mp.target, match))) except Exception as e: self.logger.debug('Problem with mapper: %s - %s ' % (mp, e), exc_info=True) relationships.extend(self.parse_elasticsearch_results(event)) return relationships
def test_get_links(self, loop, setup_planning_test, planning_svc, data_svc): # PART A: Don't fill in facts for "cability" so only "tability" # is returned in "links" tability, agent, operation, cability = setup_planning_test operation.adversary.atomic_ordering = ["123", "321"] links = loop.run_until_complete(planning_svc.get_links (operation=operation, buckets=None, agent=agent)) assert links[0].ability.ability_id == tability.ability_id # PART B: Fill in facts to allow "cability" to be returned in "links" # in addition to "tability" operation.add_link(Link.load( dict(command='', paw=agent.paw, ability=tability, status=0))) operation.chain[0].facts.append(Fact(trait='1_2_3', value='0')) operation.chain[0].facts.append(Fact(trait='a.b.c', value='1')) operation.chain[0].facts.append(Fact(trait='a.b.d', value='2')) operation.chain[0].facts.append(Fact(trait='a.b.e', value='3')) links = loop.run_until_complete(planning_svc.get_links (operation=operation, buckets=None, agent=agent)) assert links[0].ability.ability_id == cability.ability_id assert links[1].ability.ability_id == tability.ability_id assert base64.b64decode(links[0].command).decode('utf-8') == target_string
def test_save_discover_seeded_fact_not_in_command(self, event_loop, ability, executor, operation, knowledge_svc): test_executor = executor(name='psh', platform='windows') test_ability = ability(ability_id='123', executors=[test_executor]) fact1 = Fact(trait='remote.host.fqdn', value='dc') fact2 = Fact(trait='domain.user.name', value='Bob') relationship = Relationship(source=fact1, edge='has_user', target=fact2) link = Link(command='net user', paw='123456', ability=test_ability, id='111111', executor=test_executor) operation = operation(name='test-op', agents=[], adversary=Adversary(name='sample', adversary_id='XYZ', atomic_ordering=[], description='test'), source=Source(id='test-source', facts=[fact1, fact2])) event_loop.run_until_complete(operation._init_source()) event_loop.run_until_complete( link.save_fact(operation, fact2, 1, relationship)) assert fact2.origin_type == OriginType.SEEDED assert '123456' in fact2.collected_by
def test_update_relationship(self, loop, knowledge_svc): dummy = Fact(trait='utest', value='udemo', score=1, collected_by='thin_air', technique_id='T1234') dummy2 = Fact(trait='utest2', value='udemo2', score=1, collected_by='thin_air', technique_id='T4321') loop.run_until_complete(knowledge_svc.add_relationship(Relationship(source=dummy, edge='upotato', target=dummy))) loop.run_until_complete(knowledge_svc.update_relationship(criteria=dict(edge='upotato'), updates=dict(source=dummy2, edge='ubacon'))) relationships = loop.run_until_complete(knowledge_svc.get_relationships(dict(edge='ubacon'))) assert len(relationships) == 1 assert relationships[0].source == dummy2
def test_link_neq(self, ability): test_ability = ability(ability_id='123') fact_a = Fact(trait='host.user.name', value='a') fact_b = Fact(trait='host.user.name', value='b') test_link_a = Link(command='net user a', paw='123456', ability=test_ability, id=111111) test_link_a.used = [fact_a] test_link_b = Link(command='net user b', paw='123456', ability=test_ability, id=222222) test_link_b.used = [fact_b] assert test_link_a != test_link_b
def parse(self, blob): IPs = [] for ip in re.findall(r'\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b', blob): if self._is_valid_ip(ip): for mp in self.mappers: IPs.append(Relationship(source=Fact(trait=mp.source, value=ip), edge=mp.edge, target=Fact(mp.target, ''))) return IPs
def test_remove_fact(self, loop, knowledge_svc): loop.run_until_complete(knowledge_svc.add_fact(Fact(trait='rtest', value='rdemo', score=1, collected_by='thin_air', technique_id='T1234'), constraints=dict(test_field='test_value'))) loop.run_until_complete(knowledge_svc.add_fact(Fact(trait='ktest', value='rdemo', score=1, collected_by='thin_air', technique_id='T1234'))) loop.run_until_complete(knowledge_svc.delete_fact(dict(trait='rtest'))) facts = loop.run_until_complete(knowledge_svc.get_facts(dict(value='rdemo'))) assert len(facts) == 1 assert len(knowledge_svc._KnowledgeService__loaded_knowledge_module.fact_ram['constraints']) == 0
def test_goals_satisfied(self): test_goal1 = Goal(target='target', value='value', count=1) test_goal2 = Goal(target='target2', value='value2', count=1) test_facta = Fact(trait='target', value='value') test_factb = Fact(trait='target2', value='value2') multi = Objective(id='123', name='test', goals=[test_goal1, test_goal2]) assert multi.completed([test_facta]) is False assert multi.completed([test_facta, test_factb]) is True
def test_retrieve_fact(self, loop, knowledge_svc): loop.run_until_complete(knowledge_svc.add_fact(Fact(trait='ttestA', value='tdemoB', score=24, collected_by='thin_airA', technique_id='T1234'))) loop.run_until_complete(knowledge_svc.add_fact(Fact(trait='ttestB', value='tdemoA', score=42, collected_by='thin_airB', technique_id='T4321'))) facts = loop.run_until_complete(knowledge_svc.get_facts(dict(trait='ttestB'))) assert len(facts) == 1 readable = facts[0].display assert readable['value'] == 'tdemoA' assert readable['score'] == 42
def parse(self, blob): relationships = [] for match in self.line(blob): values = match.split(':') for mp in self.mappers: relationships.append( Relationship(source=Fact(mp.source, values[0]), edge=mp.edge, target=Fact(mp.target, values[1]))) return relationships
def test_link_fact_coverage(self, loop, setup_planning_test, planning_svc): _, agent, operation, ability = setup_planning_test link = Link.load(dict(command=BaseWorld.encode_string(test_string), paw=agent.paw, ability=ability, status=0)) f1 = Fact(trait='a.b.c', value='1') f2 = Fact(trait='a.b.d', value='2') f3 = Fact(trait='a.b.e', value='3') gen = loop.run_until_complete(planning_svc.add_test_variants([link], agent, facts=[f1, f2, f3])) assert len(gen) == 2 assert BaseWorld.decode_bytes(gen[1].display['command']) == target_string
def parse(self, blob): relationships = [] for match in self._get_network_names(blob): for mp in self.mappers: source = self.set_value(mp.source, match, self.used_facts) target = self.set_value(mp.target, match, self.used_facts) relationships.append( Relationship(source=Fact(mp.source, source), edge=mp.edge, target=Fact(mp.target, target))) return relationships
def parse(self, blob): relationships = [] fqdn, ip = self.nslookup_parser(blob) for mp in self.mappers: source = self.set_value(mp.source, fqdn, self.used_facts) target = self.set_value(mp.target, ip, self.used_facts) relationships.append( Relationship(source=Fact(mp.source, source), edge=mp.edge, target=Fact(mp.target, target))) return relationships
def parse(self, blob): relationships = [] vm_names = self._get_vm_names(blob) for name in vm_names: for mp in self.mappers: relationships.append( Relationship(source=Fact(mp.source, name), edge=mp.edge, target=Fact(mp.target, None)) ) return relationships
def parse(self, blob): relationships = [] for ssh_cmd in re.findall(r'ssh.* (\w.+@\w.+)', blob): for mp in self.mappers: source = self.set_value(mp.source, ssh_cmd, self.used_facts) target = self.set_value(mp.target, ssh_cmd, self.used_facts) relationships.append( Relationship(source=Fact(mp.source, source), edge=mp.edge, target=Fact(mp.target, target))) return relationships
def test_fact_origin(self, loop, knowledge_svc, ability, executor): texecutor = executor(name='sh', platform='darwin', command='mkdir test', cleanup='rm -rf test') tability = ability(ability_id='123', executors=[texecutor], repeatable=True, buckets=['test']) link = Link.load( dict(command='', paw='n1234', ability=tability, executor=next(tability.executors), status=0, id='ganymede')) type1_fact = Fact(trait='t1', value='d1', score=1, collected_by=['thin_air'], technique_id='T1234', links=[link.id], origin_type=OriginType.LEARNED) type2_fact = Fact(trait='t2', value='d2', score=1, collected_by=['thin_air'], technique_id='T1234', links=[link.id], origin_type=OriginType.LEARNED) type3_fact = Fact( trait='t3', value='d3', score=1, collected_by=['tiny_lightning_bolts_running_through_sand'], technique_id='T1234', origin_type=OriginType.SEEDED, source="Europa") loop.run_until_complete(knowledge_svc.add_fact(type1_fact)) loop.run_until_complete(knowledge_svc.add_fact(type2_fact)) loop.run_until_complete(knowledge_svc.add_fact(type3_fact)) origin_1, type_1 = loop.run_until_complete( knowledge_svc.get_fact_origin(type1_fact)) origin_2, type_2 = loop.run_until_complete( knowledge_svc.get_fact_origin(type2_fact.trait)) origin_3, type_3 = loop.run_until_complete( knowledge_svc.get_fact_origin(type3_fact.trait)) assert origin_1 == link.id assert origin_2 == link.id assert origin_3 == 'Europa' assert type_1 == OriginType.LEARNED assert type_2 == OriginType.LEARNED assert type_3 == OriginType.SEEDED
def parse(self, blob): relationships = [] for match in self.line(blob.strip()): for mp in self.mappers: strings = match.split('>') source = strings[0].strip() target = strings[1].strip() relationships.append( Relationship(source=Fact(mp.source, source), edge=mp.edge, target=Fact(mp.target, target))) return relationships
def parse(self, blob): relationships = [] for match in self.load_json(blob): for mp in self.mappers: source = self.set_value(mp.source, match['pid'], self.used_facts) target = self.set_value(mp.target, match['port'], self.used_facts) relationships.append( Relationship(source=Fact(mp.source, source), edge=mp.edge, target=Fact(mp.target, target))) return relationships
def parse(self, blob): relationships = [] all_facts = self.used_facts for mp in self.mappers: matches = self.parse_options[mp.target.split('.').pop()](blob) for match in matches: src_fact_value = [f.value for f in all_facts if f.trait == mp.source].pop() r = Relationship(source=Fact(mp.source, src_fact_value), edge=mp.edge, target=Fact(mp.target, match)) relationships.append(r) all_facts.append(r.target) return relationships
def test_build_relationships(self, loop, setup_learning_service, learning_svc): _, link = setup_learning_service learning_svc.model.add(frozenset({'host.user.name', 'target.org.name'})) learning_svc.model.add(frozenset({'host.file.extension', 'host.user.name', 'domain.user.name'})) facts = [ Fact(trait='target.org.name', value='something'), Fact(trait='host.user.name', value='admin'), Fact(trait='host.user.name', value='root'), Fact(trait='domain.user.name', value='user'), Fact(trait='not.really.here', value='should never be found') ] loop.run_until_complete(learning_svc._build_relationships(link, facts)) assert len(link.relationships) == 4
def parse_elasticsearch_results(cls, event): elasticsearch_id = event['_id'] relationships = [] for k, v in cls.flatten_dict(event["_source"]).items(): relationships.append( Relationship( source=Fact(trait='elasticsearch.result.id', value=elasticsearch_id), target=Fact( trait=cls._sanitize_fact_traits(k), value=v if isinstance(v, str) else json.dumps(v)), edge='has_property')) return relationships
def parse(self, blob): relationships = [] try: parse_data = self.nbt_parser(blob) for match in parse_data: for mp in self.mappers: relationships.append( Relationship(source=Fact(mp.source, match), edge=mp.edge, target=Fact(mp.target, None))) except Exception: pass return relationships