async def create_source(self, report): def add_fact(fact_list, trait, value): fact_list.append(Fact(trait, value, collected_by='pathfinder')) return fact_list[-1:][0] if not report: return None facts = [] relationships = [] for host in report.hosts.values(): ip_fact = add_fact(facts, 'scan.host.ip', host.ip) if host.hostname: relationships.append( Relationship( ip_fact, 'has_hostname', add_fact(facts, 'scan.host.hostname', host.hostname), ) ) for num, port in host.ports.items(): port_fact = add_fact(facts, 'scan.host.port', num) for cve_ in port.cves: cve_fact = add_fact(facts, 'scan.found.cve', cve_) relationships.append( Relationship(ip_fact, 'has_vulnerability', cve_fact) ) relationships.append( Relationship(port_fact, 'has_vulnerability', cve_fact) ) source = Source(report.id, report.name, facts, relationships) source.access = BaseWorld.Access.RED await self.data_svc.store(source) return source
def test_retrieve_relationship(self, loop, knowledge_svc): dummy = Fact(trait='ttest', value='tdemo', score=1, collected_by='thin_air', technique_id='T1234') dummy2 = Fact(trait='ttest2', value='tdemo2', score=1, collected_by='thin_air', technique_id='T1234') loop.run_until_complete( knowledge_svc.add_relationship( Relationship(source=dummy, edge='tpotato', target=dummy2))) loop.run_until_complete( knowledge_svc.add_relationship( Relationship(source=dummy2, edge='tpotato', target=dummy))) relationships = loop.run_until_complete( knowledge_svc.get_relationships(dict(edge='tpotato'))) assert len(relationships) == 2 specific = loop.run_until_complete( knowledge_svc.get_relationships(dict(source=dummy))) assert len(specific) == 1 readable = specific[0].display assert readable['edge'] == 'tpotato' assert readable['target'].trait == 'ttest2'
async def test_no_duplicate_relationship(self, knowledge_svc): dummy = Fact(trait='test', value='demo', score=1, collected_by=['thin_air'], technique_id='T1234') await knowledge_svc.add_relationship( Relationship(source=dummy, edge='potato', target=dummy)) await knowledge_svc.add_relationship( Relationship(source=dummy, edge='potato', target=dummy)) relationships = await knowledge_svc.get_relationships( dict(edge='potato')) assert len(relationships) == 1
def test_no_duplicate_relationship(self, loop, knowledge_svc): dummy = Fact(trait='test', value='demo', score=1, collected_by='thin_air', technique_id='T1234') loop.run_until_complete( knowledge_svc.add_relationship( Relationship(source=dummy, edge='potato', target=dummy))) loop.run_until_complete( knowledge_svc.add_relationship( Relationship(source=dummy, edge='potato', target=dummy))) relationships = loop.run_until_complete( knowledge_svc.get_relationships(dict(edge='potato'))) assert len(relationships) == 1
def test_link_knowledge_svc_synchronization(self, loop, executor, ability, knowledge_svc): test_executor = executor(name='psh', platform='windows') test_ability = ability(ability_id='123', executors=[test_executor]) fact = Fact(trait='remote.host.fqdn', value='dc') fact2 = Fact(trait='domain.user.name', value='Bob') relationship = Relationship(source=fact, edge='has_admin', target=fact2) test_link = Link(command='echo "this was a triumph"', paw='123456', ability=test_ability, id=111111, executor=test_executor) loop.run_until_complete( test_link._create_relationships([relationship], None)) checkable = [(x.trait, x.value) for x in test_link.facts] assert (fact.trait, fact.value) in checkable assert (fact2.trait, fact2.value) in checkable knowledge_base_f = loop.run_until_complete( knowledge_svc.get_facts(dict(source=test_link.id))) assert len(knowledge_base_f) == 2 assert test_link.id in knowledge_base_f[0].links knowledge_base_r = loop.run_until_complete( knowledge_svc.get_relationships(dict(edge='has_admin'))) assert len(knowledge_base_r) == 1
async def _store_results(self, link, facts, operation=None): facts_covered = [] for relationship in self.model: matches = [] for fact in facts: if fact.trait in relationship: matches.append(fact) facts_covered.append(fact) else: await link._save_fact(operation=operation, fact=fact, score=link.score, relationship=None) for pair in itertools.combinations(matches, r=2): if pair[0].trait != pair[1].trait: await link._create_relationships([ Relationship( source=pair[0], edge='has', target=pair[1]) ], operation=operation) # make sure we always record all the facts, even if there isn't a model set, or it would slip through otherwise e_facts = link.facts if operation: e_facts = await operation.all_facts() for f in [x for x in facts if x not in facts_covered]: await self._save_fact(link, e_facts, f, operation)
async def add_relationships(self, request: web.Request): knowledge_svc_handle = self._api_manager.knowledge_svc relationship_data = await self._api_manager.extract_data(request) try: origin_target = WILDCARD_STRING new_relationship = Relationship.load(relationship_data) if 'origin' in relationship_data: origin_target = relationship_data['origin'] else: new_relationship.origin = origin_target shorthand = new_relationship.shorthand new_relationship.source.relationships = [shorthand] new_relationship.source.source = origin_target new_relationship.source.origin_type = OriginType.USER if 'target' in relationship_data: new_relationship.target.source = origin_target new_relationship.target.origin_type = OriginType.USER new_relationship.target.relationships = [shorthand] await knowledge_svc_handle.add_fact(new_relationship.target) await knowledge_svc_handle.add_fact(new_relationship.source) await knowledge_svc_handle.add_relationship(new_relationship) store = await knowledge_svc_handle.get_relationships( criteria=dict(source=new_relationship.source, edge=new_relationship.edge if 'edge' in relationship_data else None, target=new_relationship.target if 'target' in relationship_data else None, origin=origin_target)) resp = await self._api_manager.verify_relationship_integrity(store) return web.json_response(dict(added=resp)) except Exception as e: error_msg = f'Encountered issue saving relationship {relationship_data} - {e}' self.log.warning(error_msg) raise JsonHttpBadRequest(error_msg)
def parse(self, blob): relationships = [] loaded = json.loads(blob) # Do not parse facts if the result is an array (multiple results returned). # This prevents facts from being parsed when results are directly returned from elasticat, # allowing them to be parsed and added to pseudo-links created for the results. This # restriction is present because a fact can not exist on more than one link in an operation at # a time. if isinstance(loaded, dict): event = loaded for mp in self.mappers: try: match = self.parse_options[mp.target.split('.').pop()](event) if match: guid = self.parse_process_guid(event) relationships.append(Relationship(source=Fact(mp.source, guid), edge=mp.edge, target=Fact(mp.target, match))) except Exception as e: self.logger.debug('Problem with mapper: %s - %s ' % (mp, e), exc_info=True) relationships.extend(self.parse_elasticsearch_results(event)) return relationships
async def _create_ability(self, ability_id, tactic=None, technique_name=None, technique_id=None, name=None, test=None, description=None, executor=None, platform=None, cleanup=None, payloads=None, parsers=None, requirements=None, privilege=None, timeout=60, access=None, buckets=None, repeatable=False, code=None, language=None, build_target=None, variations=None): ps = [] for module in parsers: pcs = [(ParserConfig(**m)) for m in parsers[module]] ps.append(Parser(module=module, parserconfigs=pcs)) rs = [] for requirement in requirements: for module in requirement: relation = [ Relationship(source=r['source'], edge=r.get('edge'), target=r.get('target')) for r in requirement[module] ] rs.append(Requirement(module=module, relationships=relation)) ability = Ability(ability_id=ability_id, name=name, test=test, tactic=tactic, technique_id=technique_id, technique=technique_name, code=code, language=language, executor=executor, platform=platform, description=description, build_target=build_target, cleanup=cleanup, payloads=payloads, parsers=ps, requirements=rs, privilege=privilege, timeout=timeout, repeatable=repeatable, variations=variations, buckets=buckets) ability.access = access return await self.store(ability)
def test_save_discover_seeded_fact_not_in_command(self, event_loop, ability, executor, operation, knowledge_svc): test_executor = executor(name='psh', platform='windows') test_ability = ability(ability_id='123', executors=[test_executor]) fact1 = Fact(trait='remote.host.fqdn', value='dc') fact2 = Fact(trait='domain.user.name', value='Bob') relationship = Relationship(source=fact1, edge='has_user', target=fact2) link = Link(command='net user', paw='123456', ability=test_ability, id='111111', executor=test_executor) operation = operation(name='test-op', agents=[], adversary=Adversary(name='sample', adversary_id='XYZ', atomic_ordering=[], description='test'), source=Source(id='test-source', facts=[fact1, fact2])) event_loop.run_until_complete(operation._init_source()) event_loop.run_until_complete( link.save_fact(operation, fact2, 1, relationship)) assert fact2.origin_type == OriginType.SEEDED assert '123456' in fact2.collected_by
def test_remove_relationship(self, loop, knowledge_svc): dummy = Fact(trait='rtest', value='rdemo', score=1, collected_by='thin_air', technique_id='T1234') loop.run_until_complete(knowledge_svc.add_relationship(Relationship(source=dummy, edge='rpotato', target=dummy), constraints=dict(test_field='test_value'))) loop.run_until_complete(knowledge_svc.delete_relationship(dict(edge='rpotato'))) relationships = loop.run_until_complete(knowledge_svc.get_relationships(dict(edge='rpotato'))) assert len(relationships) == 0 assert len(knowledge_svc._KnowledgeService__loaded_knowledge_module.fact_ram['constraints']) == 0
async def _build_relationships(self, link, facts): for relationship in self.model: matches = [] for fact in facts: if fact.trait in relationship: matches.append(fact) for pair in itertools.combinations(matches, r=2): if pair[0].trait != pair[1].trait: link.relationships.append(Relationship(source=pair[0], edge='has', target=pair[1]))
def test_update_relationship(self, loop, knowledge_svc): dummy = Fact(trait='utest', value='udemo', score=1, collected_by='thin_air', technique_id='T1234') dummy2 = Fact(trait='utest2', value='udemo2', score=1, collected_by='thin_air', technique_id='T4321') loop.run_until_complete(knowledge_svc.add_relationship(Relationship(source=dummy, edge='upotato', target=dummy))) loop.run_until_complete(knowledge_svc.update_relationship(criteria=dict(edge='upotato'), updates=dict(source=dummy2, edge='ubacon'))) relationships = loop.run_until_complete(knowledge_svc.get_relationships(dict(edge='ubacon'))) assert len(relationships) == 1 assert relationships[0].source == dummy2
def parse(self, blob): IPs = [] for ip in re.findall(r'\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b', blob): if self._is_valid_ip(ip): for mp in self.mappers: IPs.append(Relationship(source=Fact(trait=mp.source, value=ip), edge=mp.edge, target=Fact(mp.target, ''))) return IPs
def parse(self, blob): relationships = [] for match in self.line(blob): values = match.split(':') for mp in self.mappers: relationships.append( Relationship(source=Fact(mp.source, values[0]), edge=mp.edge, target=Fact(mp.target, values[1]))) return relationships
def parse(self, blob): relationships = [] vm_names = self._get_vm_names(blob) for name in vm_names: for mp in self.mappers: relationships.append( Relationship(source=(mp.source, name), edge=mp.edge, target=(mp.target, None))) return relationships
def parse(self, blob): relationships = [] for match in self.filename(blob): for mp in self.mappers: source = self.set_value(mp.source, match, self.used_facts) target = self.set_value(mp.target, match, self.used_facts) relationships.append( Relationship(source=(mp.source, source), edge=mp.edge, target=(mp.target, target))) return relationships
def parse(self, blob): relationships = [] for ssh_cmd in re.findall(r'ssh.* (\w.+@\w.+)', blob): for mp in self.mappers: source = self.set_value(mp.source, ssh_cmd, self.used_facts) target = self.set_value(mp.target, ssh_cmd, self.used_facts) relationships.append( Relationship(source=Fact(mp.source, source), edge=mp.edge, target=Fact(mp.target, target))) return relationships
def parse(self, blob): relationships = [] fqdn, ip = self.nslookup_parser(blob) for mp in self.mappers: source = self.set_value(mp.source, fqdn, self.used_facts) target = self.set_value(mp.target, ip, self.used_facts) relationships.append( Relationship(source=(mp.source, source), edge=mp.edge, target=(mp.target, target))) return relationships
def parse(self, blob): relationships = [] for match in self.line(blob.strip()): for mp in self.mappers: strings = match.split('>') source = strings[0].strip() target = strings[1].strip() relationships.append( Relationship(source=Fact(mp.source, source), edge=mp.edge, target=Fact(mp.target, target))) return relationships
def parse(self, blob): relationships = [] events = [event for event in blob.split('\r\n\r\n') if event != ''] for event in events: for mp in self.mappers: match = self.parse_options[mp.target.split('.').pop()](event) if match: guid = [f.value for f in self.used_facts if f.trait == mp.source].pop() relationships.append(Relationship(source=(mp.source, guid), edge=mp.edge, target=(mp.target, match.group(1)))) return relationships
def parse(self, blob): relationships = [] for match in self.load_json(blob): for mp in self.mappers: source = self.set_value(mp.source, match['pid'], self.used_facts) target = self.set_value(mp.target, match['port'], self.used_facts) relationships.append( Relationship(source=Fact(mp.source, source), edge=mp.edge, target=Fact(mp.target, target))) return relationships
def parse(self, blob): relationships = [] for match in self.line(blob): if 'The command completed successfully.' in match: for mp in self.mappers: relationships.append( Relationship(source=(mp.source, self._get_remote_host(mp.source, self.used_facts)), edge=mp.edge, target=(mp.target, None)) ) # we can only have one resulting relationship in this parser type. return immediately return relationships return relationships
def parse(self, blob): relationships = [] all_facts = self.used_facts for mp in self.mappers: matches = self.parse_options[mp.target.split('.').pop()](blob) for match in matches: src_fact_value = [f.value for f in all_facts if f.trait == mp.source].pop() r = Relationship(source=Fact(mp.source, src_fact_value), edge=mp.edge, target=Fact(mp.target, match)) relationships.append(r) all_facts.append(r.target) return relationships
def parse(self, blob): relationships = [] for match in self.line(blob): port = self._locate_port(match) if port: for mp in self.mappers: source = self.set_value(mp.source, port, self.used_facts) target = self.set_value(mp.target, port, self.used_facts) relationships.append( Relationship(source=(mp.source, source), edge=mp.edge, target=(mp.target, target))) return relationships
def parse_elasticsearch_results(cls, event): elasticsearch_id = event['_id'] relationships = [] for k, v in cls.flatten_dict(event["_source"]).items(): relationships.append( Relationship( source=Fact(trait='elasticsearch.result.id', value=elasticsearch_id), target=Fact( trait=cls._sanitize_fact_traits(k), value=v if isinstance(v, str) else json.dumps(v)), edge='has_property')) return relationships
def parse(self, blob): relationships = [] for match in self.line(blob): if match.startswith(' Packets'): if '(0%' in match: for mp in self.mappers: source = self.set_value(mp.source, match, self.used_facts) relationships.append( Relationship(source=Fact(mp.source, source), edge=mp.edge, target=Fact(mp.target, None))) return relationships
def parse(self, blob): relationships = [] try: parse_data = self.nbt_parser(blob) for match in parse_data: for mp in self.mappers: relationships.append( Relationship(source=Fact(mp.source, match), edge=mp.edge, target=Fact(mp.target, None))) except Exception: pass return relationships
def test_source_existing_relationships(event_loop): test_fact_1 = Fact(trait='test_1', value='1') test_fact_2 = Fact(trait='test_2', value='2') test_relationship = Relationship(source=test_fact_1, edge='test_edge', target=test_fact_2) test_source = Source(id='123', name='test', facts=[test_fact_1, test_fact_2], adjustments=[], relationships=[test_relationship]) event_loop.run_until_complete( BaseService.get_service('data_svc').store(test_source)) return test_source
def parse(self, blob): relationships = [] for match in self.line(blob): if self.ABILITY_SUCCESS_FLAG in match: for mp in self.mappers: relationships.append( Relationship(source=(mp.source, self._get_remote_host( mp.source, self.used_facts)), edge=mp.edge, target=(mp.target, None))) # we can only have one resulting relationship in this parser type. return immediately return relationships return relationships