def tag(self, new_tags, strict=False): if isinstance(new_tags, (str, unicode)): new_tags = [new_tags] if strict: remove = set([t.name for t in self.tags]) - set(new_tags) for tag in remove: self.modify(pull__tags__name=tag) for new_tag in new_tags: if new_tag.strip() != "": new_tag = Tag(name=new_tag) new_tag.clean() try: # check if tag is a replacement tag = Tag.objects.get(replaces=new_tag.name) except DoesNotExist: tag = Tag.get_or_create(name=new_tag.name) # search for related entities and link them for e in Entity.objects(tags__in=[tag.name]): Link.connect(self, e).add_history("Tagged") if not self.modify( {"tags__name": tag.name}, set__tags__S__fresh=True, set__tags__S__last_seen=datetime.now() ): self.modify(push__tags=ObservableTag(name=tag.name)) tag.modify(inc__count=1) return self.reload()
def tag(self, new_tags, strict=False, expiration=None): """Tags an observable. An observable can be tagged to add more information as to what it represents. Args: new_tags: An array of strings to tag the observable with. strict: Set to ``True`` to replace all existing tags with the ``new_tags``. expiration: Timedelta field after which the Tag will not be considered fresh anymore. Returns: A fresh Observable instance as reloaded from the database. """ new_tags = iterify(new_tags) if strict: remove = set([t.name for t in self.tags]) - set(new_tags) for tag in remove: self.modify(pull__tags__name=tag) tagged = False for new_tag in new_tags: if new_tag.strip() != '': tagged = True new_tag = Tag(name=new_tag) new_tag.clean() try: # check if tag is a replacement tag = Tag.objects.get(replaces=new_tag.name) except DoesNotExist: tag = Tag.get_or_create(name=new_tag.name) if not expiration: expiration = tag.default_expiration extra_tags = tag.produces + [tag] # search for related entities and link them for e in Entity.objects(tags__in=[tag.name]): self.active_link_to(e, 'Tagged', 'tags', clean_old=False) for tag in extra_tags: if not self.modify( {"tags__name": tag.name}, set__tags__S__fresh=True, set__tags__S__last_seen=datetime.utcnow()): self.modify(push__tags=ObservableTag( name=tag.name, expiration=expiration)) tag.modify(inc__count=1) if tagged: self.update(set__last_tagged=datetime.utcnow()) return self.reload()
def tag(self, new_tags, strict=False, expiration=None): """Tags an observable. An observable can be tagged to add more information as to what it represents. Args: new_tags: An array of strings to tag the observable with. strict: Set to ``True`` to replace all existing tags with the ``new_tags``. expiration: Timedelta field after which the Tag will not be considered fresh anymore. Returns: A fresh Observable instance as reloaded from the database. """ new_tags = iterify(new_tags) if strict: remove = set([t.name for t in self.tags]) - set(new_tags) for tag in remove: self.modify(pull__tags__name=tag) tagged = False for new_tag in new_tags: if new_tag.strip() != '': tagged = True new_tag = Tag(name=new_tag) new_tag.clean() try: # check if tag is a replacement tag = Tag.objects.get(replaces=new_tag.name) except DoesNotExist: tag = Tag.get_or_create(name=new_tag.name) if not expiration: expiration = tag.default_expiration extra_tags = tag.produces + [tag] # search for related entities and link them for e in Entity.objects(tags__in=[tag.name]): self.active_link_to(e, 'Tagged', 'tags', clean_old=False) for tag in extra_tags: if not self.modify( {"tags__name": tag.name}, set__tags__S__fresh=True, set__tags__S__last_seen=datetime.utcnow()): self.modify( push__tags=ObservableTag( name=tag.name, expiration=expiration)) tag.modify(inc__count=1) if tagged: self.update(set__last_tagged=datetime.utcnow()) return self.reload()
def post(self, id=None): if not id: data = request.json data['produces'] = [Tag.get_or_create(name=t.strip()) for t in request.json['produces'].split(',') if t.strip()] data['replaces'] = request.json['replaces'].split(',') return render(Tag(**data).save().info()) else: try: data = request.json data['produces'] = [Tag.get_or_create(name=t.strip()) for t in request.json['produces'].split(',') if t.strip()] data['replaces'] = request.json['replaces'].split(',') t = Tag.objects.get(id=id) t.update(**data) Observable.change_all_tags(t.name, data['name']) return render({"status": "ok"}) except TagValidationError as e: restful_abort(400, error=str(e)) except Exception as e: import traceback traceback.print_exc() restful_abort(400, error='Must specify name and produces parameters')
def clean(self): tags = [] for t in self.tags: if t: tags.append(Tag.get_or_create(name=t.lower().strip())) self.tags = [t.name for t in tags]
db.drop_database('yeti') ## Populate database with initial values MalwareFamily("mailer").save() MalwareFamily("banker").save() MalwareFamily("worm").save() MalwareFamily("ransomware").save() MalwareFamily("backdoor").save() MalwareFamily("stealer").save() MalwareFamily("passwordstealer").save() MalwareFamily("rootkit").save() MalwareFamily("trojan").save() MalwareFamily("dropper").save() t1 = Tag.get_or_create(name="zeus").add_produces(["crimeware", "banker", "malware"]) t2 = Tag.get_or_create(name="banker").add_produces(["crimeware", "malware"]) t3 = Tag.get_or_create(name="c2") t3.add_replaces(["c&c", "cc"]) Tag.get_or_create(name="crimeware").add_produces("malware") Export(name="TestExport", description="Test description", frequency=timedelta(hours=1), include_tags=[t1, t2]).save() url = Observable.add_text("hxxp://zeuscpanel.com/gate.php") url.tag(["zeus", "banker", "cc", "c2"]) print url.tags # print url.find_tags()
e.save() e = Malware(name="Dyre").save() e.family = trojan e.save() e = Exploit(name="CVE-2015-3113").save() e = Malware(name="Teslacrypt").save() e.family = ransomware e.save() e = Malware(name="Alphacrypt").save() e.family = ransomware e.save() e = Malware(name="Locky").save() e.family = ransomware e.save() t1 = Tag.get_or_create(name="zeus").add_produces( ["crimeware", "banker", "malware"]) t2 = Tag.get_or_create(name="banker").add_produces(["crimeware", "malware"]) t3 = Tag.get_or_create(name="c2") t3.add_replaces(["c&c", "cc"]) Tag.get_or_create(name="crimeware").add_produces("malware") et = ExportTemplate(name="Default") et.template = "{{ obs.value }}\n" et.save() et = ExportTemplate(name="Bluecoat") et.template = """define category cert_blocklist {% for obs in elements %}{{ obs.value }} {% endfor %}end """
def analyze(self, threat_meta): threat = threat_meta['threat'] events = threat_meta['events'] log.info("%d messages blocked for threat %s", len(events), threat['threatID']) log.debug(pprint.pformat(threat)) # make tags and context context = { 'source': self.name, 'event_type': 'email_blocked', 'tlp': 'green' } tags = [ Tag.get_or_create(name=threat['threatType']), Tag.get_or_create(name=threat['classification']) ] tags = [{'name': t.name} for t in tags] # extract Url and/or Hash info from threat threat_nodes = self._make_threat_nodes(threat, context, tags) if threat_nodes is None: return # TODO verify if we want that. Indicators are probably in campaign_info # get all forensics report for this threat threat_forensics = self._get_threat_forensics_nodes( threat, threat_nodes, context, tags) if threat_forensics is not None: # attach all node to threat_nodes for _t_node in threat_nodes: _t_node.active_link_to(threat_forensics, "Drops", self.name) for _f in threat_forensics: log.debug("{threatid} Drops {forensic}".format( threatid=_t_node.value, forensic=_f.value)) # now attach each event to the threat_nodes events_node = self._add_events_nodes(events, context, tags) for email in events_node: email.active_link_to(threat_nodes, "Delivers", self.name) for _n in threat_nodes: log.debug("{email} Delivers {n}".format(email=email.value, n=_n.value)) # so now our main threat is in threats[]. # lets link to the campaign campaign, campaign_info = self._make_entities(threat) if campaign is not None: # attach this threat to the campaign log.info("Linking campaign to %d threat nodes", len(threat_nodes)) campaign.action(threat_nodes, self.name, "Delivers") # campaign_info contains campaignMembers a list of threats (urls/malwares) # unroll and fetch them too if not duplicates # fields: id, subType, threat, threatStatus, threatTime, type threats_nodes2 = self._query_and_filter_previous_new_threat_for_campaign( campaign_info, context) log.info("Linking campaign to %d new threat nodes", len(threats_nodes2)) # campaign.action(threats_nodes2, self.name, "Delivers") # even faster, dont look at clean_old campaign.active_link_to(threats_nodes2, "Delivers", self.name, clean_old=False) return
def _get_threat_forensics_nodes_inner(self, evidence, general_context, tags): # create context from notes context = general_context.copy() _ctx = self._make_context_from_notes([evidence]) context.update(_ctx) # add evidence['type'] and unicify tags tags = [{ 'name': _ } for _ in set([evidence['type']] + [d['name'] for d in tags])] # create Tags in DB for _ in tags: Tag.get_or_create(name=_['name']) # threat_forensics = [] # technical hack: set optional comments values for optional in ['action', 'rule', 'path', 'rule']: if optional not in evidence['what']: evidence['what'][optional] = None # add attributes for the known evidence type if evidence['type'] in ['file', 'dropper']: if 'path' in evidence['what']: threat_forensics.append( File.get_or_create(value=evidence['what']['path'], context=[context])) if 'md5' in evidence['what']: threat_forensics.append( Hash.get_or_create(value=evidence['what']['md5'], context=[context])) if 'sha256' in evidence['what']: threat_forensics.append( Hash.get_or_create(value=evidence['what']['sha256'], context=[context])) elif evidence['type'] == 'cookie': pass elif evidence['type'] == 'dns': threat_forensics.append( Hostname.get_or_create(value=evidence['what']['host'], context=[context])) elif evidence['type'] == 'ids': threat_forensics.append( Text.get_or_create(value=evidence['what']['ids'], context=[context])) elif evidence['type'] == 'mutex': threat_forensics.append( Text.get_or_create(value=evidence['what']['name'], context=[context])) elif evidence['type'] == 'network': if 'ip' in evidence['what']: # FIXME port, type threat_forensics.append( Ip.get_or_create(value=evidence['what']['ip'], context=[context])) elif 'domain' in evidence['what']: threat_forensics.append( Hostname.get_or_create(value=evidence['what']['domain'], context=[context])) elif evidence['type'] == 'process': pass elif evidence['type'] == 'registry': # threat_forensics.append(evidence['what']['key']) # threat_forensics.append(evidence['what']['value']) pass elif evidence['type'] == 'url': # BUG yeti-#115 ObservableValidationError: Invalid URL: http://xxxxx-no-tld/ threat_forensics.append( Url.get_or_create(value=evidence['what']['url'], context=[context])) # add note as tag because its a signature if 'note' in evidence: threat_forensics[-1].tag(evidence['note'].replace( '.', '_').strip('_')) # tag all of that for o in threat_forensics: o.tag([t['name'] for t in tags]) return threat_forensics
def _get_threat_forensics_nodes_inner(self, evidence, general_context, tags): # create context from notes context = general_context.copy() _ctx = self._make_context_from_notes([evidence]) context.update(_ctx) # add evidence['type'] and unicify tags tags = [{ "name": _ } for _ in set([evidence["type"]] + [d["name"] for d in tags])] # create Tags in DB for _ in tags: Tag.get_or_create(name=_["name"]) # threat_forensics = [] # technical hack: set optional comments values for optional in ["action", "rule", "path", "rule"]: if optional not in evidence["what"]: evidence["what"][optional] = None # add attributes for the known evidence type if evidence["type"] in ["file", "dropper"]: if "path" in evidence["what"]: threat_forensics.append( File.get_or_create(value=evidence["what"]["path"], context=[context])) if "md5" in evidence["what"]: threat_forensics.append( Hash.get_or_create(value=evidence["what"]["md5"], context=[context])) if "sha256" in evidence["what"]: threat_forensics.append( Hash.get_or_create(value=evidence["what"]["sha256"], context=[context])) elif evidence["type"] == "cookie": pass elif evidence["type"] == "dns": threat_forensics.append( Hostname.get_or_create(value=evidence["what"]["host"], context=[context])) elif evidence["type"] == "ids": threat_forensics.append( Text.get_or_create(value=evidence["what"]["ids"], context=[context])) elif evidence["type"] == "mutex": threat_forensics.append( Text.get_or_create(value=evidence["what"]["name"], context=[context])) elif evidence["type"] == "network": if "ip" in evidence["what"]: # FIXME port, type threat_forensics.append( Ip.get_or_create(value=evidence["what"]["ip"], context=[context])) elif "domain" in evidence["what"]: threat_forensics.append( Hostname.get_or_create(value=evidence["what"]["domain"], context=[context])) elif evidence["type"] == "process": pass elif evidence["type"] == "registry": # threat_forensics.append(evidence['what']['key']) # threat_forensics.append(evidence['what']['value']) pass elif evidence["type"] == "url": # BUG yeti-#115 ObservableValidationError: Invalid URL: http://xxxxx-no-tld/ threat_forensics.append( Url.get_or_create(value=evidence["what"]["url"], context=[context])) # add note as tag because its a signature if "note" in evidence: threat_forensics[-1].tag(evidence["note"].replace( ".", "_").strip("_")) # tag all of that for o in threat_forensics: o.tag([t["name"] for t in tags]) return threat_forensics
def analyze(self, threat_meta): threat = threat_meta['threat'] events = threat_meta['events'] log.info( "%d messages blocked for threat %s", len(events), threat['threatID']) log.debug(pprint.pformat(threat)) # make tags and context context = { 'source': self.name, 'event_type': 'email_blocked', 'tlp': 'green' } tags = [ Tag.get_or_create(name=threat['threatType']), Tag.get_or_create(name=threat['classification']) ] tags = [{'name': t.name} for t in tags] # extract Url and/or Hash info from threat threat_nodes = self._make_threat_nodes(threat, context, tags) if threat_nodes is None: return # TODO verify if we want that. Indicators are probably in campaign_info # get all forensics report for this threat threat_forensics = self._get_threat_forensics_nodes( threat, threat_nodes, context, tags) if threat_forensics is not None: # attach all node to threat_nodes for _t_node in threat_nodes: _t_node.active_link_to(threat_forensics, "Drops", self.name) for _f in threat_forensics: log.debug( "{threatid} Drops {forensic}".format( threatid=_t_node.value, forensic=_f.value)) # now attach each event to the threat_nodes events_node = self._add_events_nodes(events, context, tags) for email in events_node: email.active_link_to(threat_nodes, "Delivers", self.name) for _n in threat_nodes: log.debug( "{email} Delivers {n}".format( email=email.value, n=_n.value)) # so now our main threat is in threats[]. # lets link to the campaign campaign, campaign_info = self._make_entities(threat) if campaign is not None: # attach this threat to the campaign log.info("Linking campaign to %d threat nodes", len(threat_nodes)) campaign.action(threat_nodes, self.name, "Delivers") # campaign_info contains campaignMembers a list of threats (urls/malwares) # unroll and fetch them too if not duplicates # fields: id, subType, threat, threatStatus, threatTime, type threats_nodes2 = self._query_and_filter_previous_new_threat_for_campaign( campaign_info, context) log.info( "Linking campaign to %d new threat nodes", len(threats_nodes2)) # campaign.action(threats_nodes2, self.name, "Delivers") # even faster, dont look at clean_old campaign.active_link_to( threats_nodes2, "Delivers", self.name, clean_old=False) return
def _get_threat_forensics_nodes_inner( self, evidence, general_context, tags): # create context from notes context = general_context.copy() _ctx = self._make_context_from_notes([evidence]) context.update(_ctx) # add evidence['type'] and unicify tags tags = [{ 'name': _ } for _ in set([evidence['type']] + [d['name'] for d in tags])] # create Tags in DB for _ in tags: Tag.get_or_create(name=_['name']) # threat_forensics = [] # technical hack: set optional comments values for optional in ['action', 'rule', 'path', 'rule']: if optional not in evidence['what']: evidence['what'][optional] = None # add attributes for the known evidence type if evidence['type'] in ['file', 'dropper']: if 'path' in evidence['what']: threat_forensics.append( File.get_or_create( value=evidence['what']['path'], context=[context])) if 'md5' in evidence['what']: threat_forensics.append( Hash.get_or_create( value=evidence['what']['md5'], context=[context])) if 'sha256' in evidence['what']: threat_forensics.append( Hash.get_or_create( value=evidence['what']['sha256'], context=[context])) elif evidence['type'] == 'cookie': pass elif evidence['type'] == 'dns': threat_forensics.append( Hostname.get_or_create( value=evidence['what']['host'], context=[context])) elif evidence['type'] == 'ids': threat_forensics.append( Text.get_or_create( value=evidence['what']['ids'], context=[context])) pass elif evidence['type'] == 'mutex': threat_forensics.append( Text.get_or_create( value=evidence['what']['name'], context=[context])) elif evidence['type'] == 'network': if 'ip' in evidence['what']: # FIXME port, type threat_forensics.append( Ip.get_or_create( value=evidence['what']['ip'], context=[context])) elif 'domain' in evidence['what']: threat_forensics.append( Hostname.get_or_create( value=evidence['what']['domain'], context=[context])) elif evidence['type'] == 'process': pass elif evidence['type'] == 'registry': # threat_forensics.append(evidence['what']['key']) # threat_forensics.append(evidence['what']['value']) pass elif evidence['type'] == 'url': # BUG yeti-#115 ObservableValidationError: Invalid URL: http://xxxxx-no-tld/ threat_forensics.append( Url.get_or_create( value=evidence['what']['url'], context=[context])) # add note as tag because its a signature if 'note' in evidence: threat_forensics[-1].tag( evidence['note'].replace('.', '_').strip('_')) # tag all of that for o in threat_forensics: o.tag([t['name'] for t in tags]) return threat_forensics