def derive(strings): values = set() observables = set() for string in iterify(strings): if string: try: t = Observable.guess_type(string) observable = t(value=string) observable.normalize() observables.add(observable) values.add(observable.value) except ObservableValidationError: values.add(string) new = [] for observable in observables: for a in analyzers.get(observable.__class__, []): new.extend([ n for n in a.analyze_string(observable.value) if n and n not in values ]) if len(new) == 0: return values, values else: _, extended = derive(new + list(values)) return values, extended
def tag(self, new_tags, strict=False): new_tags = iterify(new_tags) if strict: remove = set([t.name for t in self.tags]) - set(new_tags) for tag in remove: self.modify(pull__tags__name=tag) for new_tag in new_tags: if new_tag.strip() != '': new_tag = Tag(name=new_tag) new_tag.clean() try: # check if tag is a replacement tag = Tag.objects.get(replaces=new_tag.name) except DoesNotExist: tag = Tag.get_or_create(name=new_tag.name) # search for related entities and link them for e in Entity.objects(tags__in=[tag.name]): self.link_to(e, 'Tagged', 'tags') if not self.modify({"tags__name": tag.name}, set__tags__S__fresh=True, set__tags__S__last_seen=datetime.now()): self.modify(push__tags=ObservableTag(name=tag.name)) tag.modify(inc__count=1) return self.reload()
def tag(self, new_tags, strict=False, expiration=None): """Tags an observable. An observable can be tagged to add more information as to what it represents. Args: new_tags: An array of strings to tag the observable with. strict: Set to ``True`` to replace all existing tags with the ``new_tags``. expiration: Timedelta field after which the Tag will not be considered fresh anymore. Returns: A fresh Observable instance as reloaded from the database. """ new_tags = iterify(new_tags) if strict: remove = set([t.name for t in self.tags]) - set(new_tags) for tag in remove: self.modify(pull__tags__name=tag) tagged = False for new_tag in new_tags: if new_tag.strip() != '': tagged = True new_tag = Tag(name=new_tag) new_tag.clean() try: # check if tag is a replacement tag = Tag.objects.get(replaces=new_tag.name) except DoesNotExist: tag = Tag.get_or_create(name=new_tag.name) if not expiration: expiration = tag.default_expiration extra_tags = tag.produces + [tag] # search for related entities and link them for e in Entity.objects(tags__in=[tag.name]): self.active_link_to(e, 'Tagged', 'tags', clean_old=False) for tag in extra_tags: if not self.modify( {"tags__name": tag.name}, set__tags__S__fresh=True, set__tags__S__last_seen=datetime.utcnow()): self.modify( push__tags=ObservableTag( name=tag.name, expiration=expiration)) tag.modify(inc__count=1) if tagged: self.update(set__last_tagged=datetime.utcnow()) return self.reload()
def inner(*args, **kwargs): # a user needs at least one of the roles to be granted access for r in iterify(roles[0]): if current_user.has_role(r): return f(*args, **kwargs) else: abort(401)
def derive(observables): """Indicate that the module needs a specific attribute to work properly. This function is only useful in abstract modules, in order to make sure that modules that inherit from this class correctly defines needed class attributes. Args: variables: a string or an array of strings containing the name of needed class attributes. Raises: ModuleInitializationError: One of the needed attributes is not correctly defined. """ new = [] observables = list(iterify(observables)) for i, observable in enumerate(observables): try: t = Observable.guess_type(observable) temp = t(value=observable) temp.clean() observable = temp.value observables[i] = observable for a in analyzers.get(t, []): new.extend([n for n in a.analyze_string(observable) if n and n not in observables]) except ObservableValidationError: pass if len(new) == 0: return observables else: return observables + derive(new)
def derive(observables): """Indicate that the module needs a specific attribute to work properly. This function is only useful in abstract modules, in order to make sure that modules that inherit from this class correctly defines needed class attributes. Args: variables: a string or an array of strings containing the name of needed class attributes. Raises: ModuleInitializationError: One of the needed attributes is not correctly defined. """ new = [] for observable in iterify(observables): try: t = Observable.guess_type(observable) for a in analyzers.get(t, []): new.extend([ n for n in a.analyze_string(observable) if n and n not in observables ]) except ObservableValidationError: pass if len(new) == 0: return observables else: return derive(new + observables)
def info(self): i = {k: v for k, v in self._data.items() if k in ["name", "description", "last_run", "enabled", "status"]} i['frequency'] = str(self.frequency) i['expiration'] = str(self.EXPIRATION) i['acts_on'] = iterify(self.ACTS_ON) i['id'] = str(self.id) return i
def multidelete(self): data = loads(request.data) ids = iterify(data['ids']) for i, inv in enumerate(Investigation.objects(links__id__in=ids)): inv.modify({"links__id": id}, set__links__S__id="local-{}-{}".format(time.time(), i)) self.objectmanager.objects(id__in=ids).delete() return render({"deleted": ids})
def post_save(cls, sender, document, created): if issubclass(sender, Observable): if created: for analytics in cls.analytics.itervalues(): if analytics.enabled and sender.__name__ in iterify( analytics.ACTS_ON): analytics.each(document)
def tag(self, new_tags, strict=False, expiration=None): """Tags an observable. An observable can be tagged to add more information as to what it represents. Args: new_tags: An array of strings to tag the observable with. strict: Set to ``True`` to replace all existing tags with the ``new_tags``. expiration: Timedelta field after which the Tag will not be considered fresh anymore. Returns: A fresh Observable instance as reloaded from the database. """ new_tags = iterify(new_tags) if strict: remove = set([t.name for t in self.tags]) - set(new_tags) for tag in remove: self.modify(pull__tags__name=tag) tagged = False for new_tag in new_tags: if new_tag.strip() != '': tagged = True new_tag = Tag(name=new_tag) new_tag.clean() try: # check if tag is a replacement tag = Tag.objects.get(replaces=new_tag.name) except DoesNotExist: tag = Tag.get_or_create(name=new_tag.name) if not expiration: expiration = tag.default_expiration extra_tags = tag.produces + [tag] # search for related entities and link them for e in Entity.objects(tags__in=[tag.name]): self.active_link_to(e, 'Tagged', 'tags', clean_old=False) for tag in extra_tags: if not self.modify( {"tags__name": tag.name}, set__tags__S__fresh=True, set__tags__S__last_seen=datetime.utcnow()): self.modify(push__tags=ObservableTag( name=tag.name, expiration=expiration)) tag.modify(inc__count=1) if tagged: self.update(set__last_tagged=datetime.utcnow()) return self.reload()
def post_save(cls, sender, document, created): if issubclass(sender, Observable): if getattr(document, 'new', False): for analytics in cls.analytics.values(): if analytics.enabled and sender.__name__ in iterify( analytics.ACTS_ON): document.new = False analytics.each(document)
def info(self): i = { k: v for k, v in self._data.items() if k in ["name", "description", "enabled"] } i['acts_on'] = iterify(self.ACTS_ON) i['id'] = str(self.id) return i
def multidelete(self): """Deletes multiple entries from the database :query [ObjectID] ids: Array of Element IDs :>json [ObjectID] deleted: Array of Element IDs that were successfully deleted """ data = loads(request.data) ids = iterify(data['ids']) self.objectmanager.objects(id__in=ids).delete() return render({"deleted": ids})
def link_to(self, nodes, description, source, first_seen=None, last_seen=None): links = set() nodes = iterify(nodes) for node in nodes: link = Link.connect(self, node) link.add_history(source, description, first_seen, last_seen) links.add(link) return list(links)
def inner(*args, **kwargs): oname = object_name if not oname: oname = getattr(args[0], 'klass', getattr(args[0], 'objectmanager', args[0].__class__)).__name__.lower() # a user must have all permissions in order to be granted access for p in iterify(permissions): if not current_user.has_permission(oname, p): # improve this and make it redirect to login abort(401) else: return f(*args, **kwargs)
def analyze_outdated(self): class_filter = Q() for acts_on in iterify(self.ACTS_ON): class_filter |= Q(_cls__contains=acts_on) # do outdated logic fltr = Q(**{"last_analyses__{}__exists".format(self.name): False}) if self.EXPIRATION: fltr |= Q(**{"last_analyses__{}__lte".format(self.name): datetime.now() - self.EXPIRATION}) fltr &= Q(**self.CUSTOM_FILTER) & class_filter self.bulk(Observable.objects(fltr))
def analyze_outdated(self): class_filter = Q() for acts_on in iterify(self.ACTS_ON): class_filter |= Q(_cls="Observable.{}".format(acts_on)) # do outdated logic fltr = Q(**{"last_analyses__{}__exists".format(self.name): False}) if self.EXPIRATION: fltr |= Q(**{"last_analyses__{}__lte".format(self.name): datetime.utcnow() - self.EXPIRATION}) fltr &= self.CUSTOM_FILTER & class_filter self.bulk(Observable.objects(fltr).no_cache())
def multiupdate(self): """Updates multiple entries from the database :query [ObjectID] ids: Array of Element IDs :query [Object] new: JSON object representing fields to update :>json [ObjectID] updated: Array of Element IDs that were successfully updated """ data = loads(request.data) ids = iterify(data['ids']) new_data = data['new'] self.objectmanager.objects(id__in=ids).update(new_data) return render({"updated": list(self.objectmanager.objects(ids__in=ids))})
def change_all_tags(old_tags, new_tag): """Changes tags on all observables Args: old_tags: A string or array of strings representing tag names to change new_tag: The new tag name by which all ``old_tags`` should be replaced """ old_tags = iterify(old_tags) for o in Observable.objects(tags__name__in=old_tags): for old_tag in old_tags: o.change_tag(old_tag, new_tag)
def derive(observables): new = [] for observable in iterify(observables): try: t = Observable.guess_type(observable) for a in analyzers.get(t, []): new.extend([n for n in a.analyze_string(observable) if n and n not in observables]) except ObservableValidationError: pass if len(new) == 0: return observables else: return derive(new + observables)
def link_from_data(observable, data, path, klass, description): data = get_value_at(data, path) if data is None: return [] links = set() for value in iterify(data): try: node = klass.get_or_create(value=value) except FieldDoesNotExist: node = klass.get_or_create(name=value) links.update(observable.active_link_to(node, description, 'DomainTools')) return list(links)
def active_link_to(self, nodes, description, source, clean_old=True): links = set() nodes = iterify(nodes) for node in nodes: link = Link.connect(self, node) link.add_history(source, description, active=True) links.add(link) if clean_old: for link, node in self.outgoing(): if node not in nodes: active_link = link.get_active(description) if active_link: active_link.active = False link.save(validate=False) return list(links)
def post_save(cls, sender, document, created): if issubclass(sender, Observable): if created: for analytics in cls.analytics.itervalues(): if analytics.enabled and sender.__name__ in iterify(analytics.ACTS_ON): analytics.each(document)
def add(self, investigation): investigation.add(iterify(request.json["links"]), iterify(request.json["nodes"])) return render(investigation.info())
def add_replaces(self, tags): self.replaces += list(set(iterify(tags) + self.replaces)) return self.save()
def remove(self, id): i = get_object_or_404(self.objectmanager, id=id) data = loads(request.data) i.remove(iterify(data['links']), iterify(data['nodes'])) return render(i.info())
def untag(self, tags): for tag in iterify(tags): self.modify(pull__tags__name=tag)
def info(self): i = {k: v for k, v in self._data.items() if k in ["name", "description", "enabled"]} i['acts_on'] = iterify(self.ACTS_ON) i['id'] = str(self.id) return i
def add_produces(self, tags): tags = [Tag.get_or_create(name=t) for t in iterify(tags)] self.produces = tags return self.save()
def add(self, id): i = get_object_or_404(self.objectmanager, id=id) data = loads(request.data) i.add(iterify(data["links"]), iterify(data["nodes"])) return render(i.info())
def change_all_tags(old_tags, new_tag): for o in Observable.objects(tags__name__in=iterify(old_tags)): for old_tag in old_tags: o.change_tag(old_tag, new_tag)
def add_produces(self, tags): self.produces = [Tag.get_or_create(name=t) for t in iterify(tags)] return self.save()