Пример #1
0
def derive(strings):
    values = set()
    observables = set()

    for string in iterify(strings):
        if string:
            try:
                t = Observable.guess_type(string)
                observable = t(value=string)
                observable.normalize()
                observables.add(observable)
                values.add(observable.value)
            except ObservableValidationError:
                values.add(string)

    new = []
    for observable in observables:
        for a in analyzers.get(observable.__class__, []):
            new.extend([
                n for n in a.analyze_string(observable.value)
                if n and n not in values
            ])

    if len(new) == 0:
        return values, values
    else:
        _, extended = derive(new + list(values))
        return values, extended
Пример #2
0
    def tag(self, new_tags, strict=False):
        new_tags = iterify(new_tags)

        if strict:
            remove = set([t.name for t in self.tags]) - set(new_tags)
            for tag in remove:
                self.modify(pull__tags__name=tag)

        for new_tag in new_tags:
            if new_tag.strip() != '':
                new_tag = Tag(name=new_tag)
                new_tag.clean()

                try:  # check if tag is a replacement
                    tag = Tag.objects.get(replaces=new_tag.name)
                except DoesNotExist:
                    tag = Tag.get_or_create(name=new_tag.name)

                # search for related entities and link them
                for e in Entity.objects(tags__in=[tag.name]):
                    self.link_to(e, 'Tagged', 'tags')

                if not self.modify({"tags__name": tag.name}, set__tags__S__fresh=True, set__tags__S__last_seen=datetime.now()):
                    self.modify(push__tags=ObservableTag(name=tag.name))
                    tag.modify(inc__count=1)

        return self.reload()
Пример #3
0
    def tag(self, new_tags, strict=False, expiration=None):
        """Tags an observable.

        An observable can be tagged to add more information as to what it represents.

        Args:
            new_tags:
                An array of strings to tag the observable with.
            strict:
                Set to ``True`` to replace all existing tags with the ``new_tags``.
            expiration:
                Timedelta field after which the Tag will not be considered fresh anymore.

        Returns:
            A fresh Observable instance as reloaded from the database.

        """

        new_tags = iterify(new_tags)

        if strict:
            remove = set([t.name for t in self.tags]) - set(new_tags)
            for tag in remove:
                self.modify(pull__tags__name=tag)

        tagged = False
        for new_tag in new_tags:
            if new_tag.strip() != '':
                tagged = True

                new_tag = Tag(name=new_tag)
                new_tag.clean()

                try:  # check if tag is a replacement
                    tag = Tag.objects.get(replaces=new_tag.name)
                except DoesNotExist:
                    tag = Tag.get_or_create(name=new_tag.name)

                if not expiration:
                    expiration = tag.default_expiration

                extra_tags = tag.produces + [tag]

                # search for related entities and link them
                for e in Entity.objects(tags__in=[tag.name]):
                    self.active_link_to(e, 'Tagged', 'tags', clean_old=False)

                for tag in extra_tags:
                    if not self.modify(
                        {"tags__name": tag.name}, set__tags__S__fresh=True,
                            set__tags__S__last_seen=datetime.utcnow()):
                        self.modify(
                            push__tags=ObservableTag(
                                name=tag.name, expiration=expiration))
                        tag.modify(inc__count=1)

        if tagged:
            self.update(set__last_tagged=datetime.utcnow())

        return self.reload()
Пример #4
0
 def inner(*args, **kwargs):
     # a user needs at least one of the roles to be granted access
     for r in iterify(roles[0]):
         if current_user.has_role(r):
             return f(*args, **kwargs)
     else:
         abort(401)
Пример #5
0
def derive(observables):
    """Indicate that the module needs a specific attribute to work properly.

    This function is only useful in abstract modules, in order to make sure
    that modules that inherit from this class correctly defines needed class
    attributes.

    Args:
        variables: a string or an array of strings containing the name of
            needed class attributes.

    Raises:
        ModuleInitializationError: One of the needed attributes is not
            correctly defined.
    """

    new = []
    observables = list(iterify(observables))
    for i, observable in enumerate(observables):
        try:
            t = Observable.guess_type(observable)
            temp = t(value=observable)
            temp.clean()
            observable = temp.value
            observables[i] = observable
            for a in analyzers.get(t, []):
                new.extend([n for n in a.analyze_string(observable) if n and n not in observables])
        except ObservableValidationError:
            pass

    if len(new) == 0:
        return observables
    else:
        return observables + derive(new)
Пример #6
0
def derive(observables):
    """Indicate that the module needs a specific attribute to work properly.

    This function is only useful in abstract modules, in order to make sure
    that modules that inherit from this class correctly defines needed class
    attributes.

    Args:
        variables: a string or an array of strings containing the name of
            needed class attributes.

    Raises:
        ModuleInitializationError: One of the needed attributes is not
            correctly defined.
    """

    new = []
    for observable in iterify(observables):
        try:
            t = Observable.guess_type(observable)
            for a in analyzers.get(t, []):
                new.extend([
                    n for n in a.analyze_string(observable)
                    if n and n not in observables
                ])
        except ObservableValidationError:
            pass

    if len(new) == 0:
        return observables
    else:
        return derive(new + observables)
Пример #7
0
 def info(self):
     i = {k: v for k, v in self._data.items() if k in ["name", "description", "last_run", "enabled", "status"]}
     i['frequency'] = str(self.frequency)
     i['expiration'] = str(self.EXPIRATION)
     i['acts_on'] = iterify(self.ACTS_ON)
     i['id'] = str(self.id)
     return i
Пример #8
0
 def multidelete(self):
     data = loads(request.data)
     ids = iterify(data['ids'])
     for i, inv in enumerate(Investigation.objects(links__id__in=ids)):
         inv.modify({"links__id": id}, set__links__S__id="local-{}-{}".format(time.time(), i))
     self.objectmanager.objects(id__in=ids).delete()
     return render({"deleted": ids})
Пример #9
0
 def post_save(cls, sender, document, created):
     if issubclass(sender, Observable):
         if created:
             for analytics in cls.analytics.itervalues():
                 if analytics.enabled and sender.__name__ in iterify(
                         analytics.ACTS_ON):
                     analytics.each(document)
Пример #10
0
    def tag(self, new_tags, strict=False, expiration=None):
        """Tags an observable.

        An observable can be tagged to add more information as to what it represents.

        Args:
            new_tags:
                An array of strings to tag the observable with.
            strict:
                Set to ``True`` to replace all existing tags with the ``new_tags``.
            expiration:
                Timedelta field after which the Tag will not be considered fresh anymore.

        Returns:
            A fresh Observable instance as reloaded from the database.

        """

        new_tags = iterify(new_tags)

        if strict:
            remove = set([t.name for t in self.tags]) - set(new_tags)
            for tag in remove:
                self.modify(pull__tags__name=tag)

        tagged = False
        for new_tag in new_tags:
            if new_tag.strip() != '':
                tagged = True

                new_tag = Tag(name=new_tag)
                new_tag.clean()

                try:  # check if tag is a replacement
                    tag = Tag.objects.get(replaces=new_tag.name)
                except DoesNotExist:
                    tag = Tag.get_or_create(name=new_tag.name)

                if not expiration:
                    expiration = tag.default_expiration

                extra_tags = tag.produces + [tag]

                # search for related entities and link them
                for e in Entity.objects(tags__in=[tag.name]):
                    self.active_link_to(e, 'Tagged', 'tags', clean_old=False)

                for tag in extra_tags:
                    if not self.modify(
                        {"tags__name": tag.name},
                            set__tags__S__fresh=True,
                            set__tags__S__last_seen=datetime.utcnow()):
                        self.modify(push__tags=ObservableTag(
                            name=tag.name, expiration=expiration))
                        tag.modify(inc__count=1)

        if tagged:
            self.update(set__last_tagged=datetime.utcnow())

        return self.reload()
Пример #11
0
def derive(strings):
    values = set()
    observables = set()

    for string in iterify(strings):
        if string:
            try:
                t = Observable.guess_type(string)
                observable = t(value=string)
                observable.normalize()
                observables.add(observable)
                values.add(observable.value)
            except ObservableValidationError:
                values.add(string)

    new = []
    for observable in observables:
        for a in analyzers.get(observable.__class__, []):
            new.extend([
                n for n in a.analyze_string(observable.value)
                if n and n not in values
            ])

    if len(new) == 0:
        return values, values
    else:
        _, extended = derive(new + list(values))
        return values, extended
Пример #12
0
 def info(self):
     i = {k: v for k, v in self._data.items() if k in ["name", "description", "last_run", "enabled", "status"]}
     i['frequency'] = str(self.frequency)
     i['expiration'] = str(self.EXPIRATION)
     i['acts_on'] = iterify(self.ACTS_ON)
     i['id'] = str(self.id)
     return i
Пример #13
0
 def inner(*args, **kwargs):
     # a user needs at least one of the roles to be granted access
     for r in iterify(roles[0]):
         if current_user.has_role(r):
             return f(*args, **kwargs)
     else:
         abort(401)
Пример #14
0
 def post_save(cls, sender, document, created):
     if issubclass(sender, Observable):
         if getattr(document, 'new', False):
             for analytics in cls.analytics.values():
                 if analytics.enabled and sender.__name__ in iterify(
                         analytics.ACTS_ON):
                     document.new = False
                     analytics.each(document)
Пример #15
0
 def info(self):
     i = {
         k: v
         for k, v in self._data.items()
         if k in ["name", "description", "enabled"]
     }
     i['acts_on'] = iterify(self.ACTS_ON)
     i['id'] = str(self.id)
     return i
Пример #16
0
    def multidelete(self):
        """Deletes multiple entries from the database

        :query [ObjectID] ids: Array of Element IDs
        :>json [ObjectID] deleted: Array of Element IDs that were successfully deleted
        """
        data = loads(request.data)
        ids = iterify(data['ids'])
        self.objectmanager.objects(id__in=ids).delete()
        return render({"deleted": ids})
Пример #17
0
    def link_to(self, nodes, description, source, first_seen=None, last_seen=None):
        links = set()
        nodes = iterify(nodes)

        for node in nodes:
            link = Link.connect(self, node)
            link.add_history(source, description, first_seen, last_seen)
            links.add(link)

        return list(links)
Пример #18
0
    def multidelete(self):
        """Deletes multiple entries from the database

        :query [ObjectID] ids: Array of Element IDs
        :>json [ObjectID] deleted: Array of Element IDs that were successfully deleted
        """
        data = loads(request.data)
        ids = iterify(data['ids'])
        self.objectmanager.objects(id__in=ids).delete()
        return render({"deleted": ids})
Пример #19
0
    def link_to(self, nodes, description, source, first_seen=None, last_seen=None):
        links = set()
        nodes = iterify(nodes)

        for node in nodes:
            link = Link.connect(self, node)
            link.add_history(source, description, first_seen, last_seen)
            links.add(link)

        return list(links)
Пример #20
0
 def inner(*args, **kwargs):
     oname = object_name
     if not oname:
         oname = getattr(args[0], 'klass', getattr(args[0], 'objectmanager', args[0].__class__)).__name__.lower()
     # a user must have all permissions in order to be granted access
     for p in iterify(permissions):
         if not current_user.has_permission(oname, p):
             # improve this and make it redirect to login
             abort(401)
     else:
         return f(*args, **kwargs)
Пример #21
0
    def analyze_outdated(self):
        class_filter = Q()
        for acts_on in iterify(self.ACTS_ON):
            class_filter |= Q(_cls__contains=acts_on)

        # do outdated logic
        fltr = Q(**{"last_analyses__{}__exists".format(self.name): False})
        if self.EXPIRATION:
            fltr |= Q(**{"last_analyses__{}__lte".format(self.name): datetime.now() - self.EXPIRATION})
        fltr &= Q(**self.CUSTOM_FILTER) & class_filter
        self.bulk(Observable.objects(fltr))
Пример #22
0
    def analyze_outdated(self):
        class_filter = Q()
        for acts_on in iterify(self.ACTS_ON):
            class_filter |= Q(_cls="Observable.{}".format(acts_on))

        # do outdated logic
        fltr = Q(**{"last_analyses__{}__exists".format(self.name): False})
        if self.EXPIRATION:
            fltr |= Q(**{"last_analyses__{}__lte".format(self.name): datetime.utcnow() - self.EXPIRATION})
        fltr &= self.CUSTOM_FILTER & class_filter
        self.bulk(Observable.objects(fltr).no_cache())
Пример #23
0
    def multiupdate(self):
        """Updates multiple entries from the database

        :query [ObjectID] ids: Array of Element IDs
        :query [Object] new: JSON object representing fields to update
        :>json [ObjectID] updated: Array of Element IDs that were successfully updated
        """
        data = loads(request.data)
        ids = iterify(data['ids'])
        new_data = data['new']
        self.objectmanager.objects(id__in=ids).update(new_data)
        return render({"updated": list(self.objectmanager.objects(ids__in=ids))})
Пример #24
0
    def change_all_tags(old_tags, new_tag):
        """Changes tags on all observables

        Args:
            old_tags: A string or array of strings representing tag names to change
            new_tag: The new tag name by which all ``old_tags`` should be replaced

        """
        old_tags = iterify(old_tags)
        for o in Observable.objects(tags__name__in=old_tags):
            for old_tag in old_tags:
                o.change_tag(old_tag, new_tag)
Пример #25
0
    def multiupdate(self):
        """Updates multiple entries from the database

        :query [ObjectID] ids: Array of Element IDs
        :query [Object] new: JSON object representing fields to update
        :>json [ObjectID] updated: Array of Element IDs that were successfully updated
        """
        data = loads(request.data)
        ids = iterify(data['ids'])
        new_data = data['new']
        self.objectmanager.objects(id__in=ids).update(new_data)
        return render({"updated": list(self.objectmanager.objects(ids__in=ids))})
Пример #26
0
    def change_all_tags(old_tags, new_tag):
        """Changes tags on all observables

        Args:
            old_tags: A string or array of strings representing tag names to change
            new_tag: The new tag name by which all ``old_tags`` should be replaced

        """
        old_tags = iterify(old_tags)
        for o in Observable.objects(tags__name__in=old_tags):
            for old_tag in old_tags:
                o.change_tag(old_tag, new_tag)
Пример #27
0
def derive(observables):
    new = []
    for observable in iterify(observables):
        try:
            t = Observable.guess_type(observable)
            for a in analyzers.get(t, []):
                new.extend([n for n in a.analyze_string(observable) if n and n not in observables])
        except ObservableValidationError:
            pass

    if len(new) == 0:
        return observables
    else:
        return derive(new + observables)
Пример #28
0
def link_from_data(observable, data, path, klass, description):
    data = get_value_at(data, path)

    if data is None:
        return []

    links = set()

    for value in iterify(data):
        try:
            node = klass.get_or_create(value=value)
        except FieldDoesNotExist:
            node = klass.get_or_create(name=value)

        links.update(observable.active_link_to(node, description, 'DomainTools'))

    return list(links)
Пример #29
0
    def active_link_to(self, nodes, description, source, clean_old=True):
        links = set()
        nodes = iterify(nodes)

        for node in nodes:
            link = Link.connect(self, node)
            link.add_history(source, description, active=True)
            links.add(link)

            if clean_old:
                for link, node in self.outgoing():
                    if node not in nodes:
                        active_link = link.get_active(description)
                        if active_link:
                            active_link.active = False
                            link.save(validate=False)

        return list(links)
Пример #30
0
    def active_link_to(self, nodes, description, source, clean_old=True):
        links = set()
        nodes = iterify(nodes)

        for node in nodes:
            link = Link.connect(self, node)
            link.add_history(source, description, active=True)
            links.add(link)

            if clean_old:
                for link, node in self.outgoing():
                    if node not in nodes:
                        active_link = link.get_active(description)
                        if active_link:
                            active_link.active = False
                            link.save(validate=False)

        return list(links)
Пример #31
0
 def post_save(cls, sender, document, created):
     if issubclass(sender, Observable):
         if created:
             for analytics in cls.analytics.itervalues():
                 if analytics.enabled and sender.__name__ in iterify(analytics.ACTS_ON):
                     analytics.each(document)
Пример #32
0
    def add(self, investigation):
        investigation.add(iterify(request.json["links"]), iterify(request.json["nodes"]))

        return render(investigation.info())
Пример #33
0
 def add_replaces(self, tags):
     self.replaces += list(set(iterify(tags) + self.replaces))
     return self.save()
Пример #34
0
    def remove(self, id):
        i = get_object_or_404(self.objectmanager, id=id)
        data = loads(request.data)
        i.remove(iterify(data['links']), iterify(data['nodes']))

        return render(i.info())
Пример #35
0
 def untag(self, tags):
     for tag in iterify(tags):
         self.modify(pull__tags__name=tag)
Пример #36
0
 def add_replaces(self, tags):
     self.replaces += list(set(iterify(tags) + self.replaces))
     return self.save()
Пример #37
0
 def info(self):
     i = {k: v for k, v in self._data.items() if k in ["name", "description", "enabled"]}
     i['acts_on'] = iterify(self.ACTS_ON)
     i['id'] = str(self.id)
     return i
Пример #38
0
    def add_produces(self, tags):
        tags = [Tag.get_or_create(name=t) for t in iterify(tags)]

        self.produces = tags
        return self.save()
Пример #39
0
    def add(self, id):
        i = get_object_or_404(self.objectmanager, id=id)
        data = loads(request.data)
        i.add(iterify(data["links"]), iterify(data["nodes"]))

        return render(i.info())
Пример #40
0
 def change_all_tags(old_tags, new_tag):
     for o in Observable.objects(tags__name__in=iterify(old_tags)):
         for old_tag in old_tags:
             o.change_tag(old_tag, new_tag)
Пример #41
0
 def add_produces(self, tags):
     self.produces = [Tag.get_or_create(name=t) for t in iterify(tags)]
     return self.save()
Пример #42
0
 def untag(self, tags):
     for tag in iterify(tags):
         self.modify(pull__tags__name=tag)
Пример #43
0
    def remove(self, id):
        i = get_object_or_404(self.objectmanager, id=id)
        data = loads(request.data)
        i.remove(iterify(data['links']), iterify(data['nodes']))

        return render(i.info())