Beispiel #1
0
    def match(self):
        """Match observables against Yeti's intelligence repository.

        Takes an array of observables, expands them and tries to match them against specific indicators or known observables.

        To "expand" an observable means to enrich the query. For instance, if the arrays of observables contains the URL ``http://google.com``,
        the "expanded" observable array will also include the hostname ``google.com``.

        :<json [string] observables: An array of observables to be analyzed

        :>json [Entity] entities: Related ``Entity`` objects
        :>json [Observable] known: ``Observable`` objects that are already present in database
        :>json [Indicator] matches: ``Indicators`` that matched observables
        :>json Observable matches[].observable: The ``Observable`` object that matched the ``Indicator``
        :>json string unknown: Array of observable strings that didn't match any ``Indicators`` and are unknown to Yeti
        """

        params = request.json
        observables = params.pop('observables', [])
        fetch_neighbors = params.pop('fetch_neighbors', True)
        add_unknown = bool(params.pop('add_unknown', False))

        if add_unknown and current_user.has_permission('observable', 'write'):
            for o in observables:
                Observable.add_text(o)

        data = match_observables(observables, save_matches=add_unknown and current_user.has_permission('observable', 'write'), fetch_neighbors=fetch_neighbors)

        return render(data)
Beispiel #2
0
    def match(self):
        params = request.json
        observables = params.pop('observables', [])
        add_unknown = bool(params.pop('add_unknown', False))

        if add_unknown:
            for o in observables:
                Observable.add_text(o)

        data = match_observables(observables, save_matches=add_unknown)

        return render(data)
Beispiel #3
0
def derive(observables):
    """Indicate that the module needs a specific attribute to work properly.

    This function is only useful in abstract modules, in order to make sure
    that modules that inherit from this class correctly defines needed class
    attributes.

    Args:
        variables: a string or an array of strings containing the name of
            needed class attributes.

    Raises:
        ModuleInitializationError: One of the needed attributes is not
            correctly defined.
    """

    new = []
    observables = list(iterify(observables))
    for i, observable in enumerate(observables):
        try:
            t = Observable.guess_type(observable)
            temp = t(value=observable)
            temp.clean()
            observable = temp.value
            observables[i] = observable
            for a in analyzers.get(t, []):
                new.extend([n for n in a.analyze_string(observable) if n and n not in observables])
        except ObservableValidationError:
            pass

    if len(new) == 0:
        return observables
    else:
        return observables + derive(new)
Beispiel #4
0
    def analyze(self, dict):
        observable = dict['title']
        description = dict['description'].lower()
        context = {}
        context['description'] = "{} C2 server".format(description)
        context['date_added'] = datetime.strptime(dict['pubDate'], "%d-%m-%Y")
        context['source'] = self.name

        try:
            e = Observable.add_text(observable)
        except ObservableValidationError as e:
            logging.error(e)
            return

        e.add_context(context)
        e.add_source("feed")

        tags = ['malware', 'c2', description, 'crimeware']
        if description == 'pony':
            tags.extend(['stealer', 'dropper'])
        elif description == 'athena':
            tags.extend(['stealer', 'ddos'])
        elif description in ['zeus', 'citadel']:
            tags.extend(['banker'])

        e.tag(tags)
Beispiel #5
0
    def analyze(observable, results):
        links = set()

        params = {'query': observable.value}

        data = PassiveTotalApi.get('/dns/passive', results.settings, params)

        for record in data['results']:
            first_seen = datetime.strptime(
                record['firstSeen'], "%Y-%m-%d %H:%M:%S")
            last_seen = datetime.strptime(
                record['lastSeen'], "%Y-%m-%d %H:%M:%S")

            new = Observable.add_text(record['resolve'])
            if isinstance(observable, Hostname):
                links.update(
                    observable.link_to(
                        new, "{} record".format(record['recordType']),
                        'PassiveTotal', first_seen, last_seen))
            else:
                links.update(
                    new.link_to(
                        observable, "{} record".format(record['recordType']),
                        'PassiveTotal', first_seen, last_seen))

        return list(links)
Beispiel #6
0
def derive(strings):
    values = set()
    observables = set()

    for string in iterify(strings):
        if string:
            try:
                t = Observable.guess_type(string)
                observable = t(value=string)
                observable.normalize()
                observables.add(observable)
                values.add(observable.value)
            except ObservableValidationError:
                values.add(string)

    new = []
    for observable in observables:
        for a in analyzers.get(observable.__class__, []):
            new.extend([
                n for n in a.analyze_string(observable.value)
                if n and n not in values
            ])

    if len(new) == 0:
        return values, values
    else:
        _, extended = derive(new + list(values))
        return values, extended
Beispiel #7
0
    def post(self):
        q = request.get_json(silent=True)
        params = q.pop("params", {})
        observables = []

        for o in q["observables"]:
            try:
                obs = Observable.guess_type(o['value'])(value=o['value'])
                obs.clean()
                observables.append(obs.value)

                # Save observables & eventual tags to database
                if params.get('save_query', False):
                    obs = obs.save()
                    obs.tag(o.get("tags", []))
                    obs.add_source("query")
            except ObservableValidationError:
                continue

        # match observables with known indicators
        data = match_observables([o for o in observables])

        # find related observables (eg. URLs for domain, etc.)
        # related_observables = [obs.get_related() for obs in observables]
        # data = self.match_observables(related_observable)
        #
        # we need to find a way to degrade the "confidence" in
        # hits obtained from related observables

        return render(data, "analysis.html")
Beispiel #8
0
 def analyze_outdated(self):
     # do outdated logic
     fltr = Q(**{"last_analyses__{}__exists".format(self.name): False})
     if self.EXPIRATION:
         fltr |= Q(**{"last_analyses__{}__lte".format(self.name): datetime.now() - self.EXPIRATION})
     fltr &= Q(**self.CUSTOM_FILTER) & Q(_cls__contains=self.ACTS_ON)
     self.bulk(Observable.objects(fltr))
Beispiel #9
0
 def execute(self):
     self.export_file_handle = codecs.open(self.output_file, 'w+', "utf-8")
     q = Q(tags__name__in=[t.name for t in self.include_tags]) & Q(tags__name__nin=[t.name for t in self.exclude_tags])
     q &= Q(_cls__contains=self.acts_on)
     output = self.template.render(Observable.objects(q))
     self.write(output)
     self.export_file_handle.close()
Beispiel #10
0
 def each(url):
     try:
         host = ProcessUrl.analyze_string(url.value)[0]
         h = Observable.guess_type(host).get_or_create(value=host)
         h.add_source("analytics")
         Link.connect(src=url, dst=h)
     except ObservableValidationError:
         logging.error("An error occurred when trying to add {} to the database".format(host))
Beispiel #11
0
 def each(url):
     try:
         host = ProcessUrl.analyze_string(url.value)[0]
         h = Observable.guess_type(host).get_or_create(value=host)
         h.add_source("analytics")
         url.active_link_to(h, "hostname", "ProcessUrl", clean_old=False)
         return h
     except ObservableValidationError:
         logging.error("An error occurred when trying to add {} to the database".format(host))
Beispiel #12
0
 def enrich(self):
     return "ENRICH"
     if request.method == "POST":
         lines = request.form['bulk-text'].split('\n')
         for l in lines:
             obs = refang(l.split(',')[0])
             tags = refang(l.split(',')[1:])
             o = Observable.add_text(obs)
             o.tag(tags)
     return render_template('observable/query.html')
Beispiel #13
0
def match_observables(observables):
    # Remove empty observables
    observables = [observable for observable in observables if observable]
    extended_query = set(observables) | set(derive(observables))
    added_entities = set()

    data = {"matches": [], "unknown": set(observables), "entities": [], "known": [], "neighbors": []}

    for o in Observable.objects(value__in=list(extended_query)):
        data['known'].append(o.info())
        del_from_set(data['unknown'], o.value)

        for link, node in (o.incoming()):
            if isinstance(node, Observable):
                if (link.src.value not in extended_query or link.dst.value not in extended_query) and node.tags:
                    data['neighbors'].append((link.info(), node.info()))

    for o, i in Indicator.search(extended_query):
        o = Observable.add_text(o)
        match = i.info()
        match.update({"observable": o.info(), "related": [], "suggested_tags": set()})

        for nodes in i.neighbors().values():
            for l, node in nodes:
                # add node name and link description to indicator
                node_data = {"entity": node.type, "name": node.name, "link_description": l.description or l.tag}
                match["related"].append(node_data)

                # uniquely add node information to related entitites
                if node.name not in added_entities:
                    nodeinfo = node.info()
                    nodeinfo['type'] = node.type
                    data["entities"].append(nodeinfo)
                    added_entities.add(node.name)

                o_tags = o.get_tags()
                [match["suggested_tags"].add(tag) for tag in node.generate_tags() if tag not in o_tags]

        data["matches"].append(match)
        del_from_set(data["unknown"], o.value)

    return data
Beispiel #14
0
    def analyze_outdated(self):
        class_filter = Q()
        for acts_on in iterify(self.ACTS_ON):
            class_filter |= Q(_cls="Observable.{}".format(acts_on))

        # do outdated logic
        fltr = Q(**{"last_analyses__{}__exists".format(self.name): False})
        if self.EXPIRATION:
            fltr |= Q(**{"last_analyses__{}__lte".format(self.name): datetime.utcnow() - self.EXPIRATION})
        fltr &= self.CUSTOM_FILTER & class_filter
        self.bulk(Observable.objects(fltr).no_cache())
Beispiel #15
0
    def post(self, action=None):
        if action == 'merge':
            tags = request.json['merge']
            merge_into = Tag.objects.get(name=request.json['merge_into'])
            make_dict = request.json['make_dict']

            merged = 0
            for tag in tags:
                Observable.change_all_tags(tags, merge_into.name)
                oldtag = Tag.objects.get(name=tag)
                merge_into.count += oldtag.count
                merge_into.produces += [i for i in oldtag.produces if i not in merge_into.produces and i != merge_into]
                merge_into.save()
                oldtag.delete()
                merged += 1

            if make_dict:
                merge_into.add_replaces(tags)

            return render({"merged": merged, "into": merge_into.name})
Beispiel #16
0
 def post(self, id=None):
     if not id:
         data = request.json
         data['produces'] = [Tag.get_or_create(name=t.strip()) for t in request.json['produces'].split(',') if t.strip()]
         data['replaces'] = request.json['replaces'].split(',')
         return render(Tag(**data).save().info())
     else:
         try:
             data = request.json
             data['produces'] = [Tag.get_or_create(name=t.strip()) for t in request.json['produces'].split(',') if t.strip()]
             data['replaces'] = request.json['replaces'].split(',')
             t = Tag.objects.get(id=id)
             t.update(**data)
             Observable.change_all_tags(t.name, data['name'])
             return render({"status": "ok"})
         except TagValidationError as e:
             restful_abort(400, error=str(e))
         except Exception as e:
             import traceback
             traceback.print_exc()
             restful_abort(400, error='Must specify name and produces parameters')
Beispiel #17
0
    def execute(self):
        q_include = Q()
        for t in self.include_tags:
            q_include |= Q(tags__match={"name": t.name, "fresh": True})
        q_exclude = Q(tags__name__nin=[t.name for t in self.exclude_tags])
        q = (
            Q(tags__not__size=0, tags__match={"fresh": True})
            & q_include
            & q_exclude
            & Q(_cls="Observable.{}".format(self.acts_on))
        )

        return self.template.render(self.filter_ignore_tags(Observable.objects(q).no_cache()), self.output_file)
Beispiel #18
0
    def nodesearch(self, query):
        result = []

        query = re.compile("^{}".format(query), re.IGNORECASE)

        observables = Observable.objects(value=query).limit(5)
        entities = Entity.objects(name=query).limit(5)

        for results in [observables, entities]:
            for node in results:
                result.append(node.to_mongo())

        return render(result)
Beispiel #19
0
def derive(observables):
    new = []
    for observable in iterify(observables):
        try:
            t = Observable.guess_type(observable)
            for a in analyzers.get(t, []):
                new.extend([n for n in a.analyze_string(observable) if n and n not in observables])
        except ObservableValidationError:
            pass

    if len(new) == 0:
        return observables
    else:
        return derive(new + observables)
Beispiel #20
0
    def analyze(self, item, pub_date):  # pylint: disable=arguments-differ
        s_re = '\[([^\]]*)] Type: (\w+) - IP: (\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3})'
        r = re.compile(s_re)
        m = r.match(item['description'])
        malware_family = ''
        c2_IP = ''
        if m:
            malware_family = m.group(2)
            c2_IP = m.group(3)

        observable = item['title']
        description = item['description'].lower()

        context = {}
        context['description'] = "{} C2 server".format(c2_IP)
        context['date_added'] = pub_date
        context['source'] = self.name

        c2 = None
        e = None

        try:
            e = Observable.add_text(observable)
            if c2_IP:
                c2 = Ip.get_or_create(value=c2_IP)
                e.active_link_to(c2, "IP", self.name, clean_old=False)

        except ObservableValidationError as e:
            logging.error(e)
            logging.error(description)
            return

        tags = ['malware', 'c2', malware_family.lower(), 'crimeware']

        if malware_family == 'pony':
            tags.extend(['stealer', 'dropper'])
        elif malware_family == 'athena':
            tags.extend(['stealer', 'ddos'])
        elif malware_family in ['zeus', 'citadel', 'lokibot']:
            tags.extend(['banker'])

        if e:
            e.add_context(context)
            e.add_source(self.name)
            e.tag(tags)

        if c2:
            c2.add_context(context)
            c2.add_source(self.name)
            c2.tag(tags)
Beispiel #21
0
def derive(observables):
    if isinstance(observables, (str, unicode)):
        observables = [observables]

    new = []
    for observable in observables:
        t = Observable.guess_type(observable)
        for a in analyzers.get(t, []):
            new.extend([n for n in a.analyze_string(observable) if n and n not in observables])

    if len(new) == 0:
        return observables
    else:
        return derive(new + observables)
    def analyze(self, line):
        if not line or line[0].startswith("#"):
            return

        date, _type, family, hostname, url, status, registrar, ips, asns, countries = tuple(
            line)

        item_date = dateutil.parser.parse(date)
        max_age = yeti_config.get('limits', 'max_age')
        limit_date = datetime.now() - timedelta(days=max_age)

        if item_date < limit_date:
            return

        tags = []
        tags += TYPE_DICT[_type]
        tags.append(family.lower())

        context = {
            "first_seen": date,
            "status": status,
            "registrar": registrar,
            "countries": countries.split("|"),
            "asns": asns.split("|"),
            "source": self.name
        }

        try:
            url = Url.get_or_create(value=url.rstrip())
            url.add_context(context)
            url.tag(tags)

            hostname = Observable.add_text(hostname)
            hostname.tag(tags + ['blocklist'])

            for ip in ips.split("|"):
                if ip != hostname and ip is not None and ip != '':
                    try:
                        i = Ip.get_or_create(value=ip)
                        i.active_link_to(hostname,
                                         "First seen IP",
                                         self.name,
                                         clean_old=False)

                    except ObservableValidationError as e:
                        logging.error("Invalid Observable: {}".format(e))

        except ObservableValidationError as e:
            logging.error(e)
Beispiel #23
0
    def analyze_outdated(self):
        class_filter = Q()
        for acts_on in iterify(self.ACTS_ON):
            class_filter |= Q(_cls="Observable.{}".format(acts_on))

        # do outdated logic
        fltr = Q(**{"last_analyses__{}__exists".format(self.name): False})
        if self.EXPIRATION:
            fltr |= Q(
                **{
                    "last_analyses__{}__lte".format(self.name):
                    datetime.utcnow() - self.EXPIRATION
                })
        fltr &= self.CUSTOM_FILTER & class_filter
        self.bulk(Observable.objects(fltr).no_cache())
Beispiel #24
0
    def rdata_lookup(observable, api_key):
        links = set()

        for record in DNSDBApi.lookup('rdata', observable, api_key):
            new = Observable.add_text(record['rrname'])
            new.add_source('analytics')

            links.update(
                new.link_to(observable,
                            source='DNSDB Passive DNS',
                            description='{} record'.format(record['rrtype']),
                            first_seen=record['first_seen'],
                            last_seen=record['last_seen']))

        return list(links)
Beispiel #25
0
    def rdata_lookup(observable, api_key):
        links = set()

        for record in DNSDBApi.lookup('rdata', observable, api_key):
            new = Observable.add_text(record['rrname'])
            new.add_source('analytics')

            links.update(
                new.link_to(
                    observable,
                    source='DNSDB Passive DNS',
                    description='{} record'.format(record['rrtype']),
                    first_seen=record['first_seen'],
                    last_seen=record['last_seen']))

        return list(links)
Beispiel #26
0
    def each(cls, hostname, rtype=None, results=[]):
        generated = []
        h = Hostname.get_or_create(value=hostname.value)

        for rdata in results:
            logging.info("{} resolved to {} ({} record)".format(h.value, rdata, rtype))
            try:
                e = Observable.add_text(rdata)
                e.add_source("analytics")
                generated.append(e)
                l = Link.connect(h, e)
                l.add_history(tag=rtype, description='{} record'.format(rtype))
            except ObservableValidationError as e:
                logging.error("{} is not a valid datatype".format(rdata))

        h.analysis_done(cls.__name__)
        return generated
Beispiel #27
0
    def rrset_lookup(hostname, api_key):
        links = set()

        for record in DNSDBApi.lookup('rrset', hostname, api_key):
            for observable in record['rdata']:
                observable = Observable.add_text(observable)
                observable.add_source('analytics')

                links.update(
                    hostname.link_to(observable,
                                     source='DNSDB Passive DNS',
                                     description='{} record'.format(
                                         record['rrtype']),
                                     first_seen=record['first_seen'],
                                     last_seen=record['last_seen']))

        return list(links)
Beispiel #28
0
    def each(cls, hostname, rtype=None, results=[]):
        generated = []
        h = Hostname.get_or_create(value=hostname.value)

        for rdata in results:
            logging.debug("{} resolved to {} ({} record)".format(h.value, rdata, rtype))
            try:
                e = Observable.add_text(rdata)
                e.add_source("analytics")
                generated.append(e)
            except ObservableValidationError as e:
                logging.error("{} is not a valid datatype".format(rdata))

        h.active_link_to(generated, "{} record".format(rtype), "ResolveHostnames")

        h.analysis_done(cls.__name__)
        return generated
Beispiel #29
0
    def rrset_lookup(hostname, api_key):
        links = set()

        for record in DNSDBApi.lookup('rrset', hostname, api_key):
            for observable in record['rdata']:
                observable = Observable.add_text(observable)
                observable.add_source('analytics')

                links.update(
                    hostname.link_to(
                        observable,
                        source='DNSDB Passive DNS',
                        description='{} record'.format(record['rrtype']),
                        first_seen=record['first_seen'],
                        last_seen=record['last_seen']))

        return list(links)
Beispiel #30
0
    def rdata_lookup(observable, api_key):
        links = set()

        for record in DNSDBApi.lookup("rdata", observable, api_key):
            new = Observable.add_text(record["rrname"])
            new.add_source("analytics")

            links.update(
                new.link_to(
                    observable,
                    source="DNSDB Passive DNS",
                    description="{} record".format(record["rrtype"]),
                    first_seen=record["first_seen"],
                    last_seen=record["last_seen"],
                )
            )

        return list(links)
    def analyze(self, line):

        if not line or line[0].startswith("#"):
            return

        date, _type, family, hostname, url, status, registrar, ips, asns, countries = tuple(
            line)

        tags = []
        tags += TYPE_DICT[_type]
        tags.append(family.lower())

        context = {
            "first_seen": date,
            "status": status,
            "registrar": registrar,
            "countries": countries.split("|"),
            "asns": asns.split("|"),
            "source": self.name
        }

        try:
            url = Url.get_or_create(value=url.rstrip())
            url.add_context(context)
            url.tag(tags)

            hostname = Observable.add_text(hostname)
            hostname.tag(tags + ['blocklist'])

            for ip in ips.split("|"):
                if ip != hostname and ip is not None and ip != '':
                    try:
                        i = Ip.get_or_create(value=ip)
                        i.active_link_to(
                            hostname,
                            "First seen IP",
                            self.name,
                            clean_old=False)
                    except ObservableValidationError as e:
                        logging.error("Invalid Observable: {}".format(e))

        except ObservableValidationError as e:
            logging.error("Invalid line: {}\nLine: {}".format(e, line))
    def each(cls, hostname, rtype=None, results=[]):
        generated = []
        h = Hostname.get_or_create(value=hostname.value)

        for rdata in results:
            logging.debug("{} resolved to {} ({} record)".format(
                h.value, rdata, rtype))
            try:
                e = Observable.add_text(rdata)
                e.add_source("analytics")
                generated.append(e)
            except ObservableValidationError as e:
                logging.error("{} is not a valid datatype".format(rdata))

        h.active_link_to(generated, "{} record".format(rtype),
                         "ResolveHostnames")

        h.analysis_done(cls.__name__)
        return generated
Beispiel #33
0
    def rrset_lookup(hostname, api_key):
        links = set()

        for record in DNSDBApi.lookup("rrset", hostname, api_key):
            for observable in record["rdata"]:
                observable = Observable.add_text(observable)
                observable.add_source("analytics")

                links.update(
                    hostname.link_to(
                        observable,
                        source="DNSDB Passive DNS",
                        description="{} record".format(record["rrtype"]),
                        first_seen=record["first_seen"],
                        last_seen=record["last_seen"],
                    )
                )

        return list(links)
    def analyze(self, line):

        if not line or line[0].startswith("#"):
            return

        date, _type, family, hostname, url, status, registrar, ips, asns, countries = tuple(
            line)

        tags = []
        tags += TYPE_DICT[_type]
        tags.append(family.lower())

        context = {
            "first_seen": date,
            "status": status,
            "registrar": registrar,
            "countries": countries.split("|"),
            "asns": asns.split("|"),
            "source": self.name
        }

        try:
            url = Url.get_or_create(value=url)
            url.add_context(context)
            url.tag(tags)

            hostname = Observable.add_text(hostname)
            hostname.tag(tags + ['blocklist'])

            for ip in ips.split("|"):
                if ip != hostname:
                    try:
                        i = Ip.get_or_create(value=ip)
                        i.active_link_to(hostname,
                                         "First seen IP",
                                         self.name,
                                         clean_old=False)
                    except ObservableValidationError as e:
                        logging.error("Invalid Observable: {}".format(e))

        except ObservableValidationError as e:
            logging.error("Invalid line: {}\nLine: {}".format(e, line))
Beispiel #35
0
    def _get_selected_observables(self, data):
        if isinstance(data, MultiDict):
            ids = data.getlist('ids')
            query = data.get('query')
        else:
            ids = data.get('ids', None)
            query = data.get('query', None)

        if ids:
            return Observable.objects(id__in=ids)
        elif query:
            query = json.loads(query)
            fltr = query.get('filter', {})
            params = query.get('params', {})
            regex = params.pop('regex', False)
            ignorecase = params.pop('ignorecase', False)

            return get_queryset(Observable, fltr, regex, ignorecase)
        else:
            return []
Beispiel #36
0
    def analyze(observable, results):
        links = set()

        params = {
            'query': observable.value
        }

        data = PassiveTotalApi.get('/dns/passive', results.settings, params)

        for record in data['results']:
            first_seen = datetime.strptime(record['firstSeen'], "%Y-%m-%d %H:%M:%S")
            last_seen = datetime.strptime(record['lastSeen'], "%Y-%m-%d %H:%M:%S")

            new = Observable.add_text(record['resolve'])
            if isinstance(observable, Hostname):
                links.update(observable.link_to(new, "{} record".format(record['recordType']), 'PassiveTotal', first_seen, last_seen))
            else:
                links.update(new.link_to(observable, "{} record".format(record['recordType']), 'PassiveTotal', first_seen, last_seen))

        return list(links)
Beispiel #37
0
    def _get_selected_observables(self, data):
        if isinstance(data, MultiDict):
            ids = data.getlist("ids")
            query = data.get("query")
        else:
            ids = data.get("ids", None)
            query = data.get("query", None)

        if ids:
            return Observable.objects(id__in=ids)
        elif query:
            query = json.loads(query)
            fltr = query.get("filter", {})
            params = query.get("params", {})
            regex = params.pop("regex", False)
            ignorecase = params.pop("ignorecase", False)

            return get_queryset(Observable, fltr, regex, ignorecase)
        else:
            return []
Beispiel #38
0
    def _get_selected_observables(self, data):
        if isinstance(data, MultiDict):
            ids = data.getlist('ids')
            query = data.get('query')
        else:
            ids = data.get('ids', None)
            query = data.get('query', None)

        if ids:
            return Observable.objects(id__in=ids)
        elif query:
            query = json.loads(query)
            fltr = query.get('filter', {})
            params = query.get('params', {})
            regex = params.pop('regex', False)
            ignorecase = params.pop('ignorecase', False)

            return get_queryset(Observable, fltr, regex, ignorecase)
        else:
            return []
Beispiel #39
0
def derive(observables):
    """Indicate that the module needs a specific attribute to work properly.

    This function is only useful in abstract modules, in order to make sure
    that modules that inherit from this class correctly defines needed class
    attributes.

    Args:
        variables: a string or an array of strings containing the name of
            needed class attributes.

    Raises:
        ModuleInitializationError: One of the needed attributes is not
            correctly defined.
    """

    new = []
    observables = list(iterify(observables))
    for i, observable in enumerate(observables):
        try:
            t = Observable.guess_type(observable)
            temp = t(value=observable)
            temp.clean()
            observable = temp.value
            observables[i] = observable
            for a in analyzers.get(t, []):
                new.extend([
                    n for n in a.analyze_string(observable)
                    if n and n not in observables
                ])
        except ObservableValidationError:
            pass

    if len(new) == 0:
        return observables
    else:
        return observables + derive(new)
Beispiel #40
0
    def __add_attribute(self, instance, attribute, context, tags):

        if attribute["category"] == "External analysis":
            return

        if attribute.get("type") in self.TYPES_TO_IMPORT:

            context["id"] = attribute["event_id"]
            context["link"] = urljoin(
                self.instances[instance]["url"],
                "/events/{}".format(attribute["event_id"]),
            )

            context["comment"] = attribute["comment"]

            obs = Observable.add_text(attribute["value"])

            if attribute["category"]:
                obs.tag(attribute["category"].replace(" ", "_"))

            if tags:
                obs.tag(tags)

            obs.add_context(context)
Beispiel #41
0
 def info(self):
     i = Observable.info(self)
     i['mime_type'] = self.mime_type
     i['hashes'] = self.hashes
     i['filenames'] = self.filenames
     return i
Beispiel #42
0
 def info(self):
     i = Observable.info(self)
     i['mime_type'] = self.mime_type
     i['hashes'] = self.hashes
     return i
Beispiel #43
0
def match_observables(observables, save_matches=False, fetch_neighbors=True):
    # Remove empty observables
    observables = [refang(observable) for observable in observables if observable]
    extended_query = set(observables) | set(derive(observables))

    data = {
        "matches": [],
        "unknown": set(observables),
        "entities": {},
        "known": [],
        "neighbors": [],
    }

    # add to "known"
    for o in Observable.objects(value__in=list(extended_query)):
        data['known'].append(o.info())
        del_from_set(data['unknown'], o.value)

        if fetch_neighbors:
            for link, node in (o.incoming()):
                if isinstance(node, Observable):
                    if (link.src.value not in extended_query or link.dst.value not in extended_query) and node.tags:
                        data['neighbors'].append((link.info(), node.info()))

        for nodes in o.neighbors("Entity").values():
            for l, node in nodes:
                # add node name and link description to indicator
                node_data = {"entity": node.type, "name": node.name, "link_description": l.description}

                # uniquely add node information to related entitites
                ent = data['entities'].get(node.name, node.info())
                if 'matches' not in ent:
                    ent['matches'] = {"observables": []}
                if 'observables' not in ent['matches']:
                    ent['matches']['observables'] = []

                info = node.info()
                o_info = o.info()
                info['matched_observable'] = {
                    "value": o_info['value'],
                    "tags": [t['name'] for t in o_info['tags']],
                    "human_url": o_info['human_url'],
                    "url": o_info['url']
                }
                if info not in ent['matches']['observables']:
                    ent['matches']['observables'].append(info)
                data['entities'][node.name] = ent

    # add to "matches"
    for o, i in Indicator.search(extended_query):
        if save_matches:
            o = Observable.add_text(o)
        else:
            o = Observable.guess_type(o)(value=o)
            try:
                o.validate()
            except ObservableValidationError:
                pass
            try:
                o = Observable.objects.get(value=o.value)
            except Exception:
                pass

        match = i.info()
        match.update({"observable": o.info(), "related": [], "suggested_tags": set()})

        for nodes in i.neighbors("Entity").values():
            for l, node in nodes:
                # add node name and link description to indicator
                node_data = {"entity": node.type, "name": node.name, "link_description": l.description}
                match["related"].append(node_data)

                # uniquely add node information to related entitites
                ent = data['entities'].get(node.name, node.info())
                if 'matches' not in ent:
                    ent['matches'] = {"indicators": []}
                if 'indicators' not in ent['matches']:
                    ent['matches']['indicators'] = []

                info = i.info()
                info['matched_observable'] = o.value
                if info not in ent['matches']['indicators']:
                    ent['matches']['indicators'].append(info)
                data['entities'][node.name] = ent

                o_tags = o.get_tags()
                [match["suggested_tags"].add(tag) for tag in node.generate_tags() if tag not in o_tags]

        data["matches"].append(match)

    data['entities'] = data['entities'].values()
    return data
Beispiel #44
0
et.save()

et = ExportTemplate(name="Bluecoat")
et.template = """define category cert_blocklist
{% for obs in elements %}{{ obs.value }}
{% endfor %}end
"""
et.save()
Export(name="TestExport",
       acts_on="Url",
       description="Test description",
       frequency=timedelta(hours=1),
       include_tags=[t1, t2],
       template=et).save()

url = Observable.add_text("hxxp://zeuscpanel.com/gate.php")
url.tag(["zeus", "banker", "cc", "c2"])
print url.tags

# print url.find_tags()

# import pdb; pdb.set_trace()

## Create some instances of malware & co
bartalex = Malware.get_or_create(name="Bartalex")
bartalex.family = MalwareFamily.objects.get(name="dropper")
bartalex.killchain = "3"
bartalex.tags = ["bartalex"]
bartalex.save()

dridex = Malware.get_or_create(name="Dridex")
Beispiel #45
0
def each(module_name, observable_json):
    o = Observable.from_json(observable_json)
    mod = loaded_modules[module_name]
    logging.debug("Launching {} on {}".format(mod.name, o))
    mod.each(o)
    o.analysis_done(mod.name)
Beispiel #46
0
MalwareFamily("rootkit").save()
MalwareFamily("trojan").save()
MalwareFamily("dropper").save()


t1 = Tag.get_or_create(name="zeus").add_produces(["crimeware", "banker", "malware"])
t2 = Tag.get_or_create(name="banker").add_produces(["crimeware", "malware"])
t3 = Tag.get_or_create(name="c2")
t3.add_replaces(["c&c", "cc"])

Tag.get_or_create(name="crimeware").add_produces("malware")

Export(name="TestExport", description="Test description", frequency=timedelta(hours=1), include_tags=[t1, t2]).save()


url = Observable.add_text("hxxp://zeuscpanel.com/gate.php")
url.tag(["zeus", "banker", "cc", "c2"])
print url.tags

# print url.find_tags()

# import pdb; pdb.set_trace()



## Create some instances of malware & co
bartalex = Malware.get_or_create(name="Bartalex")
bartalex.family = MalwareFamily.objects.get(name="dropper")
bartalex.killchain = "delivery"
bartalex.tags = ["bartalex"]
bartalex.save()
Beispiel #47
0
def match_observables(observables, save_matches=False, fetch_neighbors=True):
    # Remove empty observables
    observables, extended_query = derive(observables)
    observables = list(observables)

    data = {
        "matches": [],
        "unknown": set(observables),
        "entities": {},
        "known": [],
        "neighbors": [],
    }

    # add to "known"
    for o in Observable.objects(value__in=list(extended_query)):
        data['known'].append(o.info())
        del_from_set(data['unknown'], o.value)

        if fetch_neighbors:
            for link, node in (o.incoming()):
                if isinstance(node, Observable):
                    if (link.src.value not in extended_query or link.dst.value
                            not in extended_query) and node.tags:
                        data['neighbors'].append((link.info(), node.info()))

        for nodes in o.neighbors("Entity").values():
            for l, node in nodes:
                # add node name and link description to indicator
                node_data = {
                    "entity": node.type,
                    "name": node.name,
                    "link_description": l.description
                }

                # uniquely add node information to related entitites
                ent = data['entities'].get(node.name, node.info())
                if 'matches' not in ent:
                    ent['matches'] = {"observables": []}
                if 'observables' not in ent['matches']:
                    ent['matches']['observables'] = []

                info = node.info()
                o_info = o.info()
                info['matched_observable'] = {
                    "value": o_info['value'],
                    "tags": [t['name'] for t in o_info['tags']],
                    "human_url": o_info['human_url'],
                    "url": o_info['url'],
                    "context": o_info['context']
                }
                if info not in ent['matches']['observables']:
                    ent['matches']['observables'].append(info)
                data['entities'][node.name] = ent

    # add to "matches"
    for o, i in Indicator.search(extended_query):
        if save_matches:
            o = Observable.add_text(o)
        else:
            o = Observable.guess_type(o)(value=o)
            try:
                o.validate()
            except ObservableValidationError:
                pass
            try:
                o = Observable.objects.get(value=o.value)
            except Exception:
                pass

        match = i.info()
        match.update({
            "observable": o.info() if o.id else o.value,
            "related": [],
            "suggested_tags": set()
        })

        for nodes in i.neighbors("Entity").values():
            for l, node in nodes:
                # add node name and link description to indicator
                node_data = {
                    "entity": node.type,
                    "name": node.name,
                    "link_description": l.description
                }
                match["related"].append(node_data)

                # uniquely add node information to related entitites
                ent = data['entities'].get(node.name, node.info())
                if 'matches' not in ent:
                    ent['matches'] = {"indicators": []}
                if 'indicators' not in ent['matches']:
                    ent['matches']['indicators'] = []

                info = i.info()
                info['matched_observable'] = o.value
                if info not in ent['matches']['indicators']:
                    ent['matches']['indicators'].append(info)
                data['entities'][node.name] = ent

                o_tags = o.get_tags()
                for tag in node.generate_tags():
                    if tag not in o_tags:
                        match["suggested_tags"].add(tag)

        data["matches"].append(match)

    data['entities'] = data['entities'].values()
    return data
Beispiel #48
0
 def delete(self, id):
     tag = self.objectmanager.objects.get(id=id)
     tag.delete()
     Observable.objects(tags__name=tag.name).update(pull__tags__name=tag.name)
     return render({"status": "ok"})
Beispiel #49
0
 def info(self):
     i = Observable.info(self)
     i["mime_type"] = self.mime_type
     i["hashes"] = self.hashes
     i["filenames"] = self.filenames
     return i
Beispiel #50
0
    def analyze(self, item):
        first_seen = item["first_seen_utc"]
        ioc_value = item["ioc_value"]
        ioc_type = item["ioc_type"]
        threat_type = item["threat_type"]
        malware_alias = item["malware_alias"]
        malware_printable = item["malware_printable"]
        last_seen_utc = item["last_seen_utc"]
        confidence_level = item["confidence_level"]
        reference = item["reference"]
        reporter = item["reporter"]
        tags = []

        context = {"source": self.name}
        context["first_seen"] = first_seen

        if reference:
            context["reference"] = reference
        else:
            context["reference"] = "Unknown"

        if reporter:
            context["reporter"] = reporter
        else:
            context["reporter"] = "Unknown"

        if threat_type:
            context["threat_type"] = threat_type

        if item["tags"]:
            tags.extend(item["tags"].split(","))

        if malware_printable:
            tags.append(malware_printable)

        if malware_alias:
            context["malware_alias"] = malware_alias

        if last_seen_utc:
            context["last_seen_utc"] = last_seen_utc

        if confidence_level:
            context["confidence_level"] = confidence_level

        value = None
        obs = None
        try:
            if "ip" in ioc_type:
                value, port = ioc_value.split(":")
                context["port"] = port
                obs = Ip.get_or_create(value=value)
            else:
                obs = Observable.add_text(ioc_value)

        except ObservableValidationError as e:
            logging.error(e)
            return

        if obs:
            obs.add_context(context)
            obs.add_source(self.name)
            if tags:
                obs.tag(tags)
            if malware_printable:
                obs.tags
def each(module_name, observable_json):
    o = Observable.from_json(observable_json)
    mod = ScheduledAnalytics.objects.get(name=module_name)
    logging.debug("Launching {} on {}".format(mod.name, o))
    mod.each(o)
    o.analysis_done(mod.name)
Beispiel #52
0
import sys
import logging
from os import path

YETI_ROOT = path.normpath(path.dirname(path.dirname(path.abspath(__file__))))
sys.path.append(YETI_ROOT)

from core.analytics import OneShotAnalytics
from core.scheduling import Scheduler
from core.observables import Observable

if __name__ == '__main__':
    logging.basicConfig(level=logging.DEBUG)

    Scheduler()

    if len(sys.argv) == 1:
        print("Re-run using a analytic name as argument")
        for f in OneShotAnalytics.objects():
            print("  {}".format(f.name))

    if len(sys.argv) > 1:
        name = sys.argv[1]
        f = OneShotAnalytics.objects.get(name=name)
        print("Running {}...".format(f.name))
        observable = Observable.guess_type(
            sys.argv[2]).get_or_create(value=sys.argv[2])
        f.analyze(observable, {})
Beispiel #53
0
    def _make_threat_nodes(threat, context, tags):
        # extract Url and Hash info
        threats = dict()
        if threat["threatStatus"] != "active":
            # FIXME, clear out false positive ?
            log.warning(
                "threatStatus %s for threat %s",
                threat["threatStatus"],
                threat["threatID"],
            )
            log.debug(pprint.pformat(threat))
            return None
        log.debug("_make_threat_nodes for threat %s", threat["threatID"])
        # threattype, classification
        # url, phish: url leads to phishing page (threat is url)
        # url, malware: url leads to malware download (threat is url, threatid is maybe sha256)
        # attachment, malware: attachement is malware (threat is sha256)
        # spam, url
        if threat["threatType"] == "url":
            if threat["classification"] == "phish":
                pass  # just keep the url
            elif threat["classification"] == "malware":
                # get url and hash
                threats["attachment"] = threat
            elif threat["classification"] == "spam":
                log.info("URL threat - ignore classification %s",
                         threat["classification"])
            else:
                log.error("Type: url, Unsupported classification %s",
                          threat["classification"])
                log.debug(pprint.pformat(threat))
                return None
            threats["url"] = threat
        elif threat["threatType"] == "attachment":
            if threat["classification"] == "malware":
                threats["attachment"] = threat
            else:
                log.error(
                    "Type: attachment, Unsupported classification %s",
                    threat["classification"],
                )
                log.debug(pprint.pformat(threat))
                return None
        else:
            log.error(
                "Unsupported threatType %s classification %s",
                threat["threatType"],
                threat["classification"],
            )
            log.debug(pprint.pformat(threat))
            return None
        # FIXME check if they exist already.
        # if they do, do not parse the threat a second time ?
        threat_nodes = []
        if "url" in threats:
            # Proofpoint sometimes supplies a hostname marked as a Url.
            # this relies on Yeti to determine the type/class and add act appropriately
            threat_nodes.append(
                Observable.guess_type(threats["url"]["threat"]).get_or_create(
                    value=threats["url"]["threat"], context=[context]))

        if "attachment" in threats:
            threat_nodes.append(
                Hash.get_or_create(value=threats["attachment"]["threatID"],
                                   context=[context]))
        for o in threat_nodes:
            o.tag([t["name"] for t in tags])
        return threat_nodes