Пример #1
0
    def match(self):
        """Match observables against Yeti's intelligence repository.

        Takes an array of observables, expands them and tries to match them against specific indicators or known observables.

        To "expand" an observable means to enrich the query. For instance, if the arrays of observables contains the URL ``http://google.com``,
        the "expanded" observable array will also include the hostname ``google.com``.

        :<json [string] observables: An array of observables to be analyzed

        :>json [Entity] entities: Related ``Entity`` objects
        :>json [Observable] known: ``Observable`` objects that are already present in database
        :>json [Indicator] matches: ``Indicators`` that matched observables
        :>json Observable matches[].observable: The ``Observable`` object that matched the ``Indicator``
        :>json string unknown: Array of observable strings that didn't match any ``Indicators`` and are unknown to Yeti
        """

        params = request.json
        observables = params.pop('observables', [])
        fetch_neighbors = params.pop('fetch_neighbors', True)
        add_unknown = bool(params.pop('add_unknown', False))

        if add_unknown and current_user.has_permission('observable', 'write'):
            for o in observables:
                Observable.add_text(o)

        data = match_observables(observables, save_matches=add_unknown and current_user.has_permission('observable', 'write'), fetch_neighbors=fetch_neighbors)

        return render(data)
Пример #2
0
    def match(self):
        """Match observables against Yeti's intelligence repository.

        Takes an array of observables, expands them and tries to match them against specific indicators or known observables.

        To "expand" an observable means to enrich the query. For instance, if the arrays of observables contains the URL ``http://google.com``,
        the "expanded" observable array will also include the hostname ``google.com``.

        :<json [string] observables: An array of observables to be analyzed

        :>json [Entity] entities: Related ``Entity`` objects
        :>json [Observable] known: ``Observable`` objects that are already present in database
        :>json [Indicator] matches: ``Indicators`` that matched observables
        :>json Observable matches[].observable: The ``Observable`` object that matched the ``Indicator``
        :>json string unknown: Array of observable strings that didn't match any ``Indicators`` and are unknown to Yeti
        """

        params = request.json
        observables = params.pop('observables', [])
        add_unknown = bool(params.pop('add_unknown', False))

        if add_unknown:
            for o in observables:
                Observable.add_text(o)

        data = match_observables(observables, save_matches=add_unknown)

        return render(data)
Пример #3
0
    def match(self):
        params = request.json
        observables = params.pop('observables', [])
        add_unknown = bool(params.pop('add_unknown', False))

        if add_unknown:
            for o in observables:
                Observable.add_text(o)

        data = match_observables(observables, save_matches=add_unknown)

        return render(data)
Пример #4
0
    def analyze(self, item):
        observable = item['title']
        description = item['description'].lower()
        context = {}
        context['description'] = "{} C2 server".format(description)
        context['date_added'] = parser.parse(item['pubDate'])
        context['source'] = self.name

        try:
            e = Observable.add_text(observable)
        except ObservableValidationError as e:
            logging.error(e)
            return

        e.add_context(context)
        e.add_source("feed")

        tags = ['malware', 'c2', description, 'crimeware']
        if description == 'pony':
            tags.extend(['stealer', 'dropper'])
        elif description == 'athena':
            tags.extend(['stealer', 'ddos'])
        elif description in ['zeus', 'citadel']:
            tags.extend(['banker'])

        e.tag(tags)
Пример #5
0
    def analyze(self, dict):
        observable = dict['title']
        description = dict['description'].lower()
        context = {}
        context['description'] = "{} C2 server".format(description)
        context['date_added'] = datetime.strptime(dict['pubDate'], "%d-%m-%Y")
        context['source'] = self.name

        try:
            e = Observable.add_text(observable)
        except ObservableValidationError as e:
            logging.error(e)
            return

        e.add_context(context)
        e.add_source("feed")

        tags = ['malware', 'c2', description, 'crimeware']
        if description == 'pony':
            tags.extend(['stealer', 'dropper'])
        elif description == 'athena':
            tags.extend(['stealer', 'ddos'])
        elif description in ['zeus', 'citadel']:
            tags.extend(['banker'])

        e.tag(tags)
Пример #6
0
    def analyze(observable, results):
        links = set()

        params = {'query': observable.value}

        data = PassiveTotalApi.get('/dns/passive', results.settings, params)

        for record in data['results']:
            first_seen = datetime.strptime(
                record['firstSeen'], "%Y-%m-%d %H:%M:%S")
            last_seen = datetime.strptime(
                record['lastSeen'], "%Y-%m-%d %H:%M:%S")

            new = Observable.add_text(record['resolve'])
            if isinstance(observable, Hostname):
                links.update(
                    observable.link_to(
                        new, "{} record".format(record['recordType']),
                        'PassiveTotal', first_seen, last_seen))
            else:
                links.update(
                    new.link_to(
                        observable, "{} record".format(record['recordType']),
                        'PassiveTotal', first_seen, last_seen))

        return list(links)
Пример #7
0
    def analyze(observable, results):
        links = set()

        params = {"query": observable.value}

        data = PassiveTotalApi.get("/dns/passive", results.settings, params)

        for record in data["results"]:
            first_seen = datetime.strptime(record["firstSeen"],
                                           "%Y-%m-%d %H:%M:%S")
            last_seen = datetime.strptime(record["lastSeen"],
                                          "%Y-%m-%d %H:%M:%S")

            new = Observable.add_text(record["resolve"])
            if isinstance(observable, Hostname):
                links.update(
                    observable.link_to(
                        new,
                        "{} record".format(record["recordType"]),
                        "PassiveTotal",
                        first_seen,
                        last_seen,
                    ))
            else:
                links.update(
                    new.link_to(
                        observable,
                        "{} record".format(record["recordType"]),
                        "PassiveTotal",
                        first_seen,
                        last_seen,
                    ))

        return list(links)
Пример #8
0
    def analyze(observable, results):
        links = set()

        params = {'query': observable.value}

        data = PassiveTotalApi.get('/dns/passive', results.settings, params)

        for record in data['results']:
            first_seen = datetime.strptime(record['firstSeen'],
                                           "%Y-%m-%d %H:%M:%S")
            last_seen = datetime.strptime(record['lastSeen'],
                                          "%Y-%m-%d %H:%M:%S")

            new = Observable.add_text(record['resolve'])
            if isinstance(observable, Hostname):
                links.update(
                    observable.link_to(
                        new, "{} record".format(record['recordType']),
                        'PassiveTotal', first_seen, last_seen))
            else:
                links.update(
                    new.link_to(observable,
                                "{} record".format(record['recordType']),
                                'PassiveTotal', first_seen, last_seen))

        return list(links)
Пример #9
0
 def enrich(self):
     return "ENRICH"
     if request.method == "POST":
         lines = request.form['bulk-text'].split('\n')
         for l in lines:
             obs = refang(l.split(',')[0])
             tags = refang(l.split(',')[1:])
             o = Observable.add_text(obs)
             o.tag(tags)
     return render_template('observable/query.html')
Пример #10
0
    def analyze(self, item):
        s_re = '\[([^\]]*)] Type: (\w+) - IP: (\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3})'
        r = re.compile(s_re)
        m = r.match(item['description'])
        malware_family = ''
        c2_IP = ''
        if m:
            malware_family = m.group(2)
            c2_IP = m.group(3)

        observable = item['title']
        description = item['description'].lower()

        context = {}
        context['description'] = "{} C2 server".format(c2_IP)
        context['date_added'] = parser.parse(item['pubDate'])
        context['source'] = self.name

        c2 = None
        e = None

        try:
            e = Observable.add_text(observable)
            if c2_IP:
                c2 = Ip.get_or_create(value=c2_IP)
                e.active_link_to(
                    c2,
                    "IP",
                    self.name,
                    clean_old=False)

        except ObservableValidationError as e:
            logging.error(e)
            logging.error(description)
            return

        tags = ['malware', 'c2', malware_family.lower(), 'crimeware']

        if malware_family == 'pony':
            tags.extend(['stealer', 'dropper'])
        elif malware_family == 'athena':
            tags.extend(['stealer', 'ddos'])
        elif malware_family in ['zeus', 'citadel','lokibot']:
            tags.extend(['banker'])

        if e:
            e.add_context(context)
            e.add_source("feed")
            e.tag(tags)

        if c2:
            c2.add_context(context)
            c2.add_source("feed")
            c2.tag(tags)
Пример #11
0
    def analyze(observable, results):
        links = set()
        json_result = CirclPDNSApi.fetch(observable, results.settings)
        json_string = json.dumps(json_result,
                                 sort_keys=True,
                                 indent=4,
                                 separators=(',', ': '))

        results.update(raw=json_string)
        result = {}
        result['source'] = 'circl_pdns_query'
        result['raw'] = json_string

        if isinstance(observable, Ip):
            for record in json_result:
                new = Observable.add_text(record['rrname'])
                new.add_source('analytics')
                links.update(
                    observable.link_to(
                        new,
                        source='Circl.lu Passive DNS',
                        description='{} record'.format(record['rrtype']),
                        first_seen=datetime.fromtimestamp(
                            record['time_first']),
                        last_seen=datetime.fromtimestamp(record['time_last'])))

        elif isinstance(observable, Hostname):
            for record in json_result:
                new = Observable.add_text(record["rdata"])
                new.add_source('analytics')
                links.update(
                    observable.link_to(
                        new,
                        source='Circl.lu Passive DNS',
                        description='{} record'.format(record['rrtype']),
                        first_seen=datetime.fromtimestamp(
                            record['time_first']),
                        last_seen=datetime.fromtimestamp(record['time_last'])))

        observable.add_context(result)
        return list(links)
    def analyze(self, line):
        first_seen, _type, family, hostname, url, status, registrar, ips, asns, countries = line

        tags = []
        tags += TYPE_DICT[_type]
        tags.append(family.lower())

        context = {
            "first_seen": parser.parse(first_seen),
            "status": status,
            "registrar": registrar,
            "countries": countries.split("|"),
            "asns": asns.split("|"),
            "source": self.name
        }

        url_obs = False
        hostname_obs = False
        try:
            url_obs = Url.get_or_create(value=url.rstrip())
            url_obs.add_context(context)
            url_obs.tag(tags)
        except (ObservableValidationError, UnicodeEncodeError) as e:
            logging.error("Invalid line: {}\nLine: {}".format(e, line))

        try:
            hostname = Observable.add_text(hostname)
            hostname.tag(tags + ['blocklist'])
        except (ObservableValidationError, UnicodeEncodeError) as e:
            logging.error("Invalid line: {}\nLine: {}".format(e, line))

        for ip in ips.split("|"):
            if ip != hostname and ip is not None and ip != '':
                try:
                    ip_obs = Ip.get_or_create(value=ip)
                    ip_obs.active_link_to((url_obs, hostname),
                                          "ip",
                                          self.name,
                                          clean_old=False)
                except (ObservableValidationError, UnicodeEncodeError) as e:
                    logging.error("Invalid Observable: {}".format(e))

                for asn in asns.split("|"):
                    try:
                        asn_obs = AutonomousSystem.get_or_create(value=asn)
                        asn_obs.active_link_to((hostname, ip_obs),
                                               "asn",
                                               self.name,
                                               clean_old=False)

                    except (ObservableValidationError,
                            UnicodeEncodeError) as e:
                        logging.error("Invalid Observable: {}".format(e))
Пример #13
0
    def analyze(self, item, pub_date):  # pylint: disable=arguments-differ
        s_re = "\[([^\]]*)] Type: (\w+) - IP: (\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3})"
        r = re.compile(s_re)
        m = r.match(item["description"])
        malware_family = ""
        c2_IP = ""
        if m:
            malware_family = m.group(2)
            c2_IP = m.group(3)

        observable = item["title"]
        description = item["description"].lower()

        context = {}
        context["description"] = "{} C2 server".format(c2_IP)
        context["date_added"] = pub_date
        context["source"] = self.name

        c2 = None
        e = None

        try:
            e = Observable.add_text(observable)
            if c2_IP:
                c2 = Ip.get_or_create(value=c2_IP)
                e.active_link_to(c2, "IP", self.name, clean_old=False)

        except ObservableValidationError as e:
            logging.error(e)
            logging.error(description)
            return

        tags = ["malware", "c2", malware_family.lower(), "crimeware"]

        if malware_family == "pony":
            tags.extend(["stealer", "dropper"])
        elif malware_family == "athena":
            tags.extend(["stealer", "ddos"])
        elif malware_family in ["zeus", "citadel", "lokibot"]:
            tags.extend(["banker"])

        if e:
            e.add_context(context)
            e.add_source(self.name)
            e.tag(tags)

        if c2:
            c2.add_context(context)
            c2.add_source(self.name)
            c2.tag(tags)
    def analyze(self, line):
        if not line or line[0].startswith("#"):
            return

        date, _type, family, hostname, url, status, registrar, ips, asns, countries = tuple(
            line)

        item_date = dateutil.parser.parse(date)
        max_age = yeti_config.get('limits', 'max_age')
        limit_date = datetime.now() - timedelta(days=max_age)

        if item_date < limit_date:
            return

        tags = []
        tags += TYPE_DICT[_type]
        tags.append(family.lower())

        context = {
            "first_seen": date,
            "status": status,
            "registrar": registrar,
            "countries": countries.split("|"),
            "asns": asns.split("|"),
            "source": self.name
        }

        try:
            url = Url.get_or_create(value=url.rstrip())
            url.add_context(context)
            url.tag(tags)

            hostname = Observable.add_text(hostname)
            hostname.tag(tags + ['blocklist'])

            for ip in ips.split("|"):
                if ip != hostname and ip is not None and ip != '':
                    try:
                        i = Ip.get_or_create(value=ip)
                        i.active_link_to(hostname,
                                         "First seen IP",
                                         self.name,
                                         clean_old=False)

                    except ObservableValidationError as e:
                        logging.error("Invalid Observable: {}".format(e))

        except ObservableValidationError as e:
            logging.error(e)
Пример #15
0
    def rdata_lookup(observable, api_key):
        links = set()

        for record in DNSDBApi.lookup('rdata', observable, api_key):
            new = Observable.add_text(record['rrname'])
            new.add_source('analytics')

            links.update(
                new.link_to(observable,
                            source='DNSDB Passive DNS',
                            description='{} record'.format(record['rrtype']),
                            first_seen=record['first_seen'],
                            last_seen=record['last_seen']))

        return list(links)
Пример #16
0
    def rdata_lookup(observable, api_key):
        links = set()

        for record in DNSDBApi.lookup('rdata', observable, api_key):
            new = Observable.add_text(record['rrname'])
            new.add_source('analytics')

            links.update(
                new.link_to(
                    observable,
                    source='DNSDB Passive DNS',
                    description='{} record'.format(record['rrtype']),
                    first_seen=record['first_seen'],
                    last_seen=record['last_seen']))

        return list(links)
Пример #17
0
    def each(cls, hostname, rtype=None, results=[]):
        generated = []
        h = Hostname.get_or_create(value=hostname.value)

        for rdata in results:
            logging.info("{} resolved to {} ({} record)".format(h.value, rdata, rtype))
            try:
                e = Observable.add_text(rdata)
                e.add_source("analytics")
                generated.append(e)
                l = Link.connect(h, e)
                l.add_history(tag=rtype, description='{} record'.format(rtype))
            except ObservableValidationError as e:
                logging.error("{} is not a valid datatype".format(rdata))

        h.analysis_done(cls.__name__)
        return generated
Пример #18
0
    def rrset_lookup(hostname, api_key):
        links = set()

        for record in DNSDBApi.lookup('rrset', hostname, api_key):
            for observable in record['rdata']:
                observable = Observable.add_text(observable)
                observable.add_source('analytics')

                links.update(
                    hostname.link_to(
                        observable,
                        source='DNSDB Passive DNS',
                        description='{} record'.format(record['rrtype']),
                        first_seen=record['first_seen'],
                        last_seen=record['last_seen']))

        return list(links)
Пример #19
0
    def each(cls, hostname, rtype=None, results=[]):
        generated = []
        h = Hostname.get_or_create(value=hostname.value)

        for rdata in results:
            logging.debug("{} resolved to {} ({} record)".format(h.value, rdata, rtype))
            try:
                e = Observable.add_text(rdata)
                e.add_source("analytics")
                generated.append(e)
            except ObservableValidationError as e:
                logging.error("{} is not a valid datatype".format(rdata))

        h.active_link_to(generated, "{} record".format(rtype), "ResolveHostnames")

        h.analysis_done(cls.__name__)
        return generated
Пример #20
0
    def rrset_lookup(hostname, api_key):
        links = set()

        for record in DNSDBApi.lookup('rrset', hostname, api_key):
            for observable in record['rdata']:
                observable = Observable.add_text(observable)
                observable.add_source('analytics')

                links.update(
                    hostname.link_to(observable,
                                     source='DNSDB Passive DNS',
                                     description='{} record'.format(
                                         record['rrtype']),
                                     first_seen=record['first_seen'],
                                     last_seen=record['last_seen']))

        return list(links)
Пример #21
0
    def rdata_lookup(observable, api_key):
        links = set()

        for record in DNSDBApi.lookup("rdata", observable, api_key):
            new = Observable.add_text(record["rrname"])
            new.add_source("analytics")

            links.update(
                new.link_to(
                    observable,
                    source="DNSDB Passive DNS",
                    description="{} record".format(record["rrtype"]),
                    first_seen=record["first_seen"],
                    last_seen=record["last_seen"],
                )
            )

        return list(links)
Пример #22
0
    def analyze(self, line):

        if not line or line[0].startswith("#"):
            return

        date, _type, family, hostname, url, status, registrar, ips, asns, countries = tuple(
            line)

        tags = []
        tags += TYPE_DICT[_type]
        tags.append(family.lower())

        context = {
            "first_seen": date,
            "status": status,
            "registrar": registrar,
            "countries": countries.split("|"),
            "asns": asns.split("|"),
            "source": self.name
        }

        try:
            url = Url.get_or_create(value=url.rstrip())
            url.add_context(context)
            url.tag(tags)

            hostname = Observable.add_text(hostname)
            hostname.tag(tags + ['blocklist'])

            for ip in ips.split("|"):
                if ip != hostname and ip is not None and ip != '':
                    try:
                        i = Ip.get_or_create(value=ip)
                        i.active_link_to(
                            hostname,
                            "First seen IP",
                            self.name,
                            clean_old=False)
                    except ObservableValidationError as e:
                        logging.error("Invalid Observable: {}".format(e))

        except ObservableValidationError as e:
            logging.error("Invalid line: {}\nLine: {}".format(e, line))
Пример #23
0
    def rrset_lookup(hostname, api_key):
        links = set()

        for record in DNSDBApi.lookup("rrset", hostname, api_key):
            for observable in record["rdata"]:
                observable = Observable.add_text(observable)
                observable.add_source("analytics")

                links.update(
                    hostname.link_to(
                        observable,
                        source="DNSDB Passive DNS",
                        description="{} record".format(record["rrtype"]),
                        first_seen=record["first_seen"],
                        last_seen=record["last_seen"],
                    )
                )

        return list(links)
Пример #24
0
    def each(cls, hostname, rtype=None, results=[]):
        generated = []
        h = Hostname.get_or_create(value=hostname.value)

        for rdata in results:
            logging.debug("{} resolved to {} ({} record)".format(
                h.value, rdata, rtype))
            try:
                e = Observable.add_text(rdata)
                e.add_source("analytics")
                generated.append(e)
            except ObservableValidationError as e:
                logging.error("{} is not a valid datatype".format(rdata))

        h.active_link_to(generated, "{} record".format(rtype),
                         "ResolveHostnames")

        h.analysis_done(cls.__name__)
        return generated
Пример #25
0
    def analyze(self, line):

        if not line or line[0].startswith("#"):
            return

        date, _type, family, hostname, url, status, registrar, ips, asns, countries = tuple(
            line)

        tags = []
        tags += TYPE_DICT[_type]
        tags.append(family.lower())

        context = {
            "first_seen": date,
            "status": status,
            "registrar": registrar,
            "countries": countries.split("|"),
            "asns": asns.split("|"),
            "source": self.name
        }

        try:
            url = Url.get_or_create(value=url)
            url.add_context(context)
            url.tag(tags)

            hostname = Observable.add_text(hostname)
            hostname.tag(tags + ['blocklist'])

            for ip in ips.split("|"):
                if ip != hostname:
                    try:
                        i = Ip.get_or_create(value=ip)
                        i.active_link_to(hostname,
                                         "First seen IP",
                                         self.name,
                                         clean_old=False)
                    except ObservableValidationError as e:
                        logging.error("Invalid Observable: {}".format(e))

        except ObservableValidationError as e:
            logging.error("Invalid line: {}\nLine: {}".format(e, line))
Пример #26
0
def match_observables(observables):
    # Remove empty observables
    observables = [observable for observable in observables if observable]
    extended_query = set(observables) | set(derive(observables))
    added_entities = set()

    data = {"matches": [], "unknown": set(observables), "entities": [], "known": [], "neighbors": []}

    for o in Observable.objects(value__in=list(extended_query)):
        data['known'].append(o.info())
        del_from_set(data['unknown'], o.value)

        for link, node in (o.incoming()):
            if isinstance(node, Observable):
                if (link.src.value not in extended_query or link.dst.value not in extended_query) and node.tags:
                    data['neighbors'].append((link.info(), node.info()))

    for o, i in Indicator.search(extended_query):
        o = Observable.add_text(o)
        match = i.info()
        match.update({"observable": o.info(), "related": [], "suggested_tags": set()})

        for nodes in i.neighbors().values():
            for l, node in nodes:
                # add node name and link description to indicator
                node_data = {"entity": node.type, "name": node.name, "link_description": l.description or l.tag}
                match["related"].append(node_data)

                # uniquely add node information to related entitites
                if node.name not in added_entities:
                    nodeinfo = node.info()
                    nodeinfo['type'] = node.type
                    data["entities"].append(nodeinfo)
                    added_entities.add(node.name)

                o_tags = o.get_tags()
                [match["suggested_tags"].add(tag) for tag in node.generate_tags() if tag not in o_tags]

        data["matches"].append(match)
        del_from_set(data["unknown"], o.value)

    return data
Пример #27
0
    def __add_attribute(self, instance, attribute, context, tags):

        if attribute["category"] == "External analysis":
            return

        if attribute.get("type") in self.TYPES_TO_IMPORT:

            context["id"] = attribute["event_id"]
            context["link"] = urljoin(
                self.instances[instance]["url"],
                "/events/{}".format(attribute["event_id"]),
            )

            context["comment"] = attribute["comment"]

            obs = Observable.add_text(attribute["value"])

            if attribute["category"]:
                obs.tag(attribute["category"].replace(" ", "_"))

            if tags:
                obs.tag(tags)

            obs.add_context(context)
Пример #28
0
def match_observables(observables, save_matches=False, fetch_neighbors=True):
    # Remove empty observables
    observables, extended_query = derive(observables)
    observables = list(observables)

    data = {
        "matches": [],
        "unknown": set(observables),
        "entities": {},
        "known": [],
        "neighbors": [],
    }

    # add to "known"
    for o in Observable.objects(value__in=list(extended_query)):
        data['known'].append(o.info())
        del_from_set(data['unknown'], o.value)

        if fetch_neighbors:
            for link, node in (o.incoming()):
                if isinstance(node, Observable):
                    if (link.src.value not in extended_query or link.dst.value
                            not in extended_query) and node.tags:
                        data['neighbors'].append((link.info(), node.info()))

        for nodes in o.neighbors("Entity").values():
            for l, node in nodes:
                # add node name and link description to indicator
                node_data = {
                    "entity": node.type,
                    "name": node.name,
                    "link_description": l.description
                }

                # uniquely add node information to related entitites
                ent = data['entities'].get(node.name, node.info())
                if 'matches' not in ent:
                    ent['matches'] = {"observables": []}
                if 'observables' not in ent['matches']:
                    ent['matches']['observables'] = []

                info = node.info()
                o_info = o.info()
                info['matched_observable'] = {
                    "value": o_info['value'],
                    "tags": [t['name'] for t in o_info['tags']],
                    "human_url": o_info['human_url'],
                    "url": o_info['url'],
                    "context": o_info['context']
                }
                if info not in ent['matches']['observables']:
                    ent['matches']['observables'].append(info)
                data['entities'][node.name] = ent

    # add to "matches"
    for o, i in Indicator.search(extended_query):
        if save_matches:
            o = Observable.add_text(o)
        else:
            o = Observable.guess_type(o)(value=o)
            try:
                o.validate()
            except ObservableValidationError:
                pass
            try:
                o = Observable.objects.get(value=o.value)
            except Exception:
                pass

        match = i.info()
        match.update({
            "observable": o.info() if o.id else o.value,
            "related": [],
            "suggested_tags": set()
        })

        for nodes in i.neighbors("Entity").values():
            for l, node in nodes:
                # add node name and link description to indicator
                node_data = {
                    "entity": node.type,
                    "name": node.name,
                    "link_description": l.description
                }
                match["related"].append(node_data)

                # uniquely add node information to related entitites
                ent = data['entities'].get(node.name, node.info())
                if 'matches' not in ent:
                    ent['matches'] = {"indicators": []}
                if 'indicators' not in ent['matches']:
                    ent['matches']['indicators'] = []

                info = i.info()
                info['matched_observable'] = o.value
                if info not in ent['matches']['indicators']:
                    ent['matches']['indicators'].append(info)
                data['entities'][node.name] = ent

                o_tags = o.get_tags()
                for tag in node.generate_tags():
                    if tag not in o_tags:
                        match["suggested_tags"].add(tag)

        data["matches"].append(match)

    data['entities'] = data['entities'].values()
    return data
Пример #29
0
def match_observables(observables, save_matches=False, fetch_neighbors=True):
    # Remove empty observables
    observables = [refang(observable) for observable in observables if observable]
    extended_query = set(observables) | set(derive(observables))

    data = {
        "matches": [],
        "unknown": set(observables),
        "entities": {},
        "known": [],
        "neighbors": [],
    }

    # add to "known"
    for o in Observable.objects(value__in=list(extended_query)):
        data['known'].append(o.info())
        del_from_set(data['unknown'], o.value)

        if fetch_neighbors:
            for link, node in (o.incoming()):
                if isinstance(node, Observable):
                    if (link.src.value not in extended_query or link.dst.value not in extended_query) and node.tags:
                        data['neighbors'].append((link.info(), node.info()))

        for nodes in o.neighbors("Entity").values():
            for l, node in nodes:
                # add node name and link description to indicator
                node_data = {"entity": node.type, "name": node.name, "link_description": l.description}

                # uniquely add node information to related entitites
                ent = data['entities'].get(node.name, node.info())
                if 'matches' not in ent:
                    ent['matches'] = {"observables": []}
                if 'observables' not in ent['matches']:
                    ent['matches']['observables'] = []

                info = node.info()
                o_info = o.info()
                info['matched_observable'] = {
                    "value": o_info['value'],
                    "tags": [t['name'] for t in o_info['tags']],
                    "human_url": o_info['human_url'],
                    "url": o_info['url']
                }
                if info not in ent['matches']['observables']:
                    ent['matches']['observables'].append(info)
                data['entities'][node.name] = ent

    # add to "matches"
    for o, i in Indicator.search(extended_query):
        if save_matches:
            o = Observable.add_text(o)
        else:
            o = Observable.guess_type(o)(value=o)
            try:
                o.validate()
            except ObservableValidationError:
                pass
            try:
                o = Observable.objects.get(value=o.value)
            except Exception:
                pass

        match = i.info()
        match.update({"observable": o.info(), "related": [], "suggested_tags": set()})

        for nodes in i.neighbors("Entity").values():
            for l, node in nodes:
                # add node name and link description to indicator
                node_data = {"entity": node.type, "name": node.name, "link_description": l.description}
                match["related"].append(node_data)

                # uniquely add node information to related entitites
                ent = data['entities'].get(node.name, node.info())
                if 'matches' not in ent:
                    ent['matches'] = {"indicators": []}
                if 'indicators' not in ent['matches']:
                    ent['matches']['indicators'] = []

                info = i.info()
                info['matched_observable'] = o.value
                if info not in ent['matches']['indicators']:
                    ent['matches']['indicators'].append(info)
                data['entities'][node.name] = ent

                o_tags = o.get_tags()
                [match["suggested_tags"].add(tag) for tag in node.generate_tags() if tag not in o_tags]

        data["matches"].append(match)

    data['entities'] = data['entities'].values()
    return data
Пример #30
0
et.save()

et = ExportTemplate(name="Bluecoat")
et.template = """define category cert_blocklist
{% for obs in elements %}{{ obs.value }}
{% endfor %}end
"""
et.save()
Export(name="TestExport",
       acts_on="Url",
       description="Test description",
       frequency=timedelta(hours=1),
       include_tags=[t1, t2],
       template=et).save()

url = Observable.add_text("hxxp://zeuscpanel.com/gate.php")
url.tag(["zeus", "banker", "cc", "c2"])
print url.tags

# print url.find_tags()

# import pdb; pdb.set_trace()

## Create some instances of malware & co
bartalex = Malware.get_or_create(name="Bartalex")
bartalex.family = MalwareFamily.objects.get(name="dropper")
bartalex.killchain = "3"
bartalex.tags = ["bartalex"]
bartalex.save()

dridex = Malware.get_or_create(name="Dridex")
Пример #31
0
MalwareFamily("rootkit").save()
MalwareFamily("trojan").save()
MalwareFamily("dropper").save()


t1 = Tag.get_or_create(name="zeus").add_produces(["crimeware", "banker", "malware"])
t2 = Tag.get_or_create(name="banker").add_produces(["crimeware", "malware"])
t3 = Tag.get_or_create(name="c2")
t3.add_replaces(["c&c", "cc"])

Tag.get_or_create(name="crimeware").add_produces("malware")

Export(name="TestExport", description="Test description", frequency=timedelta(hours=1), include_tags=[t1, t2]).save()


url = Observable.add_text("hxxp://zeuscpanel.com/gate.php")
url.tag(["zeus", "banker", "cc", "c2"])
print url.tags

# print url.find_tags()

# import pdb; pdb.set_trace()



## Create some instances of malware & co
bartalex = Malware.get_or_create(name="Bartalex")
bartalex.family = MalwareFamily.objects.get(name="dropper")
bartalex.killchain = "delivery"
bartalex.tags = ["bartalex"]
bartalex.save()
Пример #32
0
    def analyze(self, item):
        first_seen = item["first_seen_utc"]
        ioc_value = item["ioc_value"]
        ioc_type = item["ioc_type"]
        threat_type = item["threat_type"]
        malware_alias = item["malware_alias"]
        malware_printable = item["malware_printable"]
        last_seen_utc = item["last_seen_utc"]
        confidence_level = item["confidence_level"]
        reference = item["reference"]
        reporter = item["reporter"]
        tags = []

        context = {"source": self.name}
        context["first_seen"] = first_seen

        if reference:
            context["reference"] = reference
        else:
            context["reference"] = "Unknown"

        if reporter:
            context["reporter"] = reporter
        else:
            context["reporter"] = "Unknown"

        if threat_type:
            context["threat_type"] = threat_type

        if item["tags"]:
            tags.extend(item["tags"].split(","))

        if malware_printable:
            tags.append(malware_printable)

        if malware_alias:
            context["malware_alias"] = malware_alias

        if last_seen_utc:
            context["last_seen_utc"] = last_seen_utc

        if confidence_level:
            context["confidence_level"] = confidence_level

        value = None
        obs = None
        try:
            if "ip" in ioc_type:
                value, port = ioc_value.split(":")
                context["port"] = port
                obs = Ip.get_or_create(value=value)
            else:
                obs = Observable.add_text(ioc_value)

        except ObservableValidationError as e:
            logging.error(e)
            return

        if obs:
            obs.add_context(context)
            obs.add_source(self.name)
            if tags:
                obs.tag(tags)
            if malware_printable:
                obs.tags