Пример #1
0
    def analyze(self, line):
        if not line or line[0].startswith("#"):
            return

        tokens = line.split(',')
        c2_domain = []
        ips_c2 = []
        names_servers = []
        ip_names_servers = []
        context_feed = []
        if len(tokens) == 6:
            c2_domain = tokens[0]
            ips_c2 = tokens[1].split('|')
            names_servers = tokens[2].split('|')
            ip_names_servers = tokens[3].split('|')
            context_feed = tokens[4]

            m = BambenekOsintIpmaster.reg.match(context_feed)
            malware_family = ''
            if m:
                malware_family = m.group(1)

            context = {
                "status": context_feed,
                "name servers": names_servers,
                "source": self.name
            }
            tags = [malware_family]
            c2 = None
            if c2_domain:
                c2 = Hostname.get_or_create(value=c2_domain)
                c2.add_context(context)
                c2.tag(tags)
                c2.add_source('feed')
                for ip in ips_c2:
                    if ip:
                        ip_obs = Ip.get_or_create(value=ip)
                        ip_obs.tag(tags)
                        ip_obs.add_source('feed')
                        if c2:
                            c2.active_link_to(ip_obs, "IP", self.source)

            for name_server in names_servers:
                if name_server:
                    ns_obs = Hostname.get_or_create(value=name_server)
                    c2.active_link_to(ns_obs, 'NS', self.source)
                    ns_obs.tag(tags)
                    ns_obs.add_context(context)
                    ns_obs.add_source('feed')
            for ip_ns in ip_names_servers:
                if ip_ns:
                    ip_ns_obs = Ip.get_or_create(value=ip_ns)
                    c2.active_link_to(ip_ns_obs, 'IP NS', self.source)
                    ip_ns_obs.tag(tags)
                    ip_ns_obs.add_context(context)
                    ip_ns_obs.add_source('feed')
        else:
            logging.error('Parsing error in line: %s' % line)
Пример #2
0
    def analyze(self, line):
        if not line or line[0].startswith("#"):
            return

        tokens = line.split(',')
        c2_domain = []
        ips_c2 = []
        names_servers = []
        ip_names_servers = []
        context_feed = []
        if len(tokens) == 6:
            c2_domain = tokens[0]
            ips_c2 = tokens[1].split('|')
            names_servers = tokens[2].split('|')
            ip_names_servers = tokens[3].split('|')
            context_feed = tokens[4]

            m = BambenekOsintIpmaster.reg.match(context_feed)
            malware_family = ''
            if m:
                malware_family = m.group(1)

            context = {
                "status": context_feed,
                "name servers": names_servers,
                "source": self.name
            }
            tags = [malware_family]
            c2 = None
            if c2_domain:
                c2 = Hostname.get_or_create(value=c2_domain)
                c2.add_context(context)
                c2.tag(tags)
                c2.add_source('feed')
                for ip in ips_c2:
                    if ip:
                        ip_obs = Ip.get_or_create(value=ip)
                        ip_obs.tag(tags)
                        ip_obs.add_source('feed')
                        if c2:
                            c2.active_link_to(ip_obs, "IP", self.source)

            for name_server in names_servers:
                if name_server:
                    ns_obs = Hostname.get_or_create(value=name_server)
                    c2.active_link_to(ns_obs, 'NS', self.source)
                    ns_obs.tag(tags)
                    ns_obs.add_context(context)
                    ns_obs.add_source('feed')
            for ip_ns in ip_names_servers:
                if ip_ns:
                    ip_ns_obs = Ip.get_or_create(value=ip_ns)
                    c2.active_link_to(ip_ns_obs, 'IP NS', self.source)
                    ip_ns_obs.tag(tags)
                    ip_ns_obs.add_context(context)
                    ip_ns_obs.add_source('feed')
        else:
            logging.error('Parsing error in line: %s' % line)
Пример #3
0
    def analyze(self, line):

        tokens = line.split(",")
        c2_domain = []
        ips_c2 = []
        names_servers = []
        ip_names_servers = []
        context_feed = []
        if len(tokens) == 6:
            c2_domain = tokens[0]
            ips_c2 = tokens[1].split("|")
            names_servers = tokens[2].split("|")
            ip_names_servers = tokens[3].split("|")
            context_feed = tokens[4]

            m = BambenekOsintIpmaster.reg.match(context_feed)
            malware_family = ""
            if m:
                malware_family = m.group(1)

            context = {
                "status": context_feed,
                "name servers": names_servers,
                "source": self.name,
            }
            tags = [malware_family]
            c2 = None
            if c2_domain:
                c2 = Hostname.get_or_create(value=c2_domain)
                c2.add_context(context)
                c2.tag(tags)
                c2.add_source("feed")
                for ip in ips_c2:
                    if ip:
                        ip_obs = Ip.get_or_create(value=ip)
                        ip_obs.tag(tags)
                        ip_obs.add_source(self.name)
                        if c2:
                            c2.active_link_to(ip_obs, "IP", self.source)

            for name_server in names_servers:
                if name_server:
                    ns_obs = Hostname.get_or_create(value=name_server)
                    c2.active_link_to(ns_obs, "NS", self.source)
                    ns_obs.tag(tags)
                    ns_obs.add_context(context)
                    ns_obs.add_source(self.name)
            for ip_ns in ip_names_servers:
                if ip_ns:
                    ip_ns_obs = Ip.get_or_create(value=ip_ns)
                    c2.active_link_to(ip_ns_obs, "IP NS", self.source)
                    ip_ns_obs.tag(tags)
                    ip_ns_obs.add_context(context)
                    ip_ns_obs.add_source(self.name)
        else:
            logging.error("Parsing error in line: %s" % line)
Пример #4
0
    def analyze(ip, results):
        links = set()
        result = ShodanApi.fetch(ip, results.settings['shodan_api_key'])
        results.update(raw=pformat(result))

        if 'tags' in result and result['tags'] is not None:
            ip.tag(result['tags'])

        if 'asn' in result and result['asn'] is not None:
            o_asn = Text.get_or_create(value=result['asn'])
            links.update(ip.active_link_to(o_asn, 'asn#', 'Shodan Query'))

        if 'hostnames' in result and result['hostnames'] is not None:
            for hostname in result['hostnames']:
                h = Hostname.get_or_create(value=hostname)
                links.update(h.active_link_to(ip, 'A record', 'Shodan Query'))

        if 'isp' in result and result['isp'] is not None:
            o_isp = Company.get_or_create(name=result['isp'])
            links.update(ip.active_link_to(o_isp, 'hosting', 'Shodan Query'))

        for context in ip.context:
            if context['source'] == 'shodan_query':
                break
        else:
            # Remove the data part (Shodan Crawler Data, etc.)
            result.pop("data", None)

            result['source'] = 'shodan_query'
            ip.add_context(result)

        return list(links)
Пример #5
0
    def _process_data(json_result, observable):
        links = set()

        for key in ("undetected_communicating_file",
                    "detected_downloaded_file", "undetected_downloaded_file",
                    "detected_communicating_file"):
            for file_hash in json_result.get(key, []):
                new_hash = Hash.get_or_create(value=file_hash)
                new_hash.tag(observable.get_tags())
                links.update(
                    new_hash.active_link_to(observable, key, "malwares.com"))

        for host in json_result.get("hostname_history", {}).get("list", []):
            new_host = Hostname.get_or_create(value=host)
            new_host.tag(observable.get_tags())
            links.update(
                new_host.active_link_to(observable, "hostname",
                                        "malwares.com"))

        for key in ("detected_url", "undetected_url", "distribution_url"):
            for url in json_result.get(key, []):
                new_url = Url.get_or_create(value=url)
                new_url.tag(observable.get_tags())
                links.update(
                    new_url.active_link_to(observable, key, "malwares.com"))

        observable.add_context(json_result)
        return list(links)
Пример #6
0
    def analyze(observable, results):
        links = set()

        if isinstance(observable, Ip):
            params = {"q": observable.value, "rt": 2}
            json_result = ThreatMinerApi.fetch(observable, params, "host.php")
            _results, result = aux_checker(json_result)

            for r in _results:
                o_hostname = Hostname.get_or_create(value=r.get("domain"))
                links.update(observable.link_to(o_hostname,
                    description="a record", source="ThreatMiner",
                    first_seen=r["first_seen"], last_seen=r["last_seen"])
                )

            observable.add_context(result)

        elif isinstance(observable, Hostname):
            params = {"q": observable.value, "rt": 2}
            json_result = ThreatMinerApi.fetch(observable, params, "domain.php")

            _results, result = aux_checker(json_result)

            for r in _results:
                o_ip = Ip.get_or_create(value=r.get("ip"))
                links.update(observable.link_to(
                            o_ip,
                            description="a record",
                            source="ThreatMiner",
                            first_seen=r["first_seen"],
                            last_seen=r["last_seen"]
                ))

            observable.add_context(result)
        return list(links)
Пример #7
0
    def _process_data(json_result, observable):
        links = set()

        for page in json_result:
            if not page.get("page"):
                continue

            # IP iocs has more data than the rest
            if not isinstance(observable, Ip) and page['page'].get('ip'):
                new_ip = Ip.get_or_create(value=page['page']['ip'])
                links.update(
                    new_ip.active_link_to(observable, 'ip', 'UrlScanIo Query'))

            if not isinstance(observable,
                              Hostname) and page['page'].get('domain'):
                new_host = Hostname.get_or_create(value=page['page']['domain'])
                links.update(
                    new_host.active_link_to(observable, 'hostname',
                                            'UrlScanIo Query'))

            if not isinstance(observable, Url) and page['page'].get('url'):
                new_url = Url.get_or_create(value=page['page']['url'])
                links.update(
                    new_url.active_link_to(observable, 'url',
                                           'UrlScanIo Query'))

            links.update(UrlScanIoApi._process_asn_data(page, observable))
Пример #8
0
    def analyze(self, line):

        line = line.strip()
        sline = line.split()

        try:
            if line[0] != "#" and len(sline) > 3:  # ignore comments and entries with no clear reference

                if sline[0].isdigit():
                    sline.pop(0)

                hostname = sline[0]
                description = sline[1]
                reference = sline[2]
                if sline[3] == "relisted":
                    date = sline[4]
                else:
                    date = sline[3]

                context = {}
                context["source"] = self.name
                context["description"] = description  # malware, EK, etc
                context["reference"] = reference  # GG safe browsing, blog, other blacklist, etc...
                context["date_added"] = datetime.strptime(date, "%Y%m%d")

                try:
                    hn = Hostname.get_or_create(value=hostname)
                    hn.add_context(context)
                    hn.add_source("feed")
                    hn.tag([description])
                except ObservableValidationError as e:
                    logging.error(e)
        except Exception:
            pass
Пример #9
0
    def analyze(self, dict):
        context = dict

        date_string = re.search(r"\((?P<datetime>[\d\- :]+)\)", dict['title']).group('datetime')
        try:
            context['date_added'] = datetime.strptime(date_string, "%Y-%m-%d %H:%M:%S")
        except ValueError:
            pass

        g = re.match(r'^Host: (?P<host>.+), Version: (?P<version>\w)', dict['description'])
        g = g.groupdict()
        context['version'] = g['version']
        context['description'] = FeodoTracker.descriptions[g['version']]
        context['subfamily'] = FeodoTracker.variants[g['version']]
        context['source'] = self.name
        del context['title']

        variant_tag = FeodoTracker.variants[g['version']].lower()
        try:
            if re.search(r"[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}", g['host']):
                new = Ip.get_or_create(value=g['host'])
            else:
                new = Hostname.get_or_create(value=g['host'])
            new.add_context(context)
            new.add_source("feed")
            new.tag([variant_tag, 'malware', 'crimeware', 'banker', 'c2'])
        except ObservableValidationError as e:
            logging.error(e)
Пример #10
0
    def analyze(ip, results):
        links = set()
        result = ShodanApi.fetch(ip, results.settings['shodan_api_key'])
        results.update(raw=pformat(result))

        if 'tags' in result and result['tags'] is not None:
            ip.tag(result['tags'])

        if 'asn' in result and result['asn'] is not None:
            o_asn = Text.get_or_create(value=result['asn'])
            links.update(ip.active_link_to(o_asn, 'asn#', 'Shodan Query'))

        if 'hostnames' in result and result['hostnames'] is not None:
            for hostname in result['hostnames']:
                h = Hostname.get_or_create(value=hostname)
                links.update(h.active_link_to(ip, 'A record', 'Shodan Query'))

        if 'isp' in result and result['isp'] is not None:
            o_isp = Company.get_or_create(name=result['isp'])
            links.update(ip.active_link_to(o_isp, 'hosting', 'Shodan Query'))

        for context in ip.context:
            if context['source'] == 'shodan_query':
                break
        else:
            # Remove the data part (Shodan Crawler Data, etc.)
            result.pop("data", None)

            result['source'] = 'shodan_query'
            ip.add_context(result)

        return list(links)
Пример #11
0
    def analyze(observable, result):
        links = set()

        endpoint = "/ip_addresses/%s/resolutions" % observable.value
        api_key = result.settings["virutotal_api_key"]
        result = VirustotalApi.fetch(api_key, endpoint)

        if result:
            for data in result["data"]:
                context = {"source": "VirusTotal PDNS"}
                attributes = data["attributes"]
                hostname = Hostname.get_or_create(
                    value=attributes["host_name"])
                if "date" in attributes:
                    timestamp_date = attributes["date"]
                    date_last_resolv = datetime.fromtimestamp(
                        timestamp_date).isoformat()
                    context[hostname.value] = date_last_resolv

                    hostname.add_context({
                        "source": context["source"],
                        observable.value: date_last_resolv,
                    })
                links.update(
                    hostname.active_link_to(observable, "resolved",
                                            context["source"]))

        return list(links)
Пример #12
0
    def analyze(self, dict):

        hostname, date_string = dict['title'].split(' ')

        context = dict
        context['source'] = self.name
        context['status'] = re.search(r"Status: (?P<status>\S+)", dict['description']).group('status')
        context['date_added'] = datetime.strptime(date_string, "%Y-%m-%d")
        try:
            context['SBL'] = re.search(r"SBL: (?P<sbl>\S+),", dict['description']).group('sbl')
        except AttributeError as e:
            pass
        context['description'] = "Palevotracker C2"
        try:
            context['status'] = re.search(r"Status: (?P<status>\S+),", dict['description']).group('status')
        except AttributeError as e:
            pass

        try:
            hn = Hostname.get_or_create(value=hostname)
            hn.add_context(context)
            hn.add_source("feed")
            hn.tag(['palevo', 'c2', 'malware', 'crimeware', 'worm'])
        except ObservableValidationError as e:
            logging.error(e)
Пример #13
0
    def analyze(self, dict):
        context = dict

        date_string = re.search(r"\((?P<datetime>[\d\- :]+)\)",
                                dict['title']).group('datetime')
        try:
            context['date_added'] = datetime.strptime(date_string,
                                                      "%Y-%m-%d %H:%M:%S")
        except ValueError:
            pass

        g = re.match(r'^Host: (?P<host>.+), Version: (?P<version>\w)',
                     dict['description'])
        g = g.groupdict()
        context['version'] = g['version']
        context['description'] = FeodoTracker.descriptions[g['version']]
        context['subfamily'] = FeodoTracker.variants[g['version']]
        context['source'] = self.name
        del context['title']

        variant_tag = FeodoTracker.variants[g['version']].lower()
        try:
            if re.search(r"[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}",
                         g['host']):
                new = Ip.get_or_create(value=g['host'])
            else:
                new = Hostname.get_or_create(value=g['host'])
            new.add_context(context)
            new.add_source("feed")
            new.tag([variant_tag, 'malware', 'crimeware', 'banker', 'c2'])
        except ObservableValidationError as e:
            logging.error(e)
Пример #14
0
def whois_links(observable, whois):
    links = set()

    to_extract = [
        {
            "field": "organization",
            "type": Text,
            "label": "Registrant Organization",
        },
        {
            "field": "registrar",
            "type": Text,
            "label": "Registrar",
        },
        {
            "field": "contactEmail",
            "type": Email,
            "label": "Registrant Email",
        },
        {
            "field": "name",
            "type": Text,
            "label": "Registrant Name",
        },
        {
            "field": "telephone",
            "type": Text,
            "label": "Registrant Phone",
            "record_type": "phone",
        },
    ]

    for field in to_extract:
        if field["field"] in whois and whois[field["field"]] != "N/A":
            node = field["type"].get_or_create(value=whois[field["field"]])
            if field["type"] == Text:
                if "record_type" in field:
                    node.update(record_type=field["record_type"])
                else:
                    node.update(record_type=field["field"])

            links.update(
                observable.active_link_to(node, field["label"],
                                          "PassiveTotal"))

    if "nameServers" in whois:
        nameservers = []
        for ns in whois["nameServers"]:
            if ns not in ["No nameserver", "not.defined"]:
                try:
                    nameservers.append(Hostname.get_or_create(value=ns))
                except Exception as e:
                    print(e.with_traceback())

        if nameservers:
            links.update(
                observable.active_link_to(nameservers, "NS record",
                                          "PassiveTotal"))

    return list(links)
Пример #15
0
    def analyze(self, cnc):

        try:
            cnc_data = Hostname.get_or_create(value=cnc)
            cnc_data.add_context({"source": self.name})
            cnc_data.add_source(self.name)
        except ObservableValidationError as e:
            logging.error(e)
Пример #16
0
    def analyze(observable, results):
        links = set()
        context = {
            "source": "malwares.com",
        }
        params = {
            "api_key": results.settings["malwares_api_key"],
            "hostname": observable.value,
        }

        json_result = MalwaresApi.fetch(observable, params, "hostname/info")
        if json_result:
            context["raw"] = json_result
            for key in (
                    "undetected_communicating_file",
                    "detected_downloaded_file",
                    "undetected_downloaded_file",
                    "detected_communicating_file",
            ):
                item = json_result[key]

                for f in item["list"]:
                    new_hash = Hash.get_or_create(value=f["sha256"])

                    links.update(
                        new_hash.active_link_to(observable, key,
                                                "malwares.com"))
                context[key] = item["total"]

            for host in json_result.get("hostname_history",
                                        {}).get("list", []):
                new_host = Hostname.get_or_create(value=host)
                new_host.tag(observable.get_tags())
                links.update(
                    new_host.active_link_to(observable, "hostname",
                                            "malwares.com"))

            for key in ("detected_url", "undetected_url"):
                item = json_result[key]

                for i in item["list"]:
                    try:
                        new_url = Url.get_or_create(value=i["url"])
                        new_url.tag(observable.get_tags())
                        links.update(
                            new_url.active_link_to(observable, key,
                                                   "malwares.com"))
                    except ObservableValidationError:
                        logging.error("Url is not valid %s" % i["url"])
                context[key] = item["total"]
            ip_history = json_result["ip_history"]

            for i in ip_history["list"]:
                ip = Ip.get_or_create(value=i["ip"])
                links.update(
                    ip.active_link_to(observable, "ip_story", "malwares.com"))
            context["ip_story"] = ip_history["total"]
        return links
Пример #17
0
    def analyze(observable, results):
        links = set()
        context = {
            "source": "malwares.com",
        }
        params = {
            "api_key": results.settings["malwares_api_key"],
            "ip": observable.value,
        }

        json_result = MalwaresApi.fetch(observable, params, "ip/info")
        for key in (
                "undetected_communicating_file",
                "detected_downloaded_file",
                "undetected_downloaded_file",
                "detected_communicating_file",
        ):
            for item in json_result[key]["list"]:
                h = Hash.get_or_create(value=item["sha256"])
                links.update(
                    h.active_link_to(observable, key, context["source"]))
                h.add_context({
                    "source": context["source"],
                    "firs_seen": item["date"]
                })
            context[key] = json_result[key]["total"]

        for key in ("detected_url", "undetected_url"):
            for item in json_result[key]["list"]:
                url = Url.get_or_create(value=item["url"])
                links.update(
                    url.active_link_to(observable, key, context["source"]))
                url.add_context({
                    "source": context["source"],
                    "firs_seen": item["date"]
                })

            context[key] = json_result[key]["total"]
        hostname_history = json_result["hostname_history"]

        for item in hostname_history["list"]:
            try:
                hostname = Hostname.get_or_create(value=item["hostname"])
                links.update(
                    hostname.active_link_to(observable, "hostname_history",
                                            context["source"]))
                hostname.add_context({
                    "source": context["source"],
                    "firs_seen": item["date"]
                })
            except ObservableValidationError:
                logging.error("%s is not a hostname valid" % item["hostname"])

        context["hostname_history"] = hostname_history["total"]

        observable.add_context(context)
        return links
Пример #18
0
    def check_type(txt):
        from core.observables import Hostname
        try:
            localpart, hostname = txt.split("@")
        except Exception:
            return False

        return bool(
            Hostname.check_type(hostname)
            and re.match("^[a-zA-Z0-9!#%&'*+-/=?^_`{|}~]+$", localpart))
Пример #19
0
    def analyze(observable, results):
        links = set()

        data = DomainToolsApi.get("/{}/host-domains/".format(observable.value), results.settings)
        results.update(raw=json.dumps(data, indent=2))

        for record in data['response']['ip_addresses']['domain_names']:
            node = Hostname.get_or_create(value=record)
            links.update(node.active_link_to(observable, 'A record', 'DomainTools'))

        return list(links)
Пример #20
0
def whois_links(observable, whois):
    links = set()

    to_extract = [
        {
            "field": "organization",
            "type": Text,
            "label": "Registrant Organization",
        },
        {
            "field": "registrar",
            "type": Text,
            "label": "Registrar",
        },
        {
            "field": "contactEmail",
            "type": Email,
            "label": "Registrant Email",
        },
        {
            "field": "name",
            "type": Text,
            "label": "Registrant Name",
        },
        {
            "field": "telephone",
            "type": Text,
            "label": "Registrant Phone",
            "record_type": "phone"
        }
    ]

    for field in to_extract:
        if field['field'] in whois and whois[field['field']] != 'N/A':
            node = field['type'].get_or_create(value=whois[field['field']])
            if field['type'] == Text:
                if "record_type" in field:
                    node.update(record_type=field['record_type'])
                else:
                    node.update(record_type=field['field'])

            links.update(observable.active_link_to(node, field['label'], 'PassiveTotal'))

    if 'nameServers' in whois:
        nameservers = []
        for ns in whois['nameServers']:
            if ns not in ["No nameserver", "not.defined"]:
                try:
                    nameservers.append(Hostname.get_or_create(value=ns))
                except Exception, e:
                    print e

        if nameservers:
            links.update(observable.active_link_to(nameservers, "NS record", 'PassiveTotal'))
Пример #21
0
    def analyze(observable, results):
        links = set()

        data = DomainToolsApi.get("/{}/name-server-domains".format(observable.value), results.settings)
        results.update(raw=json.dumps(data, indent=2))

        for record in data['response']['primary_domains'] + data['response']['secondary_domains']:
            node = Hostname.get_or_create(value=record)
            links.update(node.active_link_to(observable, 'NS record', 'DomainTools'))

        return list(links)
Пример #22
0
    def analyze(observable, results):
        links = set()

        params = {"query": observable.value, "field": "nameserver"}

        data = PassiveTotalApi.get("/whois/search", results.settings, params)

        for record in data["results"]:
            domain = Hostname.get_or_create(value=record["domain"])
            links.update(whois_links(domain, record))

        return list(links)
Пример #23
0
    def analyze(observable, results):
        links = set()

        params = {'query': observable.value, 'field': 'nameserver'}

        data = PassiveTotalApi.get('/whois/search', results.settings, params)

        for record in data['results']:
            domain = Hostname.get_or_create(value=record['domain'])
            links.update(whois_links(domain, record))

        return list(links)
Пример #24
0
    def analyze(observable, results):
        links = []

        params = {'terms': observable.value, 'mode': 'purchase'}
        data = DomainToolsApi.get("/reverse-whois/", results.settings, params)

        for domain in data['response']['domains']:
            node = Hostname.get_or_create(value=domain)
            links += node.active_link_to(
                observable, 'Registrant Information', 'DomainTools')

        return links
Пример #25
0
    def analyze(observable, results):
        links = set()

        params = {
            'query': observable.value,
            'field': 'email'
        }

        data = PassiveTotalApi.get('/whois/search', results.settings, params)

        for record in data['results']:
            print record
            domain = Hostname.get_or_create(value=record['domain'])
            links.update(domain.active_link_to(observable, "Registrant Email", 'PassiveTotal'))

            for ns in record['nameServers']:
                if ns != "No nameserver":
                    ns = Hostname.get_or_create(value=ns)
                    links.update(domain.active_link_to(ns, "NS record", 'PassiveTotal'))

        return list(links)
Пример #26
0
    def analyze(observable, results):
        links = set()

        params = {'query': observable.value, 'field': 'nameserver'}

        data = PassiveTotalApi.get('/whois/search', results.settings, params)

        for record in data['results']:
            domain = Hostname.get_or_create(value=record['domain'])
            links.update(whois_links(domain, record))

        return list(links)
Пример #27
0
    def analyze(observable, results):
        links = []

        params = {'terms': observable.value, 'mode': 'purchase'}
        data = DomainToolsApi.get("/reverse-whois/", results.settings, params)

        for domain in data['response']['domains']:
            node = Hostname.get_or_create(value=domain)
            links += node.active_link_to(observable, 'Registrant Information',
                                         'DomainTools')

        return links
Пример #28
0
    def analyze(observable, results):
        links = []

        params = {"terms": observable.value, "mode": "purchase"}
        data = DomainToolsApi.get("/reverse-whois/", results.settings, params)

        for domain in data["response"]["domains"]:
            node = Hostname.get_or_create(value=domain)
            links += node.active_link_to(
                observable, "Registrant Information", "DomainTools"
            )

        return links
Пример #29
0
    def analyze(self, line):
        line = line.strip()
        parts = line.split()

        hostname = str(parts[1]).strip()
        context = {'source': self.name}

        try:
            host = Hostname.get_or_create(value=hostname)
            host.add_context(context)
            host.add_source(self.name)
            host.tag(['malware', 'blocklist'])
        except ObservableValidationError as e:
            logging.error(e)
Пример #30
0
    def analyze(observable, results):
        links = set()

        params = {
            'query': '*.{}'.format(observable.value)
        }

        data = PassiveTotalApi.get('/enrichment/subdomains', results.settings, params)

        for record in data['subdomains']:
            subdomain = Hostname.get_or_create(value='{}.{}'.format(record, observable.value))
            links.update(observable.active_link_to(subdomain, "Subdomain", 'PassiveTotal'))

        return list(links)
Пример #31
0
    def analyze(observable, results):
        links = set()

        params = {
            'query': '*.{}'.format(observable.value)
        }

        data = PassiveTotalApi.get('/enrichment/subdomains', results.settings, params)

        for record in data['subdomains']:
            subdomain = Hostname.get_or_create(value='{}.{}'.format(record, observable.value))
            links.update(observable.active_link_to(subdomain, "Subdomain", 'PassiveTotal'))

        return list(links)
Пример #32
0
    def analyze_hostname(cls, hostname, results):
        """Specific analyzer for Hostname observables."""

        links = set()
        result = []
        for rec in itertools.chain(
                db.passive.get(
                    db.passive.searchdns(hostname.value, subdomains=True)),
                db.passive.get(
                    db.passive.searchdns(hostname.value,
                                         reverse=True,
                                         subdomains=True)),
        ):
            LOG.debug("%s.analyze_hostname: record %r", cls.__name__, rec)
            host = Hostname.get_or_create(value=rec["value"])
            if "addr" in rec:
                links.update(
                    Ip.get_or_create(value=rec["addr"]).link_to(
                        host,
                        "dns-%s" % rec["source"].split("-", 1)[0],
                        "IVRE - DNS-%s" % rec["source"],
                        first_seen=rec["firstseen"],
                        last_seen=rec["lastseen"],
                    ))
            else:
                links.update(
                    host.link_to(
                        Hostname.get_or_create(value=rec["targetval"]),
                        "dns-%s" % rec["source"].split("-", 1)[0],
                        "IVRE - DNS-%s" % rec["source"],
                        first_seen=rec["firstseen"],
                        last_seen=rec["lastseen"],
                    ))
                result.append(rec)

        results.update(raw=pformat(result))
        return list(links)
Пример #33
0
    def analyze(observable, results):
        links = set()

        params = {"query": "*.{}".format(observable.value)}

        data = PassiveTotalApi.get("/enrichment/subdomains", results.settings,
                                   params)

        for record in data["subdomains"]:
            subdomain = Hostname.get_or_create(
                value="{}.{}".format(record, observable.value))
            links.update(
                observable.active_link_to(subdomain, "Subdomain",
                                          "PassiveTotal"))

        return list(links)
Пример #34
0
    def analyze(observable, results):
        links = set()

        if isinstance(observable, Ip):
            params = {"q": observable.value, "rt": 2}
            json_result = ThreatMinerApi.fetch(observable, params, "host.php")
            _results, result = aux_checker(json_result)

            for r in _results:
                try:
                    o_hostname = Hostname.get_or_create(value=r.get("domain"))
                    links.update(
                        observable.link_to(
                            o_hostname,
                            description="a record",
                            source="ThreatMiner",
                            first_seen=parser.parse(r["first_seen"]),
                            last_seen=parser.parse(r["last_seen"]),
                        ))
                except ObservableValidationError as e:
                    logging.error("Caught an exception: {}".format(e))

            observable.add_context(result)

        elif isinstance(observable, Hostname):
            params = {"q": observable.value, "rt": 2}
            json_result = ThreatMinerApi.fetch(observable, params,
                                               "domain.php")

            _results, result = aux_checker(json_result)

            for r in _results:
                try:
                    o_ip = Ip.get_or_create(value=r.get("ip"))
                    links.update(
                        observable.link_to(
                            o_ip,
                            description="a record",
                            source="ThreatMiner",
                            first_seen=r["first_seen"],
                            last_seen=r["last_seen"],
                        ))
                except ObservableValidationError as e:
                    logging.error("Caught an exception: {}".format(e))

            observable.add_context(result)
        return list(links)
Пример #35
0
    def analyze(self, line):
        line = line.strip()
        sline = line.split()

        hostname = sline[0]

        context = {}
        context['source'] = self.name
        context['provider'] = sline[0]

        try:
            hostname = Hostname.get_or_create(value=hostname)
            hostname.add_context(context)
            hostname.add_source(self.name)
            hostname.tag('dyndns')
        except ObservableValidationError:
            pass
Пример #36
0
    def analyze(self, line):
        try:
            line = line.strip()
            parts = line.split()

            hostname = str(parts[1]).strip()
            context = {"source": self.name}

            try:
                host = Hostname.get_or_create(value=hostname)
                host.add_context(context)
                host.add_source(self.name)
                host.tag(["fraud", "blocklist"])
            except ObservableValidationError as e:
                logging.error(e)
        except Exception as e:
            logging.debug(e)
Пример #37
0
    def each(cls, hostname, rtype=None, results=[]):
        generated = []
        h = Hostname.get_or_create(value=hostname.value)

        for rdata in results:
            logging.info("{} resolved to {} ({} record)".format(h.value, rdata, rtype))
            try:
                e = Observable.add_text(rdata)
                e.add_source("analytics")
                generated.append(e)
                l = Link.connect(h, e)
                l.add_history(tag=rtype, description='{} record'.format(rtype))
            except ObservableValidationError as e:
                logging.error("{} is not a valid datatype".format(rdata))

        h.analysis_done(cls.__name__)
        return generated
Пример #38
0
    def analyze(observable, result):
        links = set()
        endpoint = "/domains/%s/subdomains" % observable.value
        api_key = result.settings["virutotal_api_key"]
        result = VirustotalApi.fetch(api_key, endpoint)

        if result:
            for data in result["data"]:
                context = {"source": "VirusTotal"}
                attributes = data["attributes"]
                sub_domain = Hostname.get_or_create(value=data["id"])
                links.update(
                    VirustotalApi.process_domain(sub_domain, attributes))
                links.update(
                    sub_domain.active_link_to(observable, "subdomain",
                                              context["source"]))
        return list(links)
Пример #39
0
    def each(cls, hostname, rtype=None, results=[]):
        generated = []
        h = Hostname.get_or_create(value=hostname.value)

        for rdata in results:
            logging.debug("{} resolved to {} ({} record)".format(h.value, rdata, rtype))
            try:
                e = Observable.add_text(rdata)
                e.add_source("analytics")
                generated.append(e)
            except ObservableValidationError as e:
                logging.error("{} is not a valid datatype".format(rdata))

        h.active_link_to(generated, "{} record".format(rtype), "ResolveHostnames")

        h.analysis_done(cls.__name__)
        return generated
Пример #40
0
    def analyze(self, line):
        if not line or line[0].startswith("#"):
            return

        try:
            ip, domain, family, md5, link, date = tuple(map(strip, line))
            context = {
                "first_seen": date,
                "family": family,
                "report": link,
                "source": self.name
            }

            c2 = None
            sample = None

            try:
                sample = Hash.get_or_create(value=md5)
                sample.add_context(context)
                sample.tag(family.lower())
            except ObservableValidationError as e:
                logging.error("Invalid line: {}\nLine: {}".format(e, line))

            try:
                if domain:
                    if '/' in domain:
                        c2 = Url.get_or_create(value=domain)
                    else:
                        c2 = Hostname.get_or_create(value=domain)
                elif ip:
                    c2 = Ip.get_or_create(value=ip)
                else:
                    return

                c2.add_context(context)
                c2.tag(['c2', family.lower()])

            except ObservableValidationError as e:
                logging.error("Invalid line: {}\nLine: {}".format(e, line))

            if c2 and sample:
                sample.active_link_to(c2, 'c2', self.name, clean_old=False)

        except ValueError:
            logging.error("Error unpacking line: {}".format(line))
Пример #41
0
    def analyze(self, line):
        if not line or line[0].startswith("#"):
            return

        try:
            ip, domain, family, md5, link, date = tuple(map(strip, line))
            context = {
                "first_seen": date,
                "family": family,
                "report": link,
                "source": self.name
            }

            c2 = None
            sample = None

            try:
                sample = Hash.get_or_create(value=md5)
                sample.add_context(context)
                sample.tag(family.lower())
            except ObservableValidationError as e:
                logging.error("Invalid line: {}\nLine: {}".format(e, line))

            try:
                if domain:
                    if '/' in domain:
                        c2 = Url.get_or_create(value=domain)
                    else:
                        c2 = Hostname.get_or_create(value=domain)
                elif ip:
                    c2 = Ip.get_or_create(value=ip)
                else:
                    return

                c2.add_context(context)
                c2.tag(['c2', family.lower()])

            except ObservableValidationError as e:
                logging.error("Invalid line: {}\nLine: {}".format(e, line))

            if c2 and sample:
                sample.active_link_to(c2, 'c2', self.name, clean_old=False)

        except ValueError:
            logging.error("Error unpacking line: {}".format(line))
Пример #42
0
    def analyze(self, line):
        line = line.strip()
        sline = line.split()

        try:
            if line[0] != '#':
                hostname = sline[0]

                context = {}
                context['source'] = self.name
                context['provider'] = sline[0]

                hostname = Hostname.get_or_create(value=hostname)
                hostname.add_context(context)
                hostname.add_source("feed")
                hostname.tag('dyndns')
        except Exception:
            pass
Пример #43
0
    def analyze(observable, results):
        links = set()

        if isinstance(observable, Email):
            field = 'email'
        elif observable.record_type:
            field = observable.record_type
        else:
            raise ValueError("Could not determine field for this observable")

        params = {'query': observable.value, 'field': field}

        data = PassiveTotalApi.get('/whois/search', results.settings, params)

        for record in data['results']:
            domain = Hostname.get_or_create(value=record['domain'])
            links.update(whois_links(domain, record))

        return list(links)
Пример #44
0
    def each(cls, hostname, rtype=None, results=[]):
        parts = extract(hostname.value)

        if parts.suffix in SUSPICIOUS_TLDS:
            hostname.tag('suspicious_tld')

        if parts.subdomain != '':
            hostname.update(domain=False)

            domain = Hostname.get_or_create(value=parts.registered_domain, domain=True)
            domain.add_source("analytics")
            hostname.active_link_to(domain, "domain", "ProcessHostnames", clean_old=False)

            if domain.has_tag('dyndns'):
                hostname.tag('dyndns')

            return domain
        else:
            hostname.update(domain=True)
            return None
Пример #45
0
    def analyze(observable, results):
        links = set()

        params = {
            'query': observable.value,
            'field': 'nameserver'
        }

        data = PassiveTotalApi.get('/whois/search', results.settings, params)

        for record in data['results']:
            domain = Hostname.get_or_create(value=record['domain'])
            links.update(domain.active_link_to(observable, "NS record", 'PassiveTotal'))

            registrant_email = get_value_at(record, 'registrant.email')
            if registrant_email:
                registrant = Email.get_or_create(value=registrant_email)
                links.update(domain.active_link_to(registrant, "Registrant Email", 'PassiveTotal'))

        return list(links)
Пример #46
0
    def analyze(self, line):
        if line.startswith('#'):
            return

        try:
            line = line.strip()
            parts = line.split()

            hostname = str(parts[1]).strip()
            context = {'source': self.name}

            try:
                host = Hostname.get_or_create(value=hostname)
                host.add_context(context)
                host.add_source('feed')
                host.tag(['phishing', 'blocklist'])
            except ObservableValidationError as e:
                logging.error(e)
        except Exception as e:
            logging.debug(e)
Пример #47
0
    def each(cls, hostname, rtype=None, results=[]):
        parts = extract(hostname.value)

        if parts.suffix in SUSPICIOUS_TLDS:
            hostname.tag('suspicious_tld')

        if parts.subdomain != '':
            hostname.update(domain=False)

            domain = Hostname.get_or_create(value=parts.registered_domain, domain=True)
            domain.add_source("analytics")
            l = Link.connect(hostname, domain)
            l.add_history(tag='domain')

            if domain.has_tag('dyndns'):
                hostname.tag('dyndns')

            return domain
        else:
            hostname.update(domain=True)
            return None
Пример #48
0
    def analyze(self, item):
        context = item
        date_string = re.search(
            r"\((?P<datetime>[\d\- :]+)\)", context['title']).group('datetime')
        try:
            context['date_added'] = datetime.strptime(
                date_string, "%Y-%m-%d %H:%M:%S")
        except ValueError:
            pass

        g = re.match(
            r'^Host: (?P<host>.+), Version: (?P<version>\w)',
            context['description'])
        g = g.groupdict()
        context['version'] = g['version']
        context['description'] = FeodoTracker.descriptions[g['version']]
        context['subfamily'] = FeodoTracker.variants[g['version']]
        context['source'] = self.name
        del context['title']
        new = None
        variant_tag = FeodoTracker.variants[g['version']].lower()
        try:
            if re.search(r"[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}",
                         g['host']):
                new = Ip.get_or_create(value=g['host'])
            else:
                new = Hostname.get_or_create(value=g['host'])
            new.add_context(context)
            new.add_source("feed")
            new.tag([variant_tag, 'malware', 'crimeware', 'banker', 'c2'])

        except ObservableValidationError as e:
            logging.error(e)

        try:
            url_fedeo = context['guid']
            r = requests.get(url_fedeo)
            if r.status_code == 200:

                html_source = r.text

                soup = BeautifulSoup(html_source, 'html.parser')
                tab = soup.find('table', attrs='sortable')
                results = []

                if tab:
                    all_tr = tab.find_all('tr')

                    for tr in all_tr:
                        all_td = tr.find_all('td')
                        if all_td and len(all_td) == 7:
                            results.append({
                                'timestamp': all_td[0].text,
                                'md5_hash': all_td[1].text,
                                'filesize': all_td[2].text,
                                'VT': all_td[3].text,
                                'Host': all_td[4].text,
                                'Port': all_td[5].text,
                                'SSL Certif or method': all_td[6].text
                            })

                for r in results:
                    new_hash = Hash.get_or_create(value=r['md5_hash'])
                    new_hash.add_context(context)
                    new_hash.add_source('feed')
                    new_hash.tag([
                        variant_tag, 'malware', 'crimeware', 'banker', 'payload'
                    ])
                    new_hash.active_link_to(
                        new, 'c2', self.name, clean_old=False)
                    host = Url.get_or_create(
                        value='https://%s:%s' % (g['host'], r['Port']))
                    host.add_source('feed')
                    host.add_context(context)
                    host.tag(
                        [variant_tag, 'malware', 'crimeware', 'banker', 'c2'])
                    new_hash.active_link_to(
                        host, 'c2', self.name, clean_old=False)

        except ObservableValidationError as e:
            logging.error(e)
Пример #49
0
    def analyze(observable, results):
        links = set()
        json_result = ThreatCrowdAPI.fetch(observable)
        json_string = json.dumps(
            json_result, sort_keys=True, indent=4, separators=(',', ': '))
        results.update(raw=json_string)
        result = {}
        if isinstance(observable, Hostname):

            if 'resolutions' in json_result:

                result['ip on this domains'] = 0

                for ip in json_result['resolutions']:
                    if ip['ip_address'].strip() != observable.value:
                        if ip['last_resolved'] != '0000-00-00':
                            last_resolved = datetime.datetime.strptime(
                                ip['last_resolved'], "%Y-%m-%d")
                            try:
                                new_ip = Ip.get_or_create(
                                    value=ip['ip_address'].strip())
                                links.update(
                                    new_ip.active_link_to(
                                        observable, 'IP', 'ThreatCrowd',
                                        last_resolved))
                                result['ip on this domains'] += 1
                            except ObservableValidationError:
                                logging.error(
                                    "An error occurred when trying to add subdomain {} to the database".
                                    format(ip['ip_address']))

            if 'emails' in json_result:

                result['nb emails'] = 0

                for email in json_result['emails']:
                    try:
                        new_email = Email.get_or_create(value=email)
                        links.update(
                            new_email.active_link_to(
                                observable, 'Used by', 'ThreatCrowd'))
                        result['nb emails'] += 1
                    except ObservableValidationError:
                        logging.error(
                            "An error occurred when trying to add email {} to the database".
                            format(email))

            if 'subdomains' in json_result:

                result['nb subdomains'] = 0

                for subdomain in json_result['subdomains']:
                    try:
                        new_domain = Hostname.get_or_create(value=subdomain)
                        links.update(
                            observable.active_link_to(
                                new_domain, 'subdomain', 'ThreatCrowd'))
                        result['nb subdomains'] += 1
                    except ObservableValidationError:
                        logging.error(
                            "An error occurred when trying to add subdomain {} to the database".
                            format(subdomain))

        if isinstance(observable, Ip):

            if 'resolutions' in json_result:

                result['domains resolved'] = 0

                for domain in json_result['resolutions']:
                    if domain['domain'].strip() != observable.value:
                        try:
                            last_resolved = datetime.datetime.strptime(
                                domain['last_resolved'], "%Y-%m-%d")
                            new_domain = Hostname.get_or_create(
                                value=domain['domain'].strip())
                            links.update(
                                new_domain.active_link_to(
                                    observable, 'A Record', 'ThreatCrowd',
                                    last_resolved))
                            result['domains resolved'] += 1
                        except ObservableValidationError:
                            logging.error(
                                "An error occurred when trying to add domain {} to the database".
                                format(domain['domain']))

            if 'hashes' in json_result and len(json_result['hashes']) > 0:
                result['malwares'] = 0
                for h in json_result['hashes']:
                    new_hash = Hash.get_or_create(value=h)
                    links.update(
                        new_hash.active_link_to(
                            observable, 'hash', 'ThreatCrowd'))
                    result['malwares'] += 1

        if isinstance(observable, Email):
            if 'domains' in json_result and len(json_result) > 0:
                result['domains recorded by email'] = 0
                for domain in json_result['domains']:
                    new_domain = Hostname.get_or_create(value=domain)
                    links.update(
                        new_domain.active_link_to(
                            observable, 'recorded by', 'ThreatCrowd'))
                    result['domains recorded by email'] += 1

        if isinstance(observable, Hash):
            result['nb c2'] = 0

            if 'md5' in json_result:
                new_hash = Hash.get_or_create(value=json_result['md5'])
                links.update(
                    new_hash.active_link_to(observable, 'md5', 'ThreadCrowd'))

            if 'sha1' in json_result:
                new_hash = Hash.get_or_create(value=json_result['sha1'])
                links.update(
                    new_hash.active_link_to(observable, 'sha1', 'ThreadCrowd'))

            if 'sha256' in json_result:
                new_hash = Hash.get_or_create(value=json_result['sha256'])
                links.update(
                    new_hash.active_link_to(
                        observable, 'sha256', 'ThreadCrowd'))

            if 'domains' in json_result and len(json_result['domains']):
                for domain in json_result['domains']:
                    new_domain = Hostname.get_or_create(value=domain)
                    links.update(
                        observable.active_link_to(
                            new_domain, 'c2', 'ThreatCrowd'))
                    result['nb c2'] += 1

            if 'ips' in json_result and len(json_result['ips']):
                for ip in json_result['ips']:
                    new_ip = Ip.get_or_create(value=ip.strip())
                    links.update(
                        observable.active_link_to(new_ip, 'c2', 'ThreatCrowd'))
                    result['nb c2'] += 1

        if 'permalink' in json_result:
            result['permalink'] = json_result['permalink']

        result['source'] = 'threatcrowd_query'
        result['raw'] = json_string
        observable.add_context(result)
        return list(links)
Пример #50
0
    def _get_threat_forensics_nodes_inner(
            self, evidence, general_context, tags):
        # create context from notes
        context = general_context.copy()
        _ctx = self._make_context_from_notes([evidence])
        context.update(_ctx)
        # add evidence['type'] and unicify tags
        tags = [{
            'name': _
        } for _ in set([evidence['type']] + [d['name'] for d in tags])]
        # create Tags in DB
        for _ in tags:
            Tag.get_or_create(name=_['name'])
        #
        threat_forensics = []

        # technical hack: set optional comments values
        for optional in ['action', 'rule', 'path', 'rule']:
            if optional not in evidence['what']:
                evidence['what'][optional] = None

        # add attributes for the known evidence type
        if evidence['type'] in ['file', 'dropper']:
            if 'path' in evidence['what']:
                threat_forensics.append(
                    File.get_or_create(
                        value=evidence['what']['path'], context=[context]))
            if 'md5' in evidence['what']:
                threat_forensics.append(
                    Hash.get_or_create(
                        value=evidence['what']['md5'], context=[context]))
            if 'sha256' in evidence['what']:
                threat_forensics.append(
                    Hash.get_or_create(
                        value=evidence['what']['sha256'], context=[context]))
        elif evidence['type'] == 'cookie':
            pass
        elif evidence['type'] == 'dns':
            threat_forensics.append(
                Hostname.get_or_create(
                    value=evidence['what']['host'], context=[context]))
        elif evidence['type'] == 'ids':
            threat_forensics.append(
                Text.get_or_create(
                    value=evidence['what']['ids'], context=[context]))
            pass
        elif evidence['type'] == 'mutex':
            threat_forensics.append(
                Text.get_or_create(
                    value=evidence['what']['name'], context=[context]))
        elif evidence['type'] == 'network':
            if 'ip' in evidence['what']:
                # FIXME port, type
                threat_forensics.append(
                    Ip.get_or_create(
                        value=evidence['what']['ip'], context=[context]))
            elif 'domain' in evidence['what']:
                threat_forensics.append(
                    Hostname.get_or_create(
                        value=evidence['what']['domain'], context=[context]))
        elif evidence['type'] == 'process':
            pass
        elif evidence['type'] == 'registry':
            # threat_forensics.append(evidence['what']['key'])
            # threat_forensics.append(evidence['what']['value'])
            pass
        elif evidence['type'] == 'url':
            # BUG yeti-#115 ObservableValidationError: Invalid URL: http://xxxxx-no-tld/
            threat_forensics.append(
                Url.get_or_create(
                    value=evidence['what']['url'], context=[context]))
            # add note as tag because its a signature
            if 'note' in evidence:
                threat_forensics[-1].tag(
                    evidence['note'].replace('.', '_').strip('_'))
        # tag all of that
        for o in threat_forensics:
            o.tag([t['name'] for t in tags])
        return threat_forensics