Beispiel #1
0
    def analyze(observable, results):
        links = set()
        parts = extract(observable.value)

        if parts.subdomain == '':
            data = DomainToolsApi.get("/{}/whois/history".format(observable.value), results.settings)
            results.update(raw=json.dumps(data, indent=2))

            for record in data['response']['history']:
                created = datetime.strptime(record['whois']['registration']['created'], "%Y-%m-%d")
                expires = datetime.strptime(record['whois']['registration']['expires'], "%Y-%m-%d")

                registrar = Company.get_or_create(name=record['whois']['registration']['registrar'])
                registrant = Text.get_or_create(value=record['whois']['registrant'])

                links.update(observable.link_to(registrar, 'Registrar', 'DomainTools', created, expires))
                links.update(observable.link_to(registrant, 'Registrant', 'DomainTools', created, expires))

                parsed = parse_raw_whois([record['whois']['record']], normalized=True)
                email = get_value_at(parsed, 'contacts.registrant.email')
                if email:
                    email = Email.get_or_create(value=email)
                    links.update(observable.link_to(email, 'Registrant Email', 'DomainTools', created, expires))

        return list(links)
Beispiel #2
0
    def analyze(ip, results):
        links = set()

        r = IPWhois(ip.value)
        result = r.lookup_whois()
        results.update(raw=pformat(result))

        # Let's focus on the most specific information
        # Which should be in the smallest subnet
        n = 0
        smallest_subnet = None

        for network in result['nets']:
            cidr_bits = int(network['cidr'].split('/')[1].split(',')[0])
            if cidr_bits > n:
                n = cidr_bits
                smallest_subnet = network

        if smallest_subnet:
            # Create the company
            company = Company.get_or_create(
                name=smallest_subnet['description'].split("\n")[0])
            links.update(ip.active_link_to(company, 'hosting',
                                           'Network Whois'))

            # Link it to every email address referenced
            if smallest_subnet['emails']:
                for email_address in smallest_subnet['emails']:
                    email = Email.get_or_create(value=email_address)
                    links.update(company.link_to(email, None, 'Network Whois'))

            # Copy the subnet info into the main dict
            for key in smallest_subnet:
                if smallest_subnet[key]:
                    result["net_{}".format(key)] = smallest_subnet[key]

        # Add the network whois to the context if not already present
        for context in ip.context:
            if context['source'] == 'network_whois':
                break
        else:
            # Remove the nets info (the main one was copied)
            result.pop("nets", None)
            result.pop("raw", None)
            result.pop("raw_referral", None)
            result.pop("referral", None)
            result.pop("query", None)

            result['source'] = 'network_whois'
            ip.add_context(result)

        return list(links)
Beispiel #3
0
    def analyze(ip, results):
        links = set()

        r = IPWhois(ip.value)
        result = r.lookup_whois()
        results.update(raw=pformat(result))

        # Let's focus on the most specific information
        # Which should be in the smallest subnet
        n = 0
        smallest_subnet = None

        for network in result['nets']:
            cidr_bits = int(network['cidr'].split('/')[1].split(',')[0])
            if cidr_bits > n:
                n = cidr_bits
                smallest_subnet = network

        if smallest_subnet:
            # Create the company
            company = Company.get_or_create(
                name=smallest_subnet['description'].split("\n")[0])
            links.update(ip.active_link_to(company, 'hosting', 'Network Whois'))

            # Link it to every email address referenced
            if smallest_subnet['emails']:
                for email_address in smallest_subnet['emails']:
                    email = Email.get_or_create(value=email_address)
                    links.update(company.link_to(email, None, 'Network Whois'))

            # Copy the subnet info into the main dict
            for key in smallest_subnet:
                if smallest_subnet[key]:
                    result["net_{}".format(key)] = smallest_subnet[key]

        # Add the network whois to the context if not already present
        for context in ip.context:
            if context['source'] == 'network_whois':
                break
        else:
            # Remove the nets info (the main one was copied)
            result.pop("nets", None)
            result.pop("raw", None)
            result.pop("raw_referral", None)
            result.pop("referral", None)
            result.pop("query", None)

            result['source'] = 'network_whois'
            ip.add_context(result)

        return list(links)
Beispiel #4
0
    def analyze(observable, results):
        links = set()
        parts = tldextract_parser(observable.value)

        if parts.subdomain == "":
            data = DomainToolsApi.get(
                "/{}/whois/history".format(observable.value), results.settings
            )
            results.update(raw=json.dumps(data, indent=2))

            for record in data["response"]["history"]:
                created = datetime.strptime(
                    record["whois"]["registration"]["created"], "%Y-%m-%d"
                )
                expires = datetime.strptime(
                    record["whois"]["registration"]["expires"], "%Y-%m-%d"
                )

                registrar = Company.get_or_create(
                    name=record["whois"]["registration"]["registrar"]
                )
                registrant = Text.get_or_create(value=record["whois"]["registrant"])

                links.update(
                    observable.link_to(
                        registrar, "Registrar", "DomainTools", created, expires
                    )
                )
                links.update(
                    observable.link_to(
                        registrant, "Registrant", "DomainTools", created, expires
                    )
                )

                parsed = parse_raw_whois([record["whois"]["record"]], normalized=True)
                email = get_value_at(parsed, "contacts.registrant.email")
                if email:
                    email = Email.get_or_create(value=email)
                    links.update(
                        observable.link_to(
                            email, "Registrant Email", "DomainTools", created, expires
                        )
                    )

        return list(links)
Beispiel #5
0
    def analyze(observable, results):
        links = set()

        params = {
            'query': observable.value,
            'field': 'nameserver'
        }

        data = PassiveTotalApi.get('/whois/search', results.settings, params)

        for record in data['results']:
            domain = Hostname.get_or_create(value=record['domain'])
            links.update(domain.active_link_to(observable, "NS record", 'PassiveTotal'))

            registrant_email = get_value_at(record, 'registrant.email')
            if registrant_email:
                registrant = Email.get_or_create(value=registrant_email)
                links.update(domain.active_link_to(registrant, "Registrant Email", 'PassiveTotal'))

        return list(links)
Beispiel #6
0
def whois_links(observable, data):
    links = set()
    created_date = data.get("createdDate")
    expires_date = data.get("expiresDate")

    first_seen = convert_to_datetime(created_date)
    last_seen = convert_to_datetime(expires_date)

    registrar_name = data.get("registrarName")
    if registrar_name is not None:
        node = Text.get_or_create(value=registrar_name)
        try:
            links.update(
                observable.link_to(node, "Registrar name", SOURCE, first_seen,
                                   last_seen))
        except Exception as e:
            logger.error(e.message)

    contacts = data.get("contact", [])
    for contact in contacts:
        email = contact.get("email")
        if email is not None:
            node = Email.get_or_create(value=email)
            try:
                links.update(observable.link_to(node, "Contact email", SOURCE),
                             first_seen, last_seen)
            except Exception as e:
                logger.error(e.message)

    name_servers = data.get("nameServers", [])
    name_servers = name_servers if name_servers is not None else []
    for name_server in name_servers:
        node = Hostname.get_or_create(value=name_server)
        try:
            links.update(observable.link_to(node, "Name server", SOURCE),
                         first_seen, last_seen)
        except Exception as e:
            logger.error(e.message)

    return list(links)
Beispiel #7
0
    def analyze(ip):
        links = []

        results = IPWhois(ip.value)
        results = results.lookup_rdap()

        for entity in results['objects']:
            entity = results['objects'][entity]
            if entity['contact']['kind'] != 'individual':
                # Create the company
                company = Company.get_or_create(name=entity['contact']['name'], rdap=entity)
                link = Link.connect(ip, company)
                link.add_history('hosting')
                links.append(link)

                # Link it to every email address referenced
                for email_info in entity['contact']['email']:
                    email = Email.get_or_create(value=email_info['value'])
                    link = Link.connect(company, email)
                    links.append(link)

        return links
Beispiel #8
0
    def analyze(observable, results):
        links = set()
        json_result = ThreatCrowdAPI.fetch(observable)
        json_string = json.dumps(
            json_result, sort_keys=True, indent=4, separators=(",", ": ")
        )
        results.update(raw=json_string)
        result = {}
        if isinstance(observable, Hostname):

            if "resolutions" in json_result:

                result["ip on this domains"] = 0

                for ip in json_result["resolutions"]:
                    if ip["ip_address"].strip() != observable.value:
                        if ip["last_resolved"] != "0000-00-00":
                            last_resolved = datetime.datetime.strptime(
                                ip["last_resolved"], "%Y-%m-%d"
                            )
                            try:
                                new_ip = Ip.get_or_create(
                                    value=ip["ip_address"].strip()
                                )
                                links.update(
                                    new_ip.active_link_to(
                                        observable, "IP", "ThreatCrowd", last_resolved
                                    )
                                )
                                result["ip on this domains"] += 1
                            except ObservableValidationError:
                                logging.error(
                                    "An error occurred when trying to add subdomain {} to the database".format(
                                        ip["ip_address"]
                                    )
                                )

            if "emails" in json_result:

                result["nb emails"] = 0

                for email in json_result["emails"]:
                    try:
                        new_email = Email.get_or_create(value=email)
                        links.update(
                            new_email.active_link_to(
                                observable, "Used by", "ThreatCrowd"
                            )
                        )
                        result["nb emails"] += 1
                    except ObservableValidationError:
                        logging.error(
                            "An error occurred when trying to add email {} to the database".format(
                                email
                            )
                        )

            if "subdomains" in json_result:

                result["nb subdomains"] = 0

                for subdomain in json_result["subdomains"]:
                    try:
                        new_domain = Hostname.get_or_create(value=subdomain)
                        links.update(
                            observable.active_link_to(
                                new_domain, "subdomain", "ThreatCrowd"
                            )
                        )
                        result["nb subdomains"] += 1
                    except ObservableValidationError:
                        logging.error(
                            "An error occurred when trying to add subdomain {} to the database".format(
                                subdomain
                            )
                        )

        if isinstance(observable, Ip):

            if "resolutions" in json_result:

                result["domains resolved"] = 0

                for domain in json_result["resolutions"]:
                    if domain["domain"].strip() != observable.value:
                        try:
                            last_resolved = datetime.datetime.strptime(
                                domain["last_resolved"], "%Y-%m-%d"
                            )
                            new_domain = Hostname.get_or_create(
                                value=domain["domain"].strip()
                            )
                            links.update(
                                new_domain.active_link_to(
                                    observable, "A Record", "ThreatCrowd", last_resolved
                                )
                            )
                            result["domains resolved"] += 1
                        except ObservableValidationError:
                            logging.error(
                                "An error occurred when trying to add domain {} to the database".format(
                                    domain["domain"]
                                )
                            )

            if "hashes" in json_result and len(json_result["hashes"]) > 0:
                result["malwares"] = 0
                for h in json_result["hashes"]:
                    new_hash = Hash.get_or_create(value=h)
                    links.update(
                        new_hash.active_link_to(observable, "hash", "ThreatCrowd")
                    )
                    result["malwares"] += 1

        if isinstance(observable, Email):
            if "domains" in json_result and len(json_result) > 0:
                result["domains recorded by email"] = 0
                for domain in json_result["domains"]:
                    new_domain = Hostname.get_or_create(value=domain)
                    links.update(
                        new_domain.active_link_to(
                            observable, "recorded by", "ThreatCrowd"
                        )
                    )
                    result["domains recorded by email"] += 1

        if isinstance(observable, Hash):
            result["nb c2"] = 0

            if "md5" in json_result:
                new_hash = Hash.get_or_create(value=json_result["md5"])
                links.update(new_hash.active_link_to(observable, "md5", "ThreadCrowd"))

            if "sha1" in json_result:
                new_hash = Hash.get_or_create(value=json_result["sha1"])
                links.update(new_hash.active_link_to(observable, "sha1", "ThreadCrowd"))

            if "sha256" in json_result:
                new_hash = Hash.get_or_create(value=json_result["sha256"])
                links.update(
                    new_hash.active_link_to(observable, "sha256", "ThreadCrowd")
                )

            if "domains" in json_result and len(json_result["domains"]):
                for domain in json_result["domains"]:
                    new_domain = Hostname.get_or_create(value=domain)
                    links.update(
                        observable.active_link_to(new_domain, "c2", "ThreatCrowd")
                    )
                    result["nb c2"] += 1

            if "ips" in json_result and len(json_result["ips"]):
                for ip in json_result["ips"]:
                    new_ip = Ip.get_or_create(value=ip.strip())
                    links.update(observable.active_link_to(new_ip, "c2", "ThreatCrowd"))
                    result["nb c2"] += 1

        if "permalink" in json_result:
            result["permalink"] = json_result["permalink"]

        result["source"] = "threatcrowd_query"
        result["raw"] = json_string
        observable.add_context(result)
        return list(links)
Beispiel #9
0
    def _add_events_nodes(self, events, context, tags):
        log.debug('_add_events_nodes on {nb} events'.format(nb=len(events)))
        attach_unsupported = dict([
            (_, 0) for _ in ['UNSUPPORTED_TYPE', 'TOO_SMALL', None]
        ])
        event_nodes = list()
        for msg in events:
            create_t = datetime.strptime(msg['messageTime'],
                                         "%Y-%m-%dT%H:%M:%S.%fZ")
            # PPS unique value
            guid = Text.get_or_create(value='proofpoint://%s' % msg['GUID'],
                                      created=create_t,
                                      context=[context])
            log.debug('Event {msg}'.format(msg=msg['messageID']))
            message_contents = list()
            src_ip = Ip.get_or_create(value=msg['senderIP'],
                                      created=create_t,
                                      context=[context])
            src_ip.tag(['MTA'])
            guid.active_link_to([src_ip], "MTA src ip", self.name)
            # new event
            event_nodes.append(guid)
            #
            if self.config['import_email_metadata']:
                # email details
                # messageID
                message_id = Email.get_or_create(value=msg['messageID'],
                                                 created=create_t,
                                                 context=[context])
                guid.active_link_to([message_id], "seen in", self.name)
                # sender
                _s1 = Email.get_or_create(value=msg['sender'],
                                          created=create_t,
                                          context=[context])
                _s1.tag(['sender'])
                guid.active_link_to([_s1], "sender", self.name)
                if 'headerFrom' in msg:
                    # header From
                    _s2 = Email.get_or_create(value=msg['headerFrom'],
                                              created=create_t,
                                              context=[context])
                    _s2.tag(['sender'])
                    guid.active_link_to([_s2], "headerFrom", self.name)

            # FIXME is that a duplicate of attachment-malware ?
            # attachment events
            for attach in msg['messageParts']:
                if attach['sandboxStatus'] in ['THREAT']:
                    md5 = Hash.get_or_create(value=attach['md5'],
                                             created=create_t,
                                             context=[context])
                    md5.tag([t['name'] for t in tags])
                    fname = File.get_or_create(value=attach['filename'],
                                               created=create_t,
                                               context=[context])
                    fname.tag([t['name'] for t in tags])
                    # this should be a DUP from threat_nodes in analyse()
                    sha_threat = Hash.get_or_create(value=attach['sha256'],
                                                    created=create_t,
                                                    context=[context])
                    sha_threat.active_link_to([md5, fname], "relates",
                                              self.name)
                    sha_threat.tag([t['name'] for t in tags])
                    message_contents.append(sha_threat)
                    # link the 3 together
                elif attach['sandboxStatus'] in [
                        'UNSUPPORTED_TYPE', 'TOO_SMALL', None
                ]:
                    attach_unsupported[attach['sandboxStatus']] += 1
                    log.debug(pprint.pformat(attach))
                # add context to the hashes
                guid.active_link_to(message_contents, "delivers", self.name)
        _stats = ', '.join("%s: %d" % (k, v)
                           for k, v in attach_unsupported.items())
        log.warning('Ignored unsupported attachments: %s', _stats)
        for o in event_nodes:
            o.tag([t['name'] for t in tags])
        return event_nodes
Beispiel #10
0
    def analyze(observable, results):
        links = set()
        json_result = ThreatCrowdAPI.fetch(observable)
        json_string = json.dumps(json_result,
                                 sort_keys=True,
                                 indent=4,
                                 separators=(',', ': '))
        results.update(raw=json_string)
        result = {}
        if isinstance(observable, Hostname):

            if 'resolutions' in json_result:

                result['ip on this domains'] = 0

                for ip in json_result['resolutions']:
                    if ip['ip_address'].strip() != observable.value:
                        if ip['last_resolved'] != '0000-00-00':
                            last_resolved = datetime.datetime.strptime(
                                ip['last_resolved'], "%Y-%m-%d")
                            try:
                                new_ip = Ip.get_or_create(
                                    value=ip['ip_address'].strip())
                                links.update(
                                    new_ip.active_link_to(
                                        observable, 'IP', 'ThreatCrowd',
                                        last_resolved))
                                result['ip on this domains'] += 1
                            except ObservableValidationError:
                                logging.error(
                                    "An error occurred when trying to add subdomain {} to the database"
                                    .format(ip['ip_address']))

            if 'emails' in json_result:

                result['nb emails'] = 0

                for email in json_result['emails']:
                    try:
                        new_email = Email.get_or_create(value=email)
                        links.update(
                            new_email.active_link_to(observable, 'Used by',
                                                     'ThreatCrowd'))
                        result['nb emails'] += 1
                    except ObservableValidationError:
                        logging.error(
                            "An error occurred when trying to add email {} to the database"
                            .format(email))

            if 'subdomains' in json_result:

                result['nb subdomains'] = 0

                for subdomain in json_result['subdomains']:
                    try:
                        new_domain = Hostname.get_or_create(value=subdomain)
                        links.update(
                            observable.active_link_to(new_domain, 'subdomain',
                                                      'ThreatCrowd'))
                        result['nb subdomains'] += 1
                    except ObservableValidationError:
                        logging.error(
                            "An error occurred when trying to add subdomain {} to the database"
                            .format(subdomain))

        if isinstance(observable, Ip):

            if 'resolutions' in json_result:

                result['domains resolved'] = 0

                for domain in json_result['resolutions']:
                    if domain['domain'].strip() != observable.value:
                        try:
                            last_resolved = datetime.datetime.strptime(
                                domain['last_resolved'], "%Y-%m-%d")
                            new_domain = Hostname.get_or_create(
                                value=domain['domain'].strip())
                            links.update(
                                new_domain.active_link_to(
                                    observable, 'A Record', 'ThreatCrowd',
                                    last_resolved))
                            result['domains resolved'] += 1
                        except ObservableValidationError:
                            logging.error(
                                "An error occurred when trying to add domain {} to the database"
                                .format(domain['domain']))

            if 'hashes' in json_result and len(json_result['hashes']) > 0:
                result['malwares'] = 0
                for h in json_result['hashes']:
                    new_hash = Hash.get_or_create(value=h)
                    links.update(
                        new_hash.active_link_to(observable, 'hash',
                                                'ThreatCrowd'))
                    result['malwares'] += 1

        if isinstance(observable, Email):
            if 'domains' in json_result and len(json_result) > 0:
                result['domains recorded by email'] = 0
                for domain in json_result['domains']:
                    new_domain = Hostname.get_or_create(value=domain)
                    links.update(
                        new_domain.active_link_to(observable, 'recorded by',
                                                  'ThreatCrowd'))
                    result['domains recorded by email'] += 1

        if isinstance(observable, Hash):
            result['nb c2'] = 0

            if 'md5' in json_result:
                new_hash = Hash.get_or_create(value=json_result['md5'])
                links.update(
                    new_hash.active_link_to(observable, 'md5', 'ThreadCrowd'))

            if 'sha1' in json_result:
                new_hash = Hash.get_or_create(value=json_result['sha1'])
                links.update(
                    new_hash.active_link_to(observable, 'sha1', 'ThreadCrowd'))

            if 'sha256' in json_result:
                new_hash = Hash.get_or_create(value=json_result['sha256'])
                links.update(
                    new_hash.active_link_to(observable, 'sha256',
                                            'ThreadCrowd'))

            if 'domains' in json_result and len(json_result['domains']):
                for domain in json_result['domains']:
                    new_domain = Hostname.get_or_create(value=domain)
                    links.update(
                        observable.active_link_to(new_domain, 'c2',
                                                  'ThreatCrowd'))
                    result['nb c2'] += 1

            if 'ips' in json_result and len(json_result['ips']):
                for ip in json_result['ips']:
                    new_ip = Ip.get_or_create(value=ip.strip())
                    links.update(
                        observable.active_link_to(new_ip, 'c2', 'ThreatCrowd'))
                    result['nb c2'] += 1

        if 'permalink' in json_result:
            result['permalink'] = json_result['permalink']

        result['source'] = 'threatcrowd_query'
        result['raw'] = json_string
        observable.add_context(result)
        return list(links)
Beispiel #11
0
    def analyze(hostname, results):
        links = set()
        data = whois.whois(hostname.value)
        if not data["domain_name"]:
            return list(links)
        should_add_context = False

        for context in hostname.context:
            if context["source"] == "whois":
                break
        else:
            should_add_context = True
            context = {"source": "whois"}
            context["whois_server"] = data["whois_server"]
            if data["dnssec"]:
                context["dnssec"] = data["dnssec"]

            if isinstance(data["creation_date"], list):
                context["creation_date"] = sorted(data["creation_date"])[0]
            else:
                context["creation_date"] = data["creation_date"]

            if isinstance(data["updated_date"], list):
                context["updated_date"] = sorted(data["updated_date"],
                                                 reverse=True)[0]
            else:
                context["updated_date"] = data["updated_date"]

            if isinstance(data["expiration_date"], list):
                context["expiration_date"] = sorted(data["expiration_date"],
                                                    reverse=True)[0]
            else:
                context["expiration_date"] = data["expiration_date"]

            name_servers = data["name_servers"]

        if isinstance(name_servers, list):
            for ns in name_servers:
                ns_obs = Hostname.get_or_create(value=ns)
                links.update(
                    ns_obs.active_link_to(hostname, "NS", context["source"]))
        else:
            ns_obs = Hostname.get_or_create(value=name_servers)
            links.update(
                ns_obs.active_link_to(hostname, "NS", context["source"]))

        for email in data["emails"]:
            email_obs = Email.get_or_create(value=email)
            links.update(
                email_obs.active_link_to(hostname, "email registrar",
                                         context["source"]))
        if data["org"]:
            company_org = Company.get_or_create(name=data["org"])
            links.update(
                company_org.active_link_to(hostname, "Org", context["source"]))

        if data["registrar"]:
            company_registrar = Company.get_or_create(name=data["registrar"])
            links.update(
                company_registrar.active_link_to(hostname, "registrar",
                                                 context["source"]))
        if should_add_context:
            hostname.add_context(context)
        else:
            hostname.save()

        return list(links)
Beispiel #12
0
    def analyze(observable, results):
        links = set()
        json_result = ThreatCrowdAPI.fetch(observable)
        json_string = json.dumps(
            json_result, sort_keys=True, indent=4, separators=(',', ': '))
        results.update(raw=json_string)
        result = {}
        if isinstance(observable, Hostname):

            if 'resolutions' in json_result:

                result['ip on this domains'] = 0

                for ip in json_result['resolutions']:
                    if ip['ip_address'].strip() != observable.value:
                        if ip['last_resolved'] != '0000-00-00':
                            last_resolved = datetime.datetime.strptime(
                                ip['last_resolved'], "%Y-%m-%d")
                            try:
                                new_ip = Ip.get_or_create(
                                    value=ip['ip_address'].strip())
                                links.update(
                                    new_ip.active_link_to(
                                        observable, 'IP', 'ThreatCrowd',
                                        last_resolved))
                                result['ip on this domains'] += 1
                            except ObservableValidationError:
                                logging.error(
                                    "An error occurred when trying to add subdomain {} to the database".
                                    format(ip['ip_address']))

            if 'emails' in json_result:

                result['nb emails'] = 0

                for email in json_result['emails']:
                    try:
                        new_email = Email.get_or_create(value=email)
                        links.update(
                            new_email.active_link_to(
                                observable, 'Used by', 'ThreatCrowd'))
                        result['nb emails'] += 1
                    except ObservableValidationError:
                        logging.error(
                            "An error occurred when trying to add email {} to the database".
                            format(email))

            if 'subdomains' in json_result:

                result['nb subdomains'] = 0

                for subdomain in json_result['subdomains']:
                    try:
                        new_domain = Hostname.get_or_create(value=subdomain)
                        links.update(
                            observable.active_link_to(
                                new_domain, 'subdomain', 'ThreatCrowd'))
                        result['nb subdomains'] += 1
                    except ObservableValidationError:
                        logging.error(
                            "An error occurred when trying to add subdomain {} to the database".
                            format(subdomain))

        if isinstance(observable, Ip):

            if 'resolutions' in json_result:

                result['domains resolved'] = 0

                for domain in json_result['resolutions']:
                    if domain['domain'].strip() != observable.value:
                        try:
                            last_resolved = datetime.datetime.strptime(
                                domain['last_resolved'], "%Y-%m-%d")
                            new_domain = Hostname.get_or_create(
                                value=domain['domain'].strip())
                            links.update(
                                new_domain.active_link_to(
                                    observable, 'A Record', 'ThreatCrowd',
                                    last_resolved))
                            result['domains resolved'] += 1
                        except ObservableValidationError:
                            logging.error(
                                "An error occurred when trying to add domain {} to the database".
                                format(domain['domain']))

            if 'hashes' in json_result and len(json_result['hashes']) > 0:
                result['malwares'] = 0
                for h in json_result['hashes']:
                    new_hash = Hash.get_or_create(value=h)
                    links.update(
                        new_hash.active_link_to(
                            observable, 'hash', 'ThreatCrowd'))
                    result['malwares'] += 1

        if isinstance(observable, Email):
            if 'domains' in json_result and len(json_result) > 0:
                result['domains recorded by email'] = 0
                for domain in json_result['domains']:
                    new_domain = Hostname.get_or_create(value=domain)
                    links.update(
                        new_domain.active_link_to(
                            observable, 'recorded by', 'ThreatCrowd'))
                    result['domains recorded by email'] += 1

        if isinstance(observable, Hash):
            result['nb c2'] = 0

            if 'md5' in json_result:
                new_hash = Hash.get_or_create(value=json_result['md5'])
                links.update(
                    new_hash.active_link_to(observable, 'md5', 'ThreadCrowd'))

            if 'sha1' in json_result:
                new_hash = Hash.get_or_create(value=json_result['sha1'])
                links.update(
                    new_hash.active_link_to(observable, 'sha1', 'ThreadCrowd'))

            if 'sha256' in json_result:
                new_hash = Hash.get_or_create(value=json_result['sha256'])
                links.update(
                    new_hash.active_link_to(
                        observable, 'sha256', 'ThreadCrowd'))

            if 'domains' in json_result and len(json_result['domains']):
                for domain in json_result['domains']:
                    new_domain = Hostname.get_or_create(value=domain)
                    links.update(
                        observable.active_link_to(
                            new_domain, 'c2', 'ThreatCrowd'))
                    result['nb c2'] += 1

            if 'ips' in json_result and len(json_result['ips']):
                for ip in json_result['ips']:
                    new_ip = Ip.get_or_create(value=ip.strip())
                    links.update(
                        observable.active_link_to(new_ip, 'c2', 'ThreatCrowd'))
                    result['nb c2'] += 1

        if 'permalink' in json_result:
            result['permalink'] = json_result['permalink']

        result['source'] = 'threatcrowd_query'
        result['raw'] = json_string
        observable.add_context(result)
        return list(links)
Beispiel #13
0
    def _add_events_nodes(self, events, context, tags):
        log.debug('_add_events_nodes on {nb} events'.format(nb=len(events)))
        attach_unsupported = dict(
            [(_, 0) for _ in ['UNSUPPORTED_TYPE', 'TOO_SMALL', None]])
        event_nodes = list()
        for msg in events:
            create_t = datetime.strptime(
                msg['messageTime'], "%Y-%m-%dT%H:%M:%S.%fZ")
            # PPS unique value
            guid = Text.get_or_create(
                value='proofpoint://%s' % msg['GUID'],
                created=create_t,
                context=[context])
            log.debug('Event {msg}'.format(msg=msg['messageID']))
            message_contents = list()
            src_ip = Ip.get_or_create(
                value=msg['senderIP'], created=create_t, context=[context])
            src_ip.tag(['MTA'])
            guid.active_link_to([src_ip], "MTA src ip", self.name)
            # new event
            event_nodes.append(guid)
            #
            if self.config['import_email_metadata']:
                # email details
                # messageID
                message_id = Email.get_or_create(
                    value=msg['messageID'], created=create_t, context=[context])
                guid.active_link_to([message_id], "seen in", self.name)
                # sender
                _s1 = Email.get_or_create(
                    value=msg['sender'], created=create_t, context=[context])
                _s1.tag(['sender'])
                guid.active_link_to([_s1], "sender", self.name)
                if 'headerFrom' in msg:
                    # header From
                    _s2 = Email.get_or_create(
                        value=msg['headerFrom'],
                        created=create_t,
                        context=[context])
                    _s2.tag(['sender'])
                    guid.active_link_to([_s2], "headerFrom", self.name)

            # FIXME is that a duplicate of attachment-malware ?
            # attachment events
            for attach in msg['messageParts']:
                if attach['sandboxStatus'] in ['THREAT']:
                    md5 = Hash.get_or_create(
                        value=attach['md5'],
                        created=create_t,
                        context=[context])
                    md5.tag([t['name'] for t in tags])
                    fname = File.get_or_create(
                        value=attach['filename'],
                        created=create_t,
                        context=[context])
                    fname.tag([t['name'] for t in tags])
                    # this should be a DUP from threat_nodes in analyse()
                    sha_threat = Hash.get_or_create(
                        value=attach['sha256'],
                        created=create_t,
                        context=[context])
                    sha_threat.active_link_to([md5, fname], "relates",
                                              self.name)
                    sha_threat.tag([t['name'] for t in tags])
                    message_contents.append(sha_threat)
                    # link the 3 together
                elif attach['sandboxStatus'] in ['UNSUPPORTED_TYPE',
                                                 'TOO_SMALL', None]:
                    attach_unsupported[attach['sandboxStatus']] += 1
                    log.debug(pprint.pformat(attach))
                # add context to the hashes
                guid.active_link_to(message_contents, "delivers", self.name)
        _stats = ', '.join(
            "%s: %d" % (k, v) for k, v in attach_unsupported.items())
        log.warning('Ignored unsupported attachments: %s', _stats)
        for o in event_nodes:
            o.tag([t['name'] for t in tags])
        return event_nodes