Example #1
0
    def _process_data(json_result, observable):
        links = set()

        for key in ("undetected_communicating_file",
                    "detected_downloaded_file", "undetected_downloaded_file",
                    "detected_communicating_file"):
            for file_hash in json_result.get(key, []):
                new_hash = Hash.get_or_create(value=file_hash)
                new_hash.tag(observable.get_tags())
                links.update(
                    new_hash.active_link_to(observable, key, "malwares.com"))

        for host in json_result.get("hostname_history", {}).get("list", []):
            new_host = Hostname.get_or_create(value=host)
            new_host.tag(observable.get_tags())
            links.update(
                new_host.active_link_to(observable, "hostname",
                                        "malwares.com"))

        for key in ("detected_url", "undetected_url", "distribution_url"):
            for url in json_result.get(key, []):
                new_url = Url.get_or_create(value=url)
                new_url.tag(observable.get_tags())
                links.update(
                    new_url.active_link_to(observable, key, "malwares.com"))

        observable.add_context(json_result)
        return list(links)
    def analyze(self, line):
        if not line or line[0].startswith("#"):
            return

        first_seen, c2_ip, c2_port, family = tuple(line)

        tags = []
        tags.append(family.lower())
        tags.append("c2")
        tags.append("blocklist")

        context = {
            "first_seen": first_seen,
            "source": self.name
        }

        try:
            new_url = Url.get_or_create(value="http://{}:{}/".format(
                c2_ip, c2_port)
            )
            new_url.add_context(context)
            new_url.tag(tags)

        except ObservableValidationError as e:
            logging.error("Invalid line: {}\nLine: {}".format(e, line))
Example #3
0
    def analyze(self, item):
        observable_sample = item['title']
        context_sample = {}
        context_sample['description'] = "Atmos sample"
        context_sample['date_added'] = parser.parse(item['pubDate'])
        context_sample['source'] = self.name

        link_c2 = re.search("<a href[^>]+>(?P<url>[^<]+)",
                            item['description'].lower()).group("url")
        observable_c2 = link_c2
        context_c2 = {}
        context_c2['description'] = "Atmos c2"
        context_c2['date_added'] = parser.parse(item['pubDate'])
        context_c2['source'] = self.name

        try:
            sample = Hash.get_or_create(value=observable_sample)
            sample.add_context(context_sample)
            sample.add_source("feed")
            sample_tags = ['atmos', 'objectives']
            sample.tag(sample_tags)
        except ObservableValidationError as e:
            logging.error(e)
            return

        try:
            c2 = Url.get_or_create(value=observable_c2)
            c2.add_context(context_c2)
            c2.add_source("feed")
            c2_tags = ['c2', 'atmos']
            c2.tag(c2_tags)
            sample.active_link_to(c2, 'c2', self.name, clean_old=False)
        except ObservableValidationError as e:
            logging.error(e)
            return
Example #4
0
    def analyze(self, line):

        url_obs = False
        ip_obs = False
        family = line['Family']
        url = line['URL']
        ip = line['IP']
        first_seen = line['FirstSeen']
        family = family.lower()
        context = {'first_seen': first_seen, 'source': self.name}

        if url:
            try:
                url_obs = Url.get_or_create(value=url)
                url_obs.add_context(context)
                url_obs.add_source(self.name)
                url_obs.tag(["c2", family])
            except ObservableValidationError as e:
                logging.error(e)

        if ip:
            try:
                ip_obs = Ip.get_or_create(value=ip)
                ip_obs.add_context(context)
                ip_obs.tag(family.lower())
            except ObservableValidationError as e:
                logging.error(e)

        if url_obs and ip_obs:
            url_obs.active_link_to(ip_obs, 'ip', self.name)
Example #5
0
    def analyze(self, item):
        if not item:
            return

        item_date = parser.parse(item['pubDate'])
        max_age = yeti_config.get('limits', 'max_age')
        limit_date = pytz.UTC.localize(datetime.now()) - timedelta(days=max_age)

        if item_date < limit_date:
            return

        md5_search = re.search(r'md5:\t([a-fA-F\d]{32})<br />', item['description'])

        if not bool(md5_search):
            return

        context = {}
        tags = ['malware']

        if item['category'] != '':
            context['threat'] = item['category']
            signature = item['category']\
                .replace(' ', '_')\
                .replace('/', '_')\
                .replace(':', '_')\
                .replace('.', '-')\
                .replace('!', '-')

            if signature == 'clean_site':
                return

            tags.append(signature)

        context['date_added'] = item_date
        context['source'] = self.name
        context['reference'] = item['link']

        try:
            sample = Hash.get_or_create(value=md5_search.group(1))
            sample.add_context(context)
            sample.add_source("feed")
            sample.tag(tags)

        except ObservableValidationError as e:
            logging.error(e)
            return

        except Exception as e:
            print(e)

        try:
            url = Url.get_or_create(value=item['title'])
            url.add_context(context)
            url.add_source("feed")
            url.tag(tags)
            sample.active_link_to(url, 'drops', self.name)

        except ObservableValidationError as e:
            logging.error(e)
            return
Example #6
0
    def analyze(self, dict):
        url_string = re.search(r"URL: (?P<url>\S+),",
                               dict['description']).group('url')
        url_string = url_string.replace(
            'http://https://', 'https://')  # feed malformed, nasty hack
        context = {}
        date_string = re.search(r"\((?P<date>[0-9\-]+)\)",
                                dict['title']).group('date')
        context['date_added'] = datetime.strptime(date_string, "%Y-%m-%d")
        context['status'] = re.search(r"status: (?P<status>[^,]+)",
                                      dict['description']).group('status')
        context['guid'] = dict['guid']
        context['source'] = self.name
        try:
            context['md5'] = re.search(r"MD5 hash: (?P<md5>[a-f0-9]+)",
                                       dict['description']).group('md5')
        except AttributeError as e:
            pass

        try:
            n = Url.get_or_create(value=url_string)
            n.add_context(context)
            n.add_source("feed")
            n.tag(['zeus', 'delivery', 'banker', 'crimeware', 'malware'])
        except ObservableValidationError as e:
            logging.error(e)
Example #7
0
    def analyze(self, dict):
        observable_sample = dict['title']
        context_sample = {}
        context_sample['description'] = "Pony sample"
        context_sample['date_added'] = datetime.strptime(dict['pubDate'], "%d-%m-%Y")
        context_sample['source'] = self.name

        link_c2 = re.search("https?://[^ ]*", dict['description'].lower()).group()
        observable_c2 = link_c2
        context_c2 = {}
        context_c2['description'] = "Pony c2"
        context_c2['date_added'] = datetime.strptime(dict['pubDate'], "%d-%m-%Y")
        context_c2['source'] = self.name

        try:
            sample = Hash.get_or_create(value=observable_sample)
            sample.add_context(context_sample)
            sample.add_source("feed")
            sample_tags = ['pony', 'objectives']
            sample.tag(sample_tags)
        except ObservableValidationError as e:
            logging.error(e)
            return

        try:
            c2 = Url.get_or_create(value=observable_c2)
            c2.add_context(context_c2)
            c2.add_source("feed")
            c2_tags = ['c2', 'pony']
            c2.tag(c2_tags)
            sample.active_link_to(c2, 'c2', self.name, clean_old=False)
        except ObservableValidationError as e:
            logging.error(e)
            return
Example #8
0
    def analyze(self, data):
        if not data or data[0].startswith('phish_id'):
            return

        _, url, phish_detail_url, submission_time, verified, verification_time, online, target = tuple(
            data)

        tags = ['phishing']

        context = {
            'source': self.name,
            'phish_detail_url': phish_detail_url,
            'submission_time': submission_time,
            'verified': verified,
            'verification_time': verification_time,
            'online': online,
            'target': target
        }

        if url is not None and url != '':
            try:
                url = Url.get_or_create(value=url)
                url.add_context(context)
                url.add_source('feed')
                url.tag(tags)
            except ObservableValidationError as e:
                logging.error(e)
Example #9
0
    def analyze(self, item):
        url_string = re.search(r"URL: (?P<url>\S+),",
                               item['description']).group('url')

        context = {}
        date_string = re.search(r"\((?P<date>[0-9\-]+)\)",
                                item['title']).group('date')
        context['date_added'] = datetime.strptime(date_string, "%Y-%m-%d")
        context['status'] = re.search(r"status: (?P<status>[^,]+)",
                                      item['description']).group('status')
        context['version'] = int(
            re.search(r"version: (?P<version>[^,]+)",
                      item['description']).group('version'))
        context['guid'] = item['guid']
        context['source'] = self.name
        try:
            context['md5'] = re.search(r"MD5 hash: (?P<md5>[a-f0-9]+)",
                                       item['description']).group('md5')
        except AttributeError as e:
            pass

        try:
            n = Url.get_or_create(value=url_string)
            n.add_context(context)
            n.add_source("feed")
            n.tag(['zeus', 'c2', 'banker', 'crimeware', 'malware'])
        except ObservableValidationError as e:
            logging.error(e)
Example #10
0
    def analyze(self, item):
        if not item or item[0].startswith("#"):
            return

        family, threat, url, url_status, first_seen, first_active, last_active, last_update = item

        context = {
            "first_seen": first_seen,
            "first_active": first_active,
            "last_active": last_active,
            "last_update": last_update,
            "status": url_status,
            "source": self.name,
            "threat": threat
        }

        if url:
            try:
                url_obs = Url.get_or_create(value=url)
                url_obs.tag([family, 'malware'])
                url_obs.add_context(context)
                url_obs.add_source('feed')

            except ObservableValidationError as e:
                logging.error(e)
Example #11
0
    def _process_data(json_result, observable):
        links = set()

        for page in json_result:
            if not page.get("page"):
                continue

            # IP iocs has more data than the rest
            if not isinstance(observable, Ip) and page['page'].get('ip'):
                new_ip = Ip.get_or_create(value=page['page']['ip'])
                links.update(
                    new_ip.active_link_to(observable, 'ip', 'UrlScanIo Query'))

            if not isinstance(observable,
                              Hostname) and page['page'].get('domain'):
                new_host = Hostname.get_or_create(value=page['page']['domain'])
                links.update(
                    new_host.active_link_to(observable, 'hostname',
                                            'UrlScanIo Query'))

            if not isinstance(observable, Url) and page['page'].get('url'):
                new_url = Url.get_or_create(value=page['page']['url'])
                links.update(
                    new_url.active_link_to(observable, 'url',
                                           'UrlScanIo Query'))

            links.update(UrlScanIoApi._process_asn_data(page, observable))
Example #12
0
    def analyze(self, line):
        if line[0] == 'Number':
            return

        # split the entry into observables
        Number, Status, CC, Host, Port, Protocol, ASN, Last_Updated, First_Seen, Last_Seen, First_Active, Last_Active, SBL, Abuse_Contact, Details = line

        url = "{}://{}".format(Protocol, Host)
        context = {}
        context['status'] = Status
        context['port'] = Port
        context['cc'] = CC
        context['status'] = Status
        context['date_added'] = datetime.strptime(First_Seen, "%Y-%m-%d %H:%M:%S")
        context['last_seen'] = datetime.strptime(Last_Seen, "%Y-%m-%d %H:%M:%S") if Last_Seen else datetime.utcnow()
        context['sbl'] = SBL
        context['abuse_contact'] = Abuse_Contact
        context['description'] = Details if Details else "N/A"
        context['source'] = self.name
        try:
            url = Url.get_or_create(value=url)
            url.add_context(context)
            url.add_source("feed")
            url.tag(['asprox', 'c2', 'scanner'])
        except ObservableValidationError as e:
            logging.error(e)
Example #13
0
    def analyze(self, line):

        id_feed = line["id"]
        first_seen = line["dateadded"]
        url = line["url"]
        url_status = line["url_status"]
        threat = line["threat"]
        tags = line["tags"]
        urlhaus_link = line["urlhaus_link"]
        source = line["reporter"]  # pylint: disable=line-too-long

        context = {
            "id_urlhaus": id_feed,
            "status": url_status,
            "source": self.name,
            "report": urlhaus_link,
            "threat": threat,
            "reporter": source,
            "first_seen": first_seen,
        }

        if url:
            try:
                url_obs = Url.get_or_create(value=url)
                url_obs.tag(tags.split(","))
                url_obs.add_context(context)
                url_obs.add_source(self.name)
            except ObservableValidationError as e:
                logging.error(e)
Example #14
0
    def analyze(self, line):

        url_obs = False
        ip_obs = False
        family = line["Family"]
        url = line["URL"]
        ip = line["IP"]
        first_seen = line["FirstSeen"]
        family = family.lower()
        context = {"first_seen": first_seen, "source": self.name}

        if url:
            try:
                url_obs = Url.get_or_create(value=url)
                url_obs.add_context(context)
                url_obs.add_source(self.name)
                url_obs.tag(["c2", family])
            except ObservableValidationError as e:
                logging.error(e)

        if ip:
            try:
                ip_obs = Ip.get_or_create(value=ip)
                ip_obs.add_context(context)
                ip_obs.tag(family.lower())
            except ObservableValidationError as e:
                logging.error(e)

        if url_obs and ip_obs:
            url_obs.active_link_to(ip_obs, "ip", self.name)
Example #15
0
    def analyze(self, line):

        url_obs = False
        url = line["url"]
        ip = line["ip"]
        family = line["type"]
        context = {}
        context["date_added"] = line["date"]
        context["source"] = self.name
        tags = []
        tags.append(family.lower())

        try:
            if url:
                url_obs = Url.get_or_create(value=url)
                url_obs.add_context(context)
                url_obs.add_source(self.name)
                url_obs.tag(tags)

        except ObservableValidationError as e:
            logging.error(e)

        try:
            if ip:
                ip_obs = Ip.get_or_create(value=ip)
                ip_obs.add_context(context)
                ip_obs.add_source(self.name)
                ip_obs.tag(tags)
                if url_obs:
                    ip_obs.active_link_to(url_obs, "url", self.name, clean_old=False)
        except ObservableValidationError as e:
            logging.error(e)
Example #16
0
    def analyze(self, data):
        if not data or data[0].startswith('phish_id'):
            return

        _, url, phish_detail_url, submission_time, verified, verification_time, online, target = data

        item_date = dateutil.parser.parse(submission_time)
        max_age = yeti_config.get('limits', 'max_age')
        limit_date = pytz.UTC.localize(
            datetime.now()) - timedelta(days=max_age)

        if item_date < limit_date:
            return

        tags = ['phishing']

        context = {
            'source': self.name,
            'phish_detail_url': phish_detail_url,
            'submission_time': submission_time,
            'verified': verified,
            'verification_time': verification_time,
            'online': online,
            'target': target
        }

        if url is not None and url != '':
            try:
                url = Url.get_or_create(value=url)
                url.add_context(context)
                url.add_source('feed')
                url.tag(tags)

            except ObservableValidationError as e:
                logging.error(e)
Example #17
0
    def analyze(self, item, pub_date):  # pylint: disable=arguments-differ
        observable_sample = item["title"]
        context_sample = {}
        context_sample["description"] = "Pony sample"
        context_sample["date_added"] = pub_date
        context_sample["source"] = self.name

        link_c2 = re.search("https?://[^ ]*", item["description"].lower()).group()
        observable_c2 = link_c2
        context_c2 = {}
        context_c2["description"] = "Pony c2"
        context_c2["date_added"] = pub_date
        context_c2["source"] = self.name

        try:
            sample = Hash.get_or_create(value=observable_sample)
            sample.add_context(context_sample)
            sample.add_source(self.name)
            sample_tags = ["pony", "objectives"]
            sample.tag(sample_tags)
        except ObservableValidationError as e:
            logging.error(e)
            return

        try:
            c2 = Url.get_or_create(value=observable_c2)
            c2.add_context(context_c2)
            c2.add_source(self.name)
            c2_tags = ["c2", "pony"]
            c2.tag(c2_tags)
            sample.active_link_to(c2, "c2", self.name, clean_old=False)
        except ObservableValidationError as e:
            logging.error(e)
            return
Example #18
0
    def analyze(self, line, first_seen):

        context = {}
        context['date_added'] = first_seen
        context['source'] = self.name

        _, family, url, ip, first_seen, _ = line

        if not url.startswith(('http://', 'https://')):
            url = "http://" + url

        tags = []
        tags.append(family.lower())
        tags.append("rat")

        try:
            if url:
                url = Url.get_or_create(value=url)
                url.add_context(context)
                url.add_source(self.name)
                url.tag(tags)

        except ObservableValidationError as e:
            logging.error(e)

        try:
            if ip:
                ip = Ip.get_or_create(value=ip)
                ip.add_context(context)
                ip.add_source(self.name)
                ip.tag(tags)

        except ObservableValidationError as e:
            logging.error(e)
Example #19
0
    def analyze(self, line, first_seen):

        url_obs = False

        _, family, url, ip, first_seen, _ = line
        context = {}
        context['date_added'] = first_seen
        context['source'] = self.name

        tags = []
        tags.append(family.lower())

        try:
            if url:
                url_obs = Url.get_or_create(value=url)
                url_obs.add_context(context)
                url_obs.add_source(self.name)
                url_obs.tag(tags)

        except ObservableValidationError as e:
            logging.error(e)

        try:
            if ip:
                ip_obs = Ip.get_or_create(value=ip)
                ip_obs.add_context(context)
                ip_obs.add_source(self.name)
                ip_obs.tag(tags)
                if url_obs:
                    ip_obs.active_link_to(
                        url_obs, "url", self.name, clean_old=False)
        except ObservableValidationError as e:
            logging.error(e)
Example #20
0
    def analyze(self, line):
        if line[0] == 'Number':
            return

        # split the entry into observables
        Number, Status, CC, Host, Port, Protocol, ASN, Last_Updated, First_Seen, Last_Seen, First_Active, Last_Active, SBL, Abuse_Contact, Details = line

        url = "{}://{}".format(Protocol, Host)
        context = {}
        context['status'] = Status
        context['port'] = Port
        context['cc'] = CC
        context['status'] = Status
        context['date_added'] = datetime.strptime(First_Seen, "%Y-%m-%d %H:%M:%S")
        context['last_seen'] = datetime.strptime(Last_Seen, "%Y-%m-%d %H:%M:%S") if Last_Seen else datetime.utcnow()
        context['sbl'] = SBL
        context['abuse_contact'] = Abuse_Contact
        context['description'] = Details if Details else "N/A"
        context['source'] = self.name
        try:
            url = Url.get_or_create(value=url)
            url.add_context(context)
            url.add_source("feed")
            url.tag(['asprox', 'c2', 'scanner'])
        except ObservableValidationError as e:
            logging.error(e)
Example #21
0
    def analyze(self, line):

        id_feed = line['id']
        first_seen = line['dateadded']
        url = line['url']
        url_status = line['url_status']
        threat = line['threat']
        tags = line['tags']
        urlhaus_link = line['urlhaus_link']
        source = line['reporter']  # pylint: disable=line-too-long

        context = {
            "id_urlhaus": id_feed,
            "status": url_status,
            "source": self.name,
            "report": urlhaus_link,
            "threat": threat,
            "reporter": source,
            'first_seen': first_seen
        }

        if url:
            try:
                url_obs = Url.get_or_create(value=url)
                url_obs.tag(tags.split(','))
                url_obs.add_context(context)
                url_obs.add_source(self.name)
            except ObservableValidationError as e:
                logging.error(e)
    def analyze(self, dict):
        observable_sample = dict['title']
        context_sample = {}
        context_sample['description'] = "Atmos sample"
        context_sample['date_added'] = datetime.strptime(dict['pubDate'], "%d-%m-%Y")
        context_sample['source'] = self.name

        link_c2 = re.search("<a href[^>]+>(?P<url>[^<]+)", dict['description'].lower()).group("url")
        observable_c2 = link_c2
        context_c2 = {}
        context_c2['description'] = "Atmos c2"
        context_c2['date_added'] = datetime.strptime(dict['pubDate'], "%d-%m-%Y")
        context_c2['source'] = self.name

        try:
            sample = Hash.get_or_create(value=observable_sample)
            sample.add_context(context_sample)
            sample.add_source("feed")
            sample_tags = ['atmos', 'objectives']
            sample.tag(sample_tags)
        except ObservableValidationError as e:
            logging.error(e)
            return

        try:
            c2 = Url.get_or_create(value=observable_c2)
            c2.add_context(context_c2)
            c2.add_source("feed")
            c2_tags = ['c2', 'atmos']
            c2.tag(c2_tags)
            sample.active_link_to(c2, 'c2', self.name, clean_old=False)
        except ObservableValidationError as e:
            logging.error(e)
            return
Example #23
0
    def analyze(self, line):

        first_seen = line['Firstseen']
        dst_ip = line['DstIP']
        ip_obs = False
        tags = ["potentially_malicious_infrastructure", "c2"]
        port = line['DstPort']
        context = dict(source=self.name)
        context["first_seen"] = first_seen

        try:
            ip_obs = Ip.get_or_create(value=dst_ip)
            ip_obs.add_source(self.name)
            ip_obs.tag(tags)
            ip_obs.add_context(context)
        except ObservableValidationError as e:
            logging.error(e)
            return False

        try:
            _url = "https://{dst_ip}:{port}/".format(dst_ip=dst_ip, port=port)
            url = Url.get_or_create(value=_url)
            url.add_source(self.name)
            url.tag(tags)
            url.add_context(context)
            if ip_obs:
                url.active_link_to(ip_obs, 'ip', self.name)
        except ObservableValidationError as e:
            logging.error(e)
            return False
Example #24
0
    def analyze(self, dict):
        observable_sample = dict["title"]
        context_sample = {}
        context_sample["description"] = "Pony sample"
        context_sample["date_added"] = datetime.strptime(dict["pubDate"], "%d-%m-%Y")
        context_sample["source"] = self.name

        link_c2 = re.search("https?://[^ ]*", dict["description"].lower()).group()
        observable_c2 = link_c2
        context_c2 = {}
        context_c2["description"] = "Pony c2"
        context_c2["date_added"] = datetime.strptime(dict["pubDate"], "%d-%m-%Y")
        context_c2["source"] = self.name

        try:
            sample = Hash.get_or_create(value=observable_sample)
            sample.add_context(context_sample)
            sample.add_source("feed")
            sample_tags = ["pony", "objectives"]
            sample.tag(sample_tags)
        except ObservableValidationError as e:
            logging.error(e)
            return

        try:
            c2 = Url.get_or_create(value=observable_c2)
            c2.add_context(context_c2)
            c2.add_source("feed")
            c2_tags = ["c2", "pony"]
            c2.tag(c2_tags)
            sample.active_link_to(c2, "c2", self.name, clean_old=False)
        except ObservableValidationError as e:
            logging.error(e)
            return
Example #25
0
    def analyze(self, item):
        url_string = re.search(r"URL: (?P<url>\S+),",
                               item['description']).group('url')

        context = {}
        date_string = re.search(r"\((?P<date>[0-9\-]+)\)",
                                item['title']).group('date')
        context['date_added'] = datetime.strptime(date_string, "%Y-%m-%d")
        context['status'] = re.search(
            r"status: (?P<status>[^,]+)", item['description']).group('status')
        context['version'] = int(
            re.search(r"version: (?P<version>[^,]+)",
                      item['description']).group('version'))
        context['guid'] = item['guid']
        context['source'] = self.name
        try:
            context['md5'] = re.search(
                r"MD5 hash: (?P<md5>[a-f0-9]+)",
                item['description']).group('md5')
        except AttributeError as e:
            pass

        try:
            n = Url.get_or_create(value=url_string)
            n.add_context(context)
            n.add_source("feed")
            n.tag(['zeus', 'c2', 'banker', 'crimeware', 'malware'])
        except ObservableValidationError as e:
            logging.error(e)
Example #26
0
    def analyze(self, data):
        if not data or data[0].startswith('phish_id'):
            return

        _, url, phish_detail_url, submission_time, verified, verification_time, online, target = tuple(
            data)

        tags = ['phishing']

        context = {
            'source': self.name,
            'phish_detail_url': phish_detail_url,
            'submission_time': submission_time,
            'verified': verified,
            'verification_time': verification_time,
            'online': online,
            'target': target
        }

        if url is not None and url != '':
            try:    
                url = Url.get_or_create(value=url)
                url.add_context(context)
                url.add_source('feed')
                url.tag(tags)
            except ObservableValidationError as e:
                logging.error(e)
Example #27
0
    def analyze(self, item):

        if not item or item[0].startswith("#"):
            return

        id_feed, dateadded, url, url_status, threat, tags, urlhaus_link, reporter = item

        context = {
            "id_urlhaus": id_feed,
            "first_seen": dateadded,
            "status": url_status,
            "source": self.name,
            "urlhaus_link": urlhaus_link,
            "reporter": reporter,
            "threat": threat
        }

        if url:
            try:
                url_obs = Url.get_or_create(value=url)
                url_obs.tag(tags.split(','))
                url_obs.add_context(context)
                url_obs.add_source('feed')
            except ObservableValidationError as e:
                logging.error(e)
Example #28
0
    def analyze(observable, results):
        links = set()
        if isinstance(observable, Hostname):
            params = {"q": observable.value, "rt": 3}
            json_result = ThreatMinerApi.fetch(observable, params,
                                               "domain.php")

            _results, result = aux_checker(json_result)

            for r in _results:
                try:
                    o_url = Url.get_or_create(value=r.get("uri"))
                    o_url.tag(observable.get_tags())
                    links.update(
                        observable.link_to(
                            o_url,
                            description="related url",
                            source="ThreatMiner",
                            last_seen=r["last_seen"],
                        ))
                except ObservableValidationError as e:
                    logging.error("Caught an exception: {}".format(e))

            observable.add_context(result)

        elif isinstance(observable, Ip):
            params = {"q": observable.value, "rt": 3}
            json_result = ThreatMinerApi.fetch(observable, params, "host.php")

            _results, result = aux_checker(json_result)

            for r in _results:
                try:
                    o_url = Url.get_or_create(value=r.get("uri"))
                    o_url.tag(observable.get_tags())
                    links.update(
                        observable.link_to(
                            o_url,
                            description="related url",
                            source="ThreatMiner",
                            last_seen=r["last_seen"],
                        ))
                except ObservableValidationError as e:
                    logging.error("Caught an exception: {}".format(e))

            observable.add_context(result)
        return list(links)
Example #29
0
    def analyze(self, line):

        md5_obs = False
        sha256_obs = False
        url_obs = False
        malware_file = False

        context = {'source': self.name}

        first_seen, url, filetype, md5, sha256, signature = line

        if url:
            try:
                url_obs = Url.get_or_create(value=url)
                if signature != 'None':
                    url_obs.tag(signature)
                url_obs.add_context(context)
                url_obs.add_source(self.name)
            except ObservableValidationError as e:
                logging.error(e)

        if sha256:
            try:
                malware_file = File.get_or_create(
                    value='FILE:{}'.format(sha256))

                malware_file.add_context(context)
                malware_file.tag(filetype)

                sha256_obs = Hash.get_or_create(value=sha256)
                sha256_obs.tag(filetype)
                sha256_obs.add_context(context)
                if signature != 'None':
                    sha256_obs.tag(signature)
            except ObservableValidationError as e:
                logging.error(e)

        if md5:
            try:
                md5_obs = Hash.get_or_create(value=md5)
                md5_obs.add_context(context)
                md5_obs.tag(filetype)

                if signature != 'None':
                    md5_obs.tag(signature)
            except ObservableValidationError as e:
                logging.error(e)

        if malware_file:
            if signature != 'None':
                malware_file.tag(signature)

            if md5_obs:
                malware_file.active_link_to(md5_obs, 'md5', self.name)
            if sha256_obs:
                malware_file.active_link_to(sha256_obs, 'sha256', self.name)

            if url_obs:
                url_obs.active_link_to(malware_file, 'drops', self.name)
Example #30
0
    def analyze(observable, results):
        links = set()
        context = {
            "source": "malwares.com",
        }
        params = {
            "api_key": results.settings["malwares_api_key"],
            "hostname": observable.value,
        }

        json_result = MalwaresApi.fetch(observable, params, "hostname/info")
        if json_result:
            context["raw"] = json_result
            for key in (
                    "undetected_communicating_file",
                    "detected_downloaded_file",
                    "undetected_downloaded_file",
                    "detected_communicating_file",
            ):
                item = json_result[key]

                for f in item["list"]:
                    new_hash = Hash.get_or_create(value=f["sha256"])

                    links.update(
                        new_hash.active_link_to(observable, key,
                                                "malwares.com"))
                context[key] = item["total"]

            for host in json_result.get("hostname_history",
                                        {}).get("list", []):
                new_host = Hostname.get_or_create(value=host)
                new_host.tag(observable.get_tags())
                links.update(
                    new_host.active_link_to(observable, "hostname",
                                            "malwares.com"))

            for key in ("detected_url", "undetected_url"):
                item = json_result[key]

                for i in item["list"]:
                    try:
                        new_url = Url.get_or_create(value=i["url"])
                        new_url.tag(observable.get_tags())
                        links.update(
                            new_url.active_link_to(observable, key,
                                                   "malwares.com"))
                    except ObservableValidationError:
                        logging.error("Url is not valid %s" % i["url"])
                context[key] = item["total"]
            ip_history = json_result["ip_history"]

            for i in ip_history["list"]:
                ip = Ip.get_or_create(value=i["ip"])
                links.update(
                    ip.active_link_to(observable, "ip_story", "malwares.com"))
            context["ip_story"] = ip_history["total"]
        return links
Example #31
0
    def analyze(self, url):

        try:
            url_data = Url.get_or_create(value=url)
            url_data.normalize()
            url_data.tags(["payload_delivery"])
            url_data.add_source(self.name)
        except ObservableValidationError as e:
            logging.error(e)
Example #32
0
    def analyze(observable, results):
        links = set()
        context = {
            "source": "malwares.com",
        }
        params = {
            "api_key": results.settings["malwares_api_key"],
            "ip": observable.value,
        }

        json_result = MalwaresApi.fetch(observable, params, "ip/info")
        for key in (
                "undetected_communicating_file",
                "detected_downloaded_file",
                "undetected_downloaded_file",
                "detected_communicating_file",
        ):
            for item in json_result[key]["list"]:
                h = Hash.get_or_create(value=item["sha256"])
                links.update(
                    h.active_link_to(observable, key, context["source"]))
                h.add_context({
                    "source": context["source"],
                    "firs_seen": item["date"]
                })
            context[key] = json_result[key]["total"]

        for key in ("detected_url", "undetected_url"):
            for item in json_result[key]["list"]:
                url = Url.get_or_create(value=item["url"])
                links.update(
                    url.active_link_to(observable, key, context["source"]))
                url.add_context({
                    "source": context["source"],
                    "firs_seen": item["date"]
                })

            context[key] = json_result[key]["total"]
        hostname_history = json_result["hostname_history"]

        for item in hostname_history["list"]:
            try:
                hostname = Hostname.get_or_create(value=item["hostname"])
                links.update(
                    hostname.active_link_to(observable, "hostname_history",
                                            context["source"]))
                hostname.add_context({
                    "source": context["source"],
                    "firs_seen": item["date"]
                })
            except ObservableValidationError:
                logging.error("%s is not a hostname valid" % item["hostname"])

        context["hostname_history"] = hostname_history["total"]

        observable.add_context(context)
        return links
Example #33
0
 def _make_threat_nodes(threat, context, tags):
     # extract Url and Hash info
     threats = dict()
     if threat['threatStatus'] != 'active':
         # FIXME, clear out false positive ?
         log.warning("threatStatus %s for threat %s",
                     threat['threatStatus'], threat['threatID'])
         log.debug(pprint.pformat(threat))
         return None
     log.debug('_make_threat_nodes for threat %s', threat['threatID'])
     # threattype, classification
     # url, phish: url leads to phishing page (threat is url)
     # url, malware: url leads to malware download (threat is url, threatid is maybe sha256)
     # attachment, malware: attachement is malware (threat is sha256)
     # spam, url
     if threat['threatType'] == 'url':
         if threat['classification'] == 'phish':
             pass  # just keep the url
         elif threat['classification'] == 'malware':
             # get url and hash
             threats['attachment'] = threat
         elif threat['classification'] == 'spam':
             log.info('URL threat - ignore classification %s',
                      threat['classification'])
         else:
             log.error('Type: url, Unsupported classification %s',
                       threat['classification'])
             log.debug(pprint.pformat(threat))
             return None
         threats['url'] = threat
     elif threat['threatType'] == 'attachment':
         if threat['classification'] == 'malware':
             threats['attachment'] = threat
         else:
             log.error('Type: attachment, Unsupported classification %s',
                       threat['classification'])
             log.debug(pprint.pformat(threat))
             return None
     else:
         log.error('Unsupported threatType %s classification %s',
                   threat['threatType'], threat['classification'])
         log.debug(pprint.pformat(threat))
         return None
     # FIXME check if they exist already.
     # if they do, do not parse the threat a second time ?
     threat_nodes = []
     if 'url' in threats:
         threat_nodes.append(
             Url.get_or_create(value=threats['url']['threat'],
                               context=[context]))
     if 'attachment' in threats:
         threat_nodes.append(
             Hash.get_or_create(value=threats['attachment']['threatID'],
                                context=[context]))
     for o in threat_nodes:
         o.tag([t['name'] for t in tags])
     return threat_nodes
Example #34
0
    def analyze(self, url):
        context = {"source": self.name}

        try:
            url = Url.get_or_create(value=url)
            url.add_context(context)
            url.add_source(self.name)
            url.tag(["phishing"])
        except ObservableValidationError as e:
            logging.error(e)
Example #35
0
    def analyze(self, url):
        context = {'source': self.name}

        try:
            url = Url.get_or_create(value=url)
            url.add_context(context)
            url.add_source('feed')
            url.tag(['phishing'])
        except ObservableValidationError as e:
            logging.error(e)
Example #36
0
 def analyze(self, data):
     if data.startswith('http'):
         tags = ['malware']
         context = {'source': self.name}
         try:
             url = Url.get_or_create(value=data.rstrip())
             url.add_context(context)
             url.add_source('feed')
             url.tag(tags)
         except ObservableValidationError as e:
             logging.error(e)
Example #37
0
 def analyze(self, data):
     if data.startswith('http'):
         tags = ['malware']
         context = {'source': self.name}
         try:
             url = Url.get_or_create(value=data.rstrip())
             url.add_context(context)
             url.add_source(self.name)
             url.tag(tags)
         except ObservableValidationError as e:
             logging.error(e)
Example #38
0
 def analyze(self, url):
     context = {
         'source': self.name
     }
     
     try:
         if len(url) > 1023:
             logging.info('URL is too long for mongo db. url=%s' % str(url))
         else:
             url = Url.get_or_create(value=url)
             url.add_context(context)
             url.add_source('feed')
             url.tag(['phishing'])
     except ObservableValidationError as e:
         logging.error(e)
Example #39
0
    def analyze(self, line):
        if not line or line[0].startswith("#"):
            return

        try:
            ip, domain, family, md5, link, date = tuple(map(strip, line))
            context = {
                "first_seen": date,
                "family": family,
                "report": link,
                "source": self.name
            }

            c2 = None
            sample = None

            try:
                sample = Hash.get_or_create(value=md5)
                sample.add_context(context)
                sample.tag(family.lower())
            except ObservableValidationError as e:
                logging.error("Invalid line: {}\nLine: {}".format(e, line))

            try:
                if domain:
                    if '/' in domain:
                        c2 = Url.get_or_create(value=domain)
                    else:
                        c2 = Hostname.get_or_create(value=domain)
                elif ip:
                    c2 = Ip.get_or_create(value=ip)
                else:
                    return

                c2.add_context(context)
                c2.tag(['c2', family.lower()])

            except ObservableValidationError as e:
                logging.error("Invalid line: {}\nLine: {}".format(e, line))

            if c2 and sample:
                sample.active_link_to(c2, 'c2', self.name, clean_old=False)

        except ValueError:
            logging.error("Error unpacking line: {}".format(line))
Example #40
0
    def analyze(self, dict):
        # Create the new URL and store it in the DB
        url = re.search(r"Host: (?P<url>[^,]+),", dict['description']).group('url')

        context = {}
        context['source'] = self.name
        context['description'] = re.search(r"Description: (?P<description>.*)$", dict['description']).group('description')
        context['guid'] = dict['guid']
        date_string = re.search(r"\((?P<date>.*)\)", dict['title']).group('date')
        context['date_added'] = datetime.strptime(date_string, "%Y/%m/%d_%H:%M")

        try:
            url = Url.get_or_create(value=url)
            url.add_context(context)
            url.add_source("feed")
            url.tag(['malware', 'crimeware'])
        except ObservableValidationError as e:
            logging.error(e)
Example #41
0
    def analyze(self, data):
        if data.startswith('http'):
            if len(data) > 1023:
                logging.info('URL is too long for mongo db. url=%s' % str(data))
            else:
                tags = ['malware']

                context = {
                    'source': self.name
                }

                try:
                    url = Url.get_or_create(value=data)
                    url.add_context(context)
                    url.add_source('feed')
                    url.tag(tags)
                except ObservableValidationError as e:
                    logging.error(e)
    def analyze(self, line):

        if not line or line[0].startswith("#"):
            return

        date, _type, family, hostname, url, status, registrar, ips, asns, countries = tuple(
            line)

        tags = []
        tags += TYPE_DICT[_type]
        tags.append(family.lower())

        context = {
            "first_seen": date,
            "status": status,
            "registrar": registrar,
            "countries": countries.split("|"),
            "asns": asns.split("|"),
            "source": self.name
        }

        try:
            url = Url.get_or_create(value=url.rstrip())
            url.add_context(context)
            url.tag(tags)

            hostname = Observable.add_text(hostname)
            hostname.tag(tags + ['blocklist'])

            for ip in ips.split("|"):
                if ip != hostname and ip is not None and ip != '':
                    try:
                        i = Ip.get_or_create(value=ip)
                        i.active_link_to(
                            hostname,
                            "First seen IP",
                            self.name,
                            clean_old=False)
                    except ObservableValidationError as e:
                        logging.error("Invalid Observable: {}".format(e))

        except ObservableValidationError as e:
            logging.error("Invalid line: {}\nLine: {}".format(e, line))
Example #43
0
    def analyze(self, dict):
        g = re.match(r'^URL: (?P<url>.+), IP Address: (?P<ip>[\d.]+), Country: (?P<country>[A-Z]{2}), ASN: (?P<asn>\d+), MD5: (?P<md5>[a-f0-9]+)$', dict['description'])
        if g:
            context = g.groupdict()
            context['link'] = dict['link']
            context['source'] = self.name

            try:
                url_string = context.pop('url')
                context['description'] = dict['description'].encode('UTF-8')
                url = Url.get_or_create(value=url_string)
                url.add_context(context)
                url.add_source("feed")
                url.tag(['malware', 'delivery'])
            except UnicodeError:
                sys.stderr.write('Unicode error: %s' % dict['description'])
            except ObservableValidationError as e:
                logging.error(e)
            except Exception as e:
                logging.error("UNKNOWN EXCEPTION: {}".format(e))
Example #44
0
    def analyze(self, dict):
        g = re.match(
            r"^URL: (?P<url>.+), IP Address: (?P<ip>[\d.]+), Country: (?P<country>[A-Z]{2}), ASN: (?P<asn>\d+), MD5: (?P<md5>[a-f0-9]+)$",
            dict["description"],
        )
        if g:
            context = g.groupdict()
            context["link"] = dict["link"]
            context["source"] = self.name

            try:
                url_string = context.pop("url")
                context["description"] = dict["description"].encode("UTF-8")
                url = Url.get_or_create(value=url_string)
                url.add_context(context)
                url.add_source("feed")
                url.tag(["malware", "delivery"])
            except UnicodeError:
                sys.stderr.write("Unicode error: %s" % dict["description"])
            except ObservableValidationError as e:
                logging.error(e)
Example #45
0
    def analyze(self, dict):
        url_string = re.search(r"URL: (?P<url>\S+),", dict['description']).group('url')
        url_string = url_string.replace('http://https://', 'https://')  # feed malformed, nasty hack
        context = {}
        date_string = re.search(r"\((?P<date>[0-9\-]+)\)", dict['title']).group('date')
        context['date_added'] = datetime.strptime(date_string, "%Y-%m-%d")
        context['status'] = re.search(r"status: (?P<status>[^,]+)", dict['description']).group('status')
        context['guid'] = dict['guid']
        context['source'] = self.name
        try:
            context['md5'] = re.search(r"MD5 hash: (?P<md5>[a-f0-9]+)", dict['description']).group('md5')
        except AttributeError as e:
            pass

        try:
            n = Url.get_or_create(value=url_string)
            n.add_context(context)
            n.add_source("feed")
            n.tag(['zeus', 'delivery', 'banker', 'crimeware', 'malware'])
        except ObservableValidationError as e:
            logging.error(e)
    def analyze(self, dict):
        url_string = re.search(r"URL: (?P<url>\S+),", dict["description"]).group("url")

        context = {}
        date_string = re.search(r"\((?P<date>[0-9\-]+)\)", dict["title"]).group("date")
        context["date_added"] = datetime.strptime(date_string, "%Y-%m-%d")
        context["status"] = re.search(r"status: (?P<status>[^,]+)", dict["description"]).group("status")
        context["guid"] = dict["guid"]
        context["source"] = self.name
        try:
            context["md5"] = re.search(r"MD5 hash: (?P<md5>[a-f0-9]+)", dict["description"]).group("md5")
        except AttributeError as e:
            pass

        try:
            n = Url.get_or_create(value=url_string)
            n.add_context(context)
            n.add_source("feed")
            n.tag(["zeus", "objective", "banker", "crimeware", "malware"])
        except ObservableValidationError as e:
            logging.error(e)
Example #47
0
    def analyze(observable, results):
        links = set()
        json_result = MalshareAPI.fetch(
            observable, results.settings['malshare_api_key'])
        json_string = json.dumps(
            json_result, sort_keys=True, indent=4, separators=(',', ': '))
        results.update(raw=json_string)
        result = {'raw': json_string}

        if 'SOURCES' in json_result:
            for source in json_result['SOURCES']:
                new_url = None
                try:
                    new_url = Url.get_or_create(value=source.strip())
                    links.update(
                        observable.active_link_to(
                            new_url, 'c2', 'malshare_query'))
                except ObservableValidationError:
                    logging.error(
                        "An error occurred when trying to add {} to the database".
                        format(source.strip()))
            result['nb C2'] = len(json_result['SOURCES'])
        try:
            new_hash = Hash.get_or_create(value=json_result['MD5'])
            links.update(
                new_hash.active_link_to(observable, 'md5', 'malshare_query'))
            new_hash = Hash.get_or_create(value=json_result['SHA1'])
            links.update(
                new_hash.active_link_to(observable, 'sha1', 'malshare_query'))
            new_hash = Hash.get_or_create(value=json_result['SHA256'])
            links.update(
                new_hash.active_link_to(observable, 'sha256', 'malshare_query'))
        except ObservableValidationError:
            logging.error(
                "An error occurred when trying to add hashes {} to the database".
                format(json_string))

        return list(links)
Example #48
0
    def analyze(self, item):

        if not item or item[0].startswith("#"):
            return

        id_feed, dateadded, url, url_status, threat, tags, urlhaus_link = item

        context = {
            "id_urlhaus": id_feed,
            "first_seen": dateadded,
            "status": url_status,
            "source": self.name,
            "report": urlhaus_link,
            "threat": threat
        }

        if url:
            try:
                url_obs = Url.get_or_create(value=url)
                url_obs.tag(tags.split(','))
                url_obs.add_context(context)
                url_obs.add_source('feed')
            except ObservableValidationError as e:
                logging.error(e)
Example #49
0
    def _get_threat_forensics_nodes_inner(
            self, evidence, general_context, tags):
        # create context from notes
        context = general_context.copy()
        _ctx = self._make_context_from_notes([evidence])
        context.update(_ctx)
        # add evidence['type'] and unicify tags
        tags = [{
            'name': _
        } for _ in set([evidence['type']] + [d['name'] for d in tags])]
        # create Tags in DB
        for _ in tags:
            Tag.get_or_create(name=_['name'])
        #
        threat_forensics = []

        # technical hack: set optional comments values
        for optional in ['action', 'rule', 'path', 'rule']:
            if optional not in evidence['what']:
                evidence['what'][optional] = None

        # add attributes for the known evidence type
        if evidence['type'] in ['file', 'dropper']:
            if 'path' in evidence['what']:
                threat_forensics.append(
                    File.get_or_create(
                        value=evidence['what']['path'], context=[context]))
            if 'md5' in evidence['what']:
                threat_forensics.append(
                    Hash.get_or_create(
                        value=evidence['what']['md5'], context=[context]))
            if 'sha256' in evidence['what']:
                threat_forensics.append(
                    Hash.get_or_create(
                        value=evidence['what']['sha256'], context=[context]))
        elif evidence['type'] == 'cookie':
            pass
        elif evidence['type'] == 'dns':
            threat_forensics.append(
                Hostname.get_or_create(
                    value=evidence['what']['host'], context=[context]))
        elif evidence['type'] == 'ids':
            threat_forensics.append(
                Text.get_or_create(
                    value=evidence['what']['ids'], context=[context]))
            pass
        elif evidence['type'] == 'mutex':
            threat_forensics.append(
                Text.get_or_create(
                    value=evidence['what']['name'], context=[context]))
        elif evidence['type'] == 'network':
            if 'ip' in evidence['what']:
                # FIXME port, type
                threat_forensics.append(
                    Ip.get_or_create(
                        value=evidence['what']['ip'], context=[context]))
            elif 'domain' in evidence['what']:
                threat_forensics.append(
                    Hostname.get_or_create(
                        value=evidence['what']['domain'], context=[context]))
        elif evidence['type'] == 'process':
            pass
        elif evidence['type'] == 'registry':
            # threat_forensics.append(evidence['what']['key'])
            # threat_forensics.append(evidence['what']['value'])
            pass
        elif evidence['type'] == 'url':
            # BUG yeti-#115 ObservableValidationError: Invalid URL: http://xxxxx-no-tld/
            threat_forensics.append(
                Url.get_or_create(
                    value=evidence['what']['url'], context=[context]))
            # add note as tag because its a signature
            if 'note' in evidence:
                threat_forensics[-1].tag(
                    evidence['note'].replace('.', '_').strip('_'))
        # tag all of that
        for o in threat_forensics:
            o.tag([t['name'] for t in tags])
        return threat_forensics
Example #50
0
    def analyze(observable, results):
        links = set()
        json_result = VirustotalApi.fetch(
            observable, results.settings['virutotal_api_key'])
        json_string = json.dumps(
            json_result, sort_keys=True, indent=4, separators=(',', ': '))
        results.update(raw=json_string)
        result = {'raw': json_string}

        if isinstance(observable, Ip):
            # Parse results for ip
            if json_result.get('as_owner'):
                result['Owner'] = json_result['as_owner']
                o_isp = Company.get_or_create(name=json_result['as_owner'])
                links.update(
                    observable.active_link_to(
                        o_isp, 'hosting', 'virustotal_query'))

            if json_result.get('detected_urls'):
                result['detected_urls'] = json_result['detected_urls']
                for detected_url in json_result['detected_urls']:
                    o_url = Url.get_or_create(value=detected_url['url'])
                    links.update(
                        o_url.active_link_to(
                            o_url, 'hostname', 'virustotal_query'))

        elif isinstance(observable, Hostname):
            if json_result.get('permalink'):
                result['permalink'] = json_result['permalink']

            result['positives'] = json_result.get('positives', 0)

            if json_result.get('total'):
                result['total'] = json_result['total']

        elif isinstance(observable, Hash):

            result['positives'] = json_result['positives']

            if 'permalink' in json_result:
                result['permalink'] = json_result['permalink']

            if 'total' in json_result:
                result['total'] = json_result['total']

            hashes = {
                'md5': json_result['md5'],
                'sha1': json_result['sha1'],
                'sha256': json_result['sha256']
            }
            create_hashes = [
                (k, v) for k, v in hashes.items() if v != observable.value
            ]

            for k, v in create_hashes:
                new_hash = Hash.get_or_create(value=v)
                new_hash.tag(observable.get_tags())
                links.update(
                    new_hash.active_link_to(observable, k, 'virustotal_query'))

        result['source'] = 'virustotal_query'
        observable.add_context(result)
        return list(links)
Example #51
0
    def analyze(self, item):
        context = item
        date_string = re.search(
            r"\((?P<datetime>[\d\- :]+)\)", context['title']).group('datetime')
        try:
            context['date_added'] = datetime.strptime(
                date_string, "%Y-%m-%d %H:%M:%S")
        except ValueError:
            pass

        g = re.match(
            r'^Host: (?P<host>.+), Version: (?P<version>\w)',
            context['description'])
        g = g.groupdict()
        context['version'] = g['version']
        context['description'] = FeodoTracker.descriptions[g['version']]
        context['subfamily'] = FeodoTracker.variants[g['version']]
        context['source'] = self.name
        del context['title']
        new = None
        variant_tag = FeodoTracker.variants[g['version']].lower()
        try:
            if re.search(r"[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}",
                         g['host']):
                new = Ip.get_or_create(value=g['host'])
            else:
                new = Hostname.get_or_create(value=g['host'])
            new.add_context(context)
            new.add_source("feed")
            new.tag([variant_tag, 'malware', 'crimeware', 'banker', 'c2'])

        except ObservableValidationError as e:
            logging.error(e)

        try:
            url_fedeo = context['guid']
            r = requests.get(url_fedeo)
            if r.status_code == 200:

                html_source = r.text

                soup = BeautifulSoup(html_source, 'html.parser')
                tab = soup.find('table', attrs='sortable')
                results = []

                if tab:
                    all_tr = tab.find_all('tr')

                    for tr in all_tr:
                        all_td = tr.find_all('td')
                        if all_td and len(all_td) == 7:
                            results.append({
                                'timestamp': all_td[0].text,
                                'md5_hash': all_td[1].text,
                                'filesize': all_td[2].text,
                                'VT': all_td[3].text,
                                'Host': all_td[4].text,
                                'Port': all_td[5].text,
                                'SSL Certif or method': all_td[6].text
                            })

                for r in results:
                    new_hash = Hash.get_or_create(value=r['md5_hash'])
                    new_hash.add_context(context)
                    new_hash.add_source('feed')
                    new_hash.tag([
                        variant_tag, 'malware', 'crimeware', 'banker', 'payload'
                    ])
                    new_hash.active_link_to(
                        new, 'c2', self.name, clean_old=False)
                    host = Url.get_or_create(
                        value='https://%s:%s' % (g['host'], r['Port']))
                    host.add_source('feed')
                    host.add_context(context)
                    host.tag(
                        [variant_tag, 'malware', 'crimeware', 'banker', 'c2'])
                    new_hash.active_link_to(
                        host, 'c2', self.name, clean_old=False)

        except ObservableValidationError as e:
            logging.error(e)