Exemple #1
0
    def process(self, i, router):
        if i.itype != 'fqdn':
            return

        if 'search' in i.tags:
            return

        try:
            r = resolve_ns(i.indicator)
        except Timeout:
            self.logger.info('timeout trying to resolve: {}'.format(
                i.indicator))
            return

        for rr in r:
            ip = Indicator(**i.__dict__())
            ip.indicator = str(rr)
            try:
                resolve_itype(ip.indicator)
            except InvalidIndicator as e:
                self.logger.error(ip)
                self.logger.error(e)
            else:
                ip.itype = 'ipv4'
                ip.rdata = i.indicator
                ip.confidence = (int(ip.confidence) / 4)
                router.indicators_create(ip)

                # also create a passive dns tag
                ip.tags = 'pdns'
                ip.confidence = 10
                router.indicators_create(ip)
Exemple #2
0
    def process(self, i, router):
        if i.itype == 'fqdn':
            r = resolve_ns(i.indicator, t='CNAME')
            self.logger.debug('CNAME: {}'.format(r))
            for rr in r:
                fqdn = Indicator(**i.__dict__)

                fqdn.indicator = str(rr).rstrip('.')
                fqdn.itype = 'fqdn'
                fqdn.confidence = (int(fqdn.confidence) / 2)
                x = router.indicators_create(fqdn)
                self.logger.debug(x)

            if i.is_subdomain():
                fqdn = Indicator(**i.__dict__)
                fqdn.indicator = i.is_subdomain()
                fqdn.confidence = (int(fqdn.confidence) / 3)
                x = router.indicators_create(fqdn)
                self.logger.debug(x)

            r = resolve_ns(i.indicator)
            self.logger.debug(r)
            for rr in r:
                ip = Indicator(**i.__dict__)
                ip.indicator = str(rr)
                ip.itype = 'ipv4'
                ip.confidence = (int(ip.confidence) / 4)
                x = router.indicators_create(ip)
                self.logger.debug(x)

            r = resolve_ns(i.indicator, t='NS')
            self.logger.debug('NS: {}'.format(r))
            for rr in r:
                ip = Indicator(**i.__dict__)
                ip.indicator = str(rr).rstrip('.')
                ip.itype = 'fqdn'
                ip.confidence = (int(ip.confidence) / 5)
                x = router.indicators_create(ip)
                self.logger.debug(x)

            r = resolve_ns(i.indicator, t='MX')
            self.logger.debug('MX: {}'.format(r))
            for rr in r:
                ip = Indicator(**i.__dict__)
                ip.indicator = str(rr).rstrip('.')
                ip.itype = 'fqdn'
                ip.confidence = (int(ip.confidence) / 6)
                x = router.indicators_create(ip)
                self.logger.debug(x)
Exemple #3
0
    def process(self, i, router):
        if i.itype == 'fqdn':
            r = resolve_ns(i.indicator, t='CNAME')
            self.logger.debug('CNAME: {}'.format(r))
            for rr in r:
                fqdn = Indicator(**i.__dict__)

                fqdn.indicator = str(rr).rstrip('.')
                fqdn.itype = 'fqdn'
                fqdn.confidence = (int(fqdn.confidence) / 2)
                x = router.indicators_create(fqdn)
                self.logger.debug(x)

            if i.is_subdomain():
                fqdn = Indicator(**i.__dict__)
                fqdn.indicator = i.is_subdomain()
                fqdn.confidence = (int(fqdn.confidence) / 3)
                x = router.indicators_create(fqdn)
                self.logger.debug(x)

            r = resolve_ns(i.indicator)
            self.logger.debug(r)
            for rr in r:
                ip = Indicator(**i.__dict__)
                ip.indicator = str(rr)
                ip.itype = 'ipv4'
                ip.confidence = (int(ip.confidence) / 4)
                x = router.indicators_create(ip)
                self.logger.debug(x)

            r = resolve_ns(i.indicator, t='NS')
            self.logger.debug('NS: {}'.format(r))
            for rr in r:
                ip = Indicator(**i.__dict__)
                ip.indicator = str(rr).rstrip('.')
                ip.itype = 'fqdn'
                ip.confidence = (int(ip.confidence) / 5)
                x = router.indicators_create(ip)
                self.logger.debug(x)

            r = resolve_ns(i.indicator, t='MX')
            self.logger.debug('MX: {}'.format(r))
            for rr in r:
                ip = Indicator(**i.__dict__)
                ip.indicator = str(rr).rstrip('.')
                ip.itype = 'fqdn'
                ip.confidence = (int(ip.confidence) / 6)
                x = router.indicators_create(ip)
                self.logger.debug(x)
Exemple #4
0
    def process(self, i, router):
        if i.itype != 'fqdn':
            return

        if 'search' in i.tags:
            return

        try:
            r = resolve_ns(i.indicator, t='CNAME')
        except Timeout:
            self.logger.info('timeout trying to resolve: {}'.format(i.indicator))
            r = []

        for rr in r:
            # http://serverfault.com/questions/44618/is-a-wildcard-cname-dns-record-valid
            rr = str(rr).rstrip('.').lstrip('*.')
            if rr in ['', 'localhost', '0.0.0.0']:
                continue

            fqdn = Indicator(**i.__dict__())
            fqdn.indicator = rr
            fqdn.lasttime = arrow.utcnow()

            try:
                resolve_itype(fqdn.indicator)
            except InvalidIndicator as e:
                self.logger.error(fqdn)
                self.logger.error(e)
                return

            fqdn.itype = 'fqdn'
            fqdn.confidence = (fqdn.confidence - 1)
            router.indicators_create(fqdn)
    def process(self, i, router, **kwargs):
        if i.itype != 'fqdn':
            return

        if 'search' in i.tags:
            return

        try:
            r = resolve_ns(i.indicator, t='NS')
        except Timeout:
            self.logger.info('timeout trying to resolve: {}'.format(i.indicator))
            return

        for rr in r:
            rr = str(rr).rstrip('.')
            if rr in ["", 'localhost', '0.0.0.0']:
                continue

            i_ns = Indicator(**i.__dict__())
            i_ns.indicator = rr

            try:
                i_ns_itype = resolve_itype(i_ns.indicator)
            except InvalidIndicator as e:
                self.logger.error(i_ns)
                self.logger.error(e)
            else:
                i_ns.lasttime = i_ns.reporttime = arrow.utcnow()
                i_ns.itype = i_ns_itype
                i_ns.rdata = "{} nameserver".format(i.indicator)
                if 'hunter' not in i_ns.tags:
                    i_ns.tags.append('hunter')
                i_ns.confidence = (i_ns.confidence - 4) if i_ns.confidence >= 4 else 0
                router.indicators_create(i_ns)
                self.logger.debug("FQDN NS Hunter: {}".format(i_ns))
    def process(self, i, router):
        if i.itype != 'ipv4':
            return

        if 'whitelist' not in i.tags:
            return
        
        # only run this hunter if it's a single address (no CIDRs)
        if ipaddress.IPv4Network(i.indicator).prefixlen != 32:
            return

        prefix = i.indicator.split('.')
        prefix = prefix[:3]
        prefix.append('0/24')
        prefix = '.'.join(prefix)

        try:
            ii = Indicator(**i.__dict__())
        except InvalidIndicator as e:
            self.logger.error(e)
            return

        ii.lasttime = ii.reporttime = arrow.utcnow()

        ii.indicator = prefix
        ii.tags = ['whitelist', 'hunter']
        ii.confidence = (ii.confidence - 2) if ii.confidence >= 2 else 0
        router.indicators_create(ii)
    def process(self, i, router):
        if i.itype == 'fqdn' and i.provider != 'spamhaus.org':
            try:
                r = self._resolve(i.indicator)

                try:
                    r = CODES[r]
                except Exception as e:
                    # https://www.spamhaus.org/faq/section/DNSBL%20Usage
                    self.logger.error(e)
                    self.logger.info('check spamhaus return codes')
                    r = None

                if r:
                    f = Indicator(**i.__dict__)

                    f.tags = [r['tags']]
                    f.description = r['description']
                    f.confidence = CONFIDENCE
                    f.provider = PROVIDER
                    f.reference_tlp = 'white'
                    f.reference = 'http://www.spamhaus.org/query/dbl?domain={}'.format(f.indicator)
                    x = router.indicators_create(f)
                    self.logger.debug(x)
            except KeyError as e:
                self.logger.error(e)
            except dns.resolver.NoAnswer:
                self.logger.info('no answer...')
            except dns.resolver.NXDOMAIN:
                self.logger.info('nxdomain...')
            except EmptyLabel:
                self.logger.error('empty label: {}'.format(i.indicator))
Exemple #8
0
    def process(self, i, router, **kwargs):
        if i.itype != 'fqdn':
            return

        if 'search' in i.tags:
            return

        try:
            r = resolve_ns(i.indicator)
        except Timeout:
            self.logger.info('timeout trying to resolve: {}'.format(
                i.indicator))
            return

        for rr in r:
            rr = str(rr)
            if rr in ["", 'localhost', '0.0.0.0']:
                continue

            ip = Indicator(**i.__dict__())
            ip.lasttime = ip.reporttime = arrow.utcnow()

            ip.indicator = rr
            try:
                resolve_itype(ip.indicator)
            except InvalidIndicator as e:
                self.logger.error(ip)
                self.logger.error(e)
            else:
                ip.itype = 'ipv4'
                ip.rdata = i.indicator
                ip.tags = ['pdns', 'hunter']
                ip.confidence = 10
                router.indicators_create(ip)
                self.logger.debug("FQDN Hunter: {}".format(ip))
    def process(self, i, router):
        if i.itype == 'fqdn' and i.provider != 'spamhaus.org':
            try:
                r = self._resolve(i.indicator)
                self.logger.debug(r)
                try:
                    r = CODES[r]
                except Exception as e:
                    # https://www.spamhaus.org/faq/section/DNSBL%20Usage
                    self.logger.error(e)
                    self.logger.info('check spamhaus return codes')
                    r = None

                if r:
                    f = Indicator(**i.__dict__)

                    f.tags = [r['tags']]
                    f.description = r['description']
                    f.confidence = CONFIDENCE
                    f.provider = PROVIDER
                    f.reference_tlp = 'white'
                    f.reference = 'http://www.spamhaus.org/query/dbl?domain={}'.format(
                        f.indicator)
                    x = router.indicators_create(f)
                    self.logger.debug(x)
            except KeyError as e:
                self.logger.error(e)
            except dns.resolver.NoAnswer:
                self.logger.debug('no answer...')
            except dns.resolver.NXDOMAIN:
                self.logger.debug('nxdomain...')
            except EmptyLabel:
                self.logger.error('empty label: {}'.format(i.indicator))
Exemple #10
0
    def process(self, i, router):
        if i.itype != 'ipv4':
            return

        if 'whitelist' not in i.tags:
            return

        if i.indicator.endswith('/24'):
            return

        prefix = i.indicator.split('.')
        prefix = prefix[:3]
        prefix.append('0/24')
        prefix = '.'.join(prefix)

        try:
            ii = Indicator(**i.__dict__())
        except InvalidIndicator as e:
            self.logger.error(e)
            return

        ii.lasttime = arrow.utcnow()

        ii.indicator = prefix
        ii.tags = ['whitelist']
        ii.confidence = (ii.confidence - 2) if ii.confidence >= 2 else 0
        router.indicators_create(ii)
    def process(self, i, router):
        if i.itype != 'url':
            return

        if 'search' in i.tags:
            return

        u = urlparse(i.indicator)
        if not u.hostname:
            return

        try:
            resolve_itype(u.hostname)
        except InvalidIndicator as e:
            self.logger.error(u.hostname)
            self.logger.error(e)
        else:
            fqdn = Indicator(**i.__dict__())
            fqdn.lasttime = arrow.utcnow()
            fqdn.indicator = u.hostname
            fqdn.itype = 'fqdn'
            fqdn.confidence = (int(fqdn.confidence) / 2)
            fqdn.rdata = i.indicator

            self.logger.debug('sending to router: {}'.format(fqdn))
            router.indicators_create(fqdn)
def process(i):
    if not ENABLED:
        return

    if i.itype != 'url':
        return

    if i.probability:
        return

    for t in i.tags:
        if t == 'predicted':
            return

    if not predict(i.indicator):
        return

    i = Indicator(**i.__dict__())
    i.lasttime = arrow.utcnow()
    i.confidence = 4
    i.probability = 84
    i.provider = 'csirtgadgets.com'
    i.reference = 'https://github.com/csirtgadgets/csirtg-urlsml-py' + '#' + VERSION

    tags = set(i.tags)
    tags.add('predicted')
    i.tags = list(tags)

    return i
Exemple #13
0
def process(i):
    if not ENABLED:
        return

    if i.itype not in ['ipv4', 'ipv6']:
        return

    if i.provider == 'spamhaus.org' and not is_ipv4_net(i.indicator):
        return

    try:
        r = _resolve(i.indicator)
    except Exception as e:
        return

    r = CODES.get(str(r), None)
    if not r:
        return

    f = Indicator(**i.__dict__())

    f.tags = [r['tags']]
    f.description = r['description']
    f.confidence = CONFIDENCE
    f.provider = PROVIDER
    f.reference_tlp = 'white'
    f.reference = 'http://www.spamhaus.org/query/bl?ip={}'.format(f.indicator)
    f.lasttime = arrow.utcnow()
    f.probability = 0
    return f
Exemple #14
0
def process(i):
    if not ENABLED:
        return

    if i.itype != 'fqdn':
        return

    if i.provider == 'spamhaus.org':
        return

    r = _resolve(i.indicator)
    r = CODES.get(str(r), None)

    if not r:
        return

    confidence = CONFIDENCE
    if ' legit ' in r['description']:
        confidence = 1

    f = Indicator(**i.__dict__())

    f.tags = [r['tags']]
    f.description = r['description']
    f.confidence = confidence
    f.provider = PROVIDER
    f.reference_tlp = 'white'
    f.reference = 'http://www.spamhaus.org/query/dbl?domain={}'.format(
        f.indicator)
    f.lasttime = arrow.utcnow()
    f.probability = 0
    return f
    def process(self, i, router):
        if (i.itype == 'ipv4'
                or i.itype == 'ipv6') and i.provider != 'spamhaus.org':
            try:
                r = self._resolve(i.indicator)
                try:
                    r = CODES.get(str(r), None)
                except Exception as e:
                    # https://www.spamhaus.org/faq/section/DNSBL%20Usage
                    self.logger.error(e)
                    self.logger.info('check spamhaus return codes')
                    r = None

                if r:
                    f = Indicator(**i.__dict__)

                    f.tags = [r['tags']]
                    f.description = r['description']
                    f.confidence = CONFIDENCE
                    f.provider = PROVIDER
                    f.reference_tlp = 'white'
                    f.reference = 'http://www.spamhaus.org/query/bl?ip={}'.format(
                        f.indicator)
                    x = router.indicators_create(f)
                    self.logger.debug(x)
            except dns.resolver.NoAnswer:
                self.logger.info('no answer...')
            except dns.resolver.NXDOMAIN:
                self.logger.info('nxdomain...')
            except Exception as e:
                self.logger.error(e)
                import traceback
                traceback.print_exc()
    def process(self, i, router):
        if (i.itype == 'ipv4' or i.itype == 'ipv6') and i.provider != 'spamhaus.org':
            try:
                r = self._resolve(i.indicator)
                try:
                    r = CODES.get(str(r), None)
                except Exception as e:
                    # https://www.spamhaus.org/faq/section/DNSBL%20Usage
                    self.logger.error(e)
                    self.logger.info('check spamhaus return codes')
                    r = None

                if r:
                    f = Indicator(**i.__dict__)

                    f.tags = [r['tags']]
                    f.description = r['description']
                    f.confidence = CONFIDENCE
                    f.provider = PROVIDER
                    f.reference_tlp = 'white'
                    f.reference = 'http://www.spamhaus.org/query/bl?ip={}'.format(f.indicator)
                    x = router.indicators_create(f)
                    self.logger.debug(x)
            except dns.resolver.NoAnswer:
                self.logger.info('no answer...')
            except dns.resolver.NXDOMAIN:
                self.logger.info('nxdomain...')
            except Exception as e:
                self.logger.error(e)
                import traceback
                traceback.print_exc()
    def process(self, i, router):
        if i.itype != 'fqdn':
            return

        if 'search' in i.tags:
            return

        try:
            r = resolve_ns(i.indicator)
        except Timeout:
            self.logger.info('timeout trying to resolve: {}'.format(i.indicator))
            return

        for rr in r:
            if str(rr).rstrip('.') in ["", 'localhost']:
                continue

            ip = Indicator(**i.__dict__())
            ip.indicator = str(rr)
            ip.lasttime = arrow.utcnow()

            try:
                resolve_itype(ip.indicator)
            except InvalidIndicator as e:
                self.logger.error(ip)
                self.logger.error(e)
            else:
                ip.itype = 'ipv4'
                ip.rdata = i.indicator
                ip.confidence = (ip.confidence - 4) if ip.confidence >= 4 else 0
                router.indicators_create(ip)
Exemple #18
0
    def process(self, i, router):
        if i.itype != 'fqdn':
            return

        if 'search' in i.tags:
            return

        try:
            r = resolve_ns(i.indicator, t='MX')
        except Timeout:
            self.logger.info('timeout trying to resolve MX for: {}'.format(i.indicator))
            return

        for rr in r:
            rr = re.sub(r'^\d+ ', '', str(rr))
            fqdn = Indicator(**i.__dict__())
            fqdn.indicator = rr.rstrip('.')
            try:
                resolve_itype(fqdn.indicator)
            except InvalidIndicator as e:
                if not str(e).startswith('unknown itype for "localhost"'):
                    self.logger.error(fqdn)
                    self.logger.error(e)
            else:
                fqdn.itype = 'fqdn'
                fqdn.rdata = i.indicator
                fqdn.confidence = (int(fqdn.confidence) / 6)
                router.indicators_create(fqdn)
Exemple #19
0
    def process(self, i, router):
        if i.itype != 'fqdn':
            return

        try:
            r = resolve_ns(i.indicator, t='CNAME')
        except Timeout:
            self.logger.info('timeout trying to resolve: {}'.format(
                i.indicator))
            r = []

        for rr in r:
            fqdn = Indicator(**i.__dict__())
            fqdn.indicator = str(rr).rstrip('.')

            try:
                resolve_itype(fqdn.indicator)
            except InvalidIndicator as e:
                self.logger.error(fqdn)
                self.logger.error(e)
                return

            fqdn.itype = 'fqdn'
            fqdn.confidence = (int(fqdn.confidence) / 2)
            router.indicators_create(fqdn)
    def process(self, i, router):
        if i.itype != 'fqdn':
            return

        try:
            r = resolve_ns(i.indicator, t='CNAME')
        except Timeout:
            self.logger.info('timeout trying to resolve: {}'.format(i.indicator))
            r = []

        for rr in r:
            # http://serverfault.com/questions/44618/is-a-wildcard-cname-dns-record-valid
            rr = str(rr).rstrip('.').lstrip('*.')
            if rr in ['', 'localhost']:
                continue

            fqdn = Indicator(**i.__dict__())
            fqdn.indicator = rr
            fqdn.lasttime = arrow.utcnow()

            try:
                resolve_itype(fqdn.indicator)
            except InvalidIndicator as e:
                self.logger.error(fqdn)
                self.logger.error(e)
                return

            fqdn.itype = 'fqdn'
            fqdn.confidence = (fqdn.confidence - 1)
            router.indicators_create(fqdn)
    def process(self, i, router):
        if i.itype != 'fqdn':
            return

        if 'search' in i.tags:
            return

        if not i.is_subdomain():
            return

        fqdn = Indicator(**i.__dict__())
        fqdn.indicator = i.is_subdomain()
        fqdn.lasttime = fqdn.reporttime = arrow.utcnow()

        try:
            resolve_itype(fqdn.indicator)
        except InvalidIndicator as e:
            self.logger.error(fqdn)
            self.logger.error(e)
        else:
            fqdn.confidence = (fqdn.confidence -
                               3) if fqdn.confidence >= 3 else 0
            if 'hunter' not in fqdn.tags:
                fqdn.tags.append('hunter')
            router.indicators_create(fqdn)
Exemple #22
0
    def process(self, i, router):
        if i.itype != 'fqdn':
            return

        if 'search' in i.tags:
            return

        try:
            r = resolve_ns(i.indicator)
        except Timeout:
            self.logger.info('timeout trying to resolve: {}'.format(
                i.indicator))
            return

        for rr in r:
            if str(rr).rstrip('.') in ["", 'localhost']:
                continue

            ip = Indicator(**i.__dict__())
            ip.indicator = str(rr)
            ip.lasttime = arrow.utcnow()

            try:
                resolve_itype(ip.indicator)
            except InvalidIndicator as e:
                self.logger.error(ip)
                self.logger.error(e)
            else:
                ip.itype = 'ipv4'
                ip.rdata = i.indicator
                ip.confidence = (ip.confidence -
                                 4) if ip.confidence >= 4 else 0
                router.indicators_create(ip)
    def process(self, i, router):
        if i.itype != 'ipv4' and i.itype != 'ipv6':
            return

        if i.provider == 'spamhaus.org' and not is_ipv4_net(i.indicator):
            return

        try:
            r = self._resolve(i.indicator)
            try:
                r = CODES.get(str(r), None)
            except Exception as e:
                # https://www.spamhaus.org/faq/section/DNSBL%20Usage
                self.logger.error(e)
                self.logger.info('check spamhaus return codes')
                r = None

            if r:
                f = Indicator(**i.__dict__())

                f.tags = [r['tags']]
                f.description = r['description']
                f.confidence = CONFIDENCE
                f.provider = PROVIDER
                f.reference_tlp = 'white'
                f.reference = 'http://www.spamhaus.org/query/bl?ip={}'.format(f.indicator)
                f.lasttime = arrow.utcnow()
                x = router.indicators_create(f)

        except Exception as e:
            self.logger.error(e)
            import traceback
            traceback.print_exc()
    def process(self, i, router):
        if i.itype != 'ipv4' and i.itype != 'ipv6':
            return

        if i.provider == 'spamhaus.org' and not is_ipv4_net(i.indicator):
            return

        try:
            r = self._resolve(i.indicator)
            try:
                r = CODES.get(str(r), None)
            except Exception as e:
                # https://www.spamhaus.org/faq/section/DNSBL%20Usage
                self.logger.error(e)
                self.logger.info('check spamhaus return codes')
                r = None

            if r:
                f = Indicator(**i.__dict__())

                f.tags = [r['tags']]
                f.description = r['description']
                f.confidence = CONFIDENCE
                f.provider = PROVIDER
                f.reference_tlp = 'white'
                f.reference = 'http://www.spamhaus.org/query/bl?ip={}'.format(
                    f.indicator)
                x = router.indicators_create(f)

        except Exception as e:
            self.logger.error(e)
            import traceback
            traceback.print_exc()
    def process(self, i, router):
        if i.itype == 'fqdn' and i.provider != 'spamhaus.org':
            try:
                r = self._resolve(i.indicator)
                try:
                    r = CODES.get(str(r), None)
                except Exception as e:
                    # https://www.spamhaus.org/faq/section/DNSBL%20Usage
                    self.logger.error(e)
                    self.logger.info('check spamhaus return codes')
                    r = None

                if r:
                    confidence = CONFIDENCE
                    if ' legit ' in r['description']:
                        confidence = 6

                    f = Indicator(**i.__dict__())

                    f.tags = [r['tags']]
                    f.description = r['description']
                    f.confidence = confidence
                    f.provider = PROVIDER
                    f.reference_tlp = 'white'
                    f.reference = 'http://www.spamhaus.org/query/dbl?domain={}'.format(f.indicator)
                    f.lasttime = arrow.utcnow()
                    x = router.indicators_create(f)
                    self.logger.debug(x)
            except KeyError as e:
                self.logger.error(e)
Exemple #26
0
    def process(self, i, router, **kwargs):
        if i.itype != 'url':
            return

        if 'search' in i.tags:
            return

        # prevent recursion with fqdn_wl hunter
        if ('whitelist') in i.tags and (i.rdata is not None or i.rdata != ''):
            return

        u = urlparse(i.indicator)
        if not u.hostname:
            return

        try:
            resolve_itype(u.hostname)
        except InvalidIndicator as e:
            self.logger.error(u.hostname)
            self.logger.error(e)
        else:
            fqdn = Indicator(**i.__dict__())
            fqdn.lasttime = fqdn.reporttime = arrow.utcnow()
            fqdn.indicator = u.hostname
            fqdn.itype = 'fqdn'
            if 'hunter' not in fqdn.tags:
                fqdn.tags.append('hunter')
            fqdn.confidence = (int(fqdn.confidence) / 2)
            fqdn.rdata = i.indicator

            self.logger.debug('[Hunter: Url] sending to router {}'.format(fqdn))
            router.indicators_create(fqdn)
Exemple #27
0
def process(i):
    if not ENABLED:
        return

    if i.itype != 'fqdn':
        return

    try:
        r = resolve_ns(i.indicator)
        if not r:
            return
    except Timeout:
        return

    rv = []

    for rr in r:
        rr = str(rr)
        if rr in ["", 'localhost']:
            continue

        ip = Indicator(**i.__dict__())
        ip.lasttime = arrow.utcnow()

        ip.indicator = rr
        try:
            resolve_itype(ip.indicator)
        except:
            continue

        ip.itype = 'ipv4'
        ip.rdata = i.indicator
        ip.confidence = 1
        ip.probability = 0
        rv.append(ip)

        pdns = Indicator(**copy.deepcopy(i.__dict__()))

        # also create a passive dns tag
        pdns.tags = 'pdns'
        pdns.confidence = 4
        pdns.probability = i.probability
        pdns.indicator = ip.indicator
        pdns.rdata = i.indicator
        rv.append(pdns)

    return rv
Exemple #28
0
    def process(self, i, router):
        if i.itype == 'url':
            u = urlparse(i.indicator)
            if u.netloc:
                fqdn = Indicator(**i.__dict__)
                fqdn.indicator = u.netloc
                fqdn.itype = 'fqdn'
                fqdn.confidence = (int(fqdn.confidence) / 2)
                fqdn.rdata = i.indicator

            self.logger.debug('sending to router..')
            x = router.indicators_create(fqdn)
Exemple #29
0
    def process(self, i, router):
        if i.itype == 'url':
            u = urlparse(i.indicator)
            if u.netloc:
                fqdn = Indicator(**i.__dict__)
                fqdn.indicator = u.netloc
                fqdn.itype = 'fqdn'
                fqdn.confidence = (int(fqdn.confidence) / 2)
                fqdn.rdata = i.indicator

            self.logger.debug('sending to router..')
            x = router.indicators_create(fqdn)
    def process(self, i, router):
        if i.itype != 'fqdn':
            return

        if 'search' in i.tags:
            return

        try:
            r = resolve_ns(i.indicator)
        except Timeout:
            self.logger.info('timeout trying to resolve: {}'.format(
                i.indicator))
            return

        for rr in r:
            rr = str(rr)
            if rr in ["", 'localhost', '0.0.0.0']:
                continue

            ip = Indicator(**i.__dict__())
            ip.lasttime = arrow.utcnow()

            ip.indicator = rr
            try:
                resolve_itype(ip.indicator)
            except InvalidIndicator as e:
                self.logger.error(ip)
                self.logger.error(e)
            else:
                ip.itype = 'ipv4'
                ip.rdata = i.indicator
                ip.confidence = (ip.confidence -
                                 2) if ip.confidence >= 2 else 0
                router.indicators_create(ip)

                # also create a passive dns tag
                ip.tags = 'pdns'
                ip.confidence = 10
                router.indicators_create(ip)
    def process(self, i, router, **kwargs):
        if i.itype != 'fqdn':
            return

        if 'search' in i.tags:
            return

        try:
            r = resolve_ns(i.indicator, t='MX')
        except Timeout:
            self.logger.info('timeout trying to resolve MX for: {}'.format(
                i.indicator))
            return

        try:
            for rr in r:
                rr = re.sub(r'^\d+ ', '', str(rr))
                rr = str(rr).rstrip('.')

                if rr in ["", 'localhost', '0.0.0.0']:
                    continue
                elif re.match('^\d+$', rr) or re.match(r'^.{0,3}$', rr):
                    # exclude spurious entries like those too short to be real
                    continue

                fqdn = Indicator(**i.__dict__())
                fqdn.indicator = rr.rstrip('.')
                fqdn.lasttime = fqdn.reporttime = arrow.utcnow()

                try:
                    resolve_itype(fqdn.indicator)
                except InvalidIndicator as e:
                    self.logger.info(fqdn)
                    self.logger.info(e)
                else:
                    fqdn.itype = 'fqdn'
                    if 'hunter' not in fqdn.tags:
                        fqdn.tags.append('hunter')
                    fqdn.rdata = '{} mx'.format(i.indicator)
                    fqdn.confidence = (fqdn.confidence -
                                       5) if fqdn.confidence >= 5 else 0
                    router.indicators_create(fqdn)
                    self.logger.debug("FQDN MX Hunter: {}".format(fqdn))

        except Exception as e:
            self.logger.error(
                '[Hunter: FqdnMx] {}: giving up on rr {} from indicator {}'.
                format(e, rr, i))
Exemple #32
0
def process(i):
    return
    if not ENABLED:
        return

    if i.itype != 'fqdn':
        return

    if 'search' in i.tags:
        return

    try:
        r = resolve_ns(i.indicator, t='MX')
        if not r:
            return
    except Timeout:
        return

    rv = []

    for rr in r:
        rr = re.sub(r'^\d+ ', '', str(rr))
        rr = str(rr).rstrip('.')

        if rr in ["", 'localhost']:
            continue

        # 10
        if re.match('^\d+$', rr):
            continue

        fqdn = Indicator(**i.__dict__())
        fqdn.probability = 0
        fqdn.indicator = rr.rstrip('.')
        fqdn.lasttime = arrow.utcnow()

        try:
            resolve_itype(fqdn.indicator)
        except:
            continue

        fqdn.itype = 'fqdn'
        fqdn.rdata = i.indicator
        fqdn.confidence = 0
        rv.append(fqdn)

    return rv
Exemple #33
0
    def process(self, i, router):
        if i.itype != 'fqdn':
            return

        if 'search' in i.tags:
            return

        try:
            r = resolve_ns(i.indicator, t='MX')
        except Timeout:
            self.logger.info('timeout trying to resolve MX for: {}'.format(
                i.indicator))
            return

        try:
            for rr in r:
                rr = re.sub(r'^\d+ ', '', str(rr))
                rr = str(rr).rstrip('.')

                if rr in ["", 'localhost', '0.0.0.0']:
                    continue

                fqdn = Indicator(**i.__dict__())
                fqdn.indicator = rr.rstrip('.')
                fqdn.lasttime = fqdn.reporttime = arrow.utcnow()

                # 10
                if re.match('^\d+$', rr):
                    return

                try:
                    resolve_itype(fqdn.indicator)
                except InvalidIndicator as e:
                    self.logger.info(fqdn)
                    self.logger.info(e)
                else:
                    fqdn.itype = 'fqdn'
                    if 'hunter' not in fqdn.tags:
                        fqdn.tags.append('hunter')
                    fqdn.rdata = i.indicator
                    fqdn.confidence = (fqdn.confidence -
                                       5) if fqdn.confidence >= 5 else 0
                    router.indicators_create(fqdn)

        except Exception as e:
            self.logger.error(
                '[Hunter: FqdnMx] {}: giving up on indicator {}'.format(e, rr))
Exemple #34
0
    def process(self, i, router):
        if i.itype == 'url':
            u = urlparse(i.indicator)
            if u.netloc:
                try:
                    resolve_itype(u.netloc)
                except InvalidIndicator as e:
                    self.logger.error(u.netloc)
                    self.logger.error(e)
                else:
                    fqdn = Indicator(**i.__dict__())
                    fqdn.indicator = u.netloc
                    fqdn.itype = 'fqdn'
                    fqdn.confidence = (int(fqdn.confidence) / 2)
                    fqdn.rdata = i.indicator

                    self.logger.debug('sending to router..')
                    router.indicators_create(fqdn)
Exemple #35
0
    def process(self, i, router):
        if i.itype == 'url':
            u = urlparse(i.indicator)
            if u.hostname:
                try:
                    resolve_itype(u.hostname)
                except InvalidIndicator as e:
                    self.logger.error(u.hostname)
                    self.logger.error(e)
                else:
                    fqdn = Indicator(**i.__dict__())
                    fqdn.indicator = u.hostname
                    fqdn.itype = 'fqdn'
                    fqdn.confidence = (int(fqdn.confidence) / 2)
                    fqdn.rdata = i.indicator

                    self.logger.debug('sending to router..')
                    router.indicators_create(fqdn)
    def process(self, i, router):
        if i.itype not in ['ipv4', 'ipv6']:
            return

        if 'whitelist' not in i.tags:
            return

        prefix = i.indicator.split('.')
        prefix = prefix[:3]
        prefix.append('0/24')
        prefix = '.'.join(prefix)

        ii = Indicator(**i.__dict__())
        ii.lasttime = arrow.utcnow()

        ii.indicator = prefix
        ii.tags = ['whitelist']
        ii.confidence = (ii.confidence - 2) if ii.confidence >= 2 else 0
        router.indicators_create(ii)
def process(i):
    if i.itype != 'fqdn':
        return

    if not i.is_subdomain():
        return

    fqdn = Indicator(**i.__dict__())
    fqdn.probability = 0
    fqdn.indicator = i.is_subdomain()
    fqdn.lasttime = arrow.utcnow()

    try:
        resolve_itype(fqdn.indicator)
    except:
        return

    fqdn.confidence = 1
    return fqdn
    def process(self, i, router):
        if i.itype not in ['ipv4', 'ipv6']:
            return

        if 'whitelist' not in i.tags:
            return

        prefix = i.indicator.split('.')
        prefix = prefix[:3]
        prefix.append('0/24')
        prefix = '.'.join(prefix)

        ii = Indicator(**i.__dict__())
        ii.lasttime = arrow.utcnow()

        ii.indicator = prefix
        ii.tags = ['whitelist']
        ii.confidence = (ii.confidence - 2) if ii.confidence >= 2 else 0
        router.indicators_create(ii)
    def process(self, i, router, **kwargs):
        if 'search' in i.tags:
            return

        if i.itype != 'ipv4' and i.itype != 'ipv6':
            return

        if i.provider == 'spamhaus.org' and not is_ipv4_net(i.indicator):
            return

        try:
            r = self._resolve(i.indicator)
            try:
                r = CODES.get(str(r), None)
            except Exception as e:
                # https://www.spamhaus.org/faq/section/DNSBL%20Usage
                self.logger.error(e)
                self.logger.info('check spamhaus return codes')
                r = None

            if r:
                f = Indicator(**i.__dict__())

                f.tags = [r['tags']]
                if 'hunter' not in f.tags:
                    f.tags.append('hunter')
                f.description = r['description']
                f.confidence = CONFIDENCE
                f.provider = PROVIDER
                f.reference_tlp = 'white'
                f.reference = 'http://www.spamhaus.org/query/bl?ip={}'.format(
                    f.indicator)
                f.lasttime = f.reporttime = arrow.utcnow()
                x = router.indicators_create(f)
                self.logger.debug("Spamhaus IP: {}".format(x))

        except Exception as e:
            self.logger.error(
                "[Hunter: SpamhausIp] {}: giving up on indicator {}".format(
                    e, i))
            import traceback
            traceback.print_exc()
    def process(self, i, router):
        if i.itype != 'fqdn':
            return

        if 'search' in i.tags:
            return

        if not i.is_subdomain():
            return

        fqdn = Indicator(**i.__dict__())
        fqdn.indicator = i.is_subdomain()
        fqdn.lasttime = arrow.utcnow()

        try:
            resolve_itype(fqdn.indicator)
        except InvalidIndicator as e:
            self.logger.error(fqdn)
            self.logger.error(e)
        else:
            fqdn.confidence = (fqdn.confidence - 3) if fqdn.confidence >= 3 else 0
            router.indicators_create(fqdn)
    def process(self, i, router):
        if i.itype != 'fqdn':
            return

        if 'search' in i.tags:
            return

        try:
            r = resolve_ns(i.indicator, t='MX')
        except Timeout:
            self.logger.info('timeout trying to resolve MX for: {}'.format(i.indicator))
            return

        for rr in r:
            rr = re.sub(r'^\d+ ', '', str(rr))
            rr = str(rr).rstrip('.')

            if rr in ["", 'localhost']:
                continue

            fqdn = Indicator(**i.__dict__())
            fqdn.indicator = rr.rstrip('.')
            fqdn.lasttime = arrow.utcnow()

            # 10
            if re.match('^\d+$', rr):
                return

            try:
                resolve_itype(fqdn.indicator)
            except InvalidIndicator as e:
                self.logger.info(fqdn)
                self.logger.info(e)
            else:
                fqdn.itype = 'fqdn'
                fqdn.rdata = i.indicator
                fqdn.confidence = (fqdn.confidence - 5) if fqdn.confidence >= 5 else 0
                router.indicators_create(fqdn)
Exemple #42
0
    def process(self, i, router):
        if i.itype != 'url':
            return

        u = urlparse(i.indicator)
        if not u.hostname:
            return

        try:
            resolve_itype(u.hostname)
        except InvalidIndicator as e:
            self.logger.error(u.hostname)
            self.logger.error(e)
        else:
            fqdn = Indicator(**i.__dict__())
            fqdn.lasttime = arrow.utcnow()
            fqdn.indicator = u.hostname
            fqdn.itype = 'fqdn'
            fqdn.confidence = (int(fqdn.confidence) / 2)
            fqdn.rdata = i.indicator

            self.logger.debug('sending to router: {}'.format(fqdn))
            router.indicators_create(fqdn)
Exemple #43
0
    def process(self, i, router):
        if i.itype == 'fqdn':
            try:
                r = resolve_ns(i.indicator, t='CNAME')
            except Timeout:
                self.logger.info('timeout trying to resolve: {}'.format(i.indicator))
                r = []

            for rr in r:
                fqdn = Indicator(**i.__dict__())
                fqdn.indicator = str(rr).rstrip('.')
                try:
                    resolve_itype(fqdn.indicator)
                except InvalidIndicator as e:
                    self.logger.error(fqdn)
                    self.logger.error(e)
                else:
                    fqdn.itype = 'fqdn'
                    fqdn.confidence = (int(fqdn.confidence) / 2)
                    router.indicators_create(fqdn)

            if i.is_subdomain():
                fqdn = Indicator(**i.__dict__())
                fqdn.indicator = i.is_subdomain()
                try:
                    resolve_itype(fqdn.indicator)
                except InvalidIndicator as e:
                    self.logger.error(fqdn)
                    self.logger.error(e)
                else:
                    fqdn.confidence = (int(fqdn.confidence) / 3)
                    router.indicators_create(fqdn)

            try:
                r = resolve_ns(i.indicator)
            except Timeout:
                self.logger.info('timeout trying to resolve: {}'.format(i.indicator))
                r = []

            for rr in r:
                ip = Indicator(**i.__dict__())
                ip.indicator = str(rr)
                try:
                    resolve_itype(ip.indicator)
                except InvalidIndicator as e:
                    self.logger.error(ip)
                    self.logger.error(e)
                else:
                    ip.itype = 'ipv4'
                    ip.rdata = i.indicator
                    ip.confidence = (int(ip.confidence) / 4)
                    router.indicators_create(ip)

            try:
                r = resolve_ns(i.indicator, t='NS')
            except Timeout:
                self.logger.info('timeout trying to resolve NS for: {}'.format(i.indicator))
                r = []

            for rr in r:
                fqdn = Indicator(**i.__dict__())
                fqdn.indicator = str(rr).rstrip('.')
                try:
                    resolve_itype(fqdn.indicator)
                except InvalidIndicator as e:
                    self.logger.error(fqdn)
                    self.logger.error(e)
                else:
                    fqdn.itype = 'fqdn'
                    fqdn.rdata = i.indicator
                    fqdn.confidence = (int(fqdn.confidence) / 5)
                    router.indicators_create(fqdn)

            try:
                r = resolve_ns(i.indicator, t='MX')
            except Timeout:
                self.logger.info('timeout trying to resolve MX for: {}'.format(i.indicator))
                r = []

            for rr in r:
                rr = re.sub(r'^\d+ ', '', str(rr))
                fqdn = Indicator(**i.__dict__())
                fqdn.indicator = rr.rstrip('.')
                try:
                    resolve_itype(fqdn.indicator)
                except InvalidIndicator as e:
                    if not str(e).startswith('unknown itype for "localhost"'):
                        self.logger.error(fqdn)
                        self.logger.error(e)
                else:
                    fqdn.itype = 'fqdn'
                    fqdn.rdata = i.indicator
                    fqdn.confidence = (int(fqdn.confidence) / 6)
                    router.indicators_create(fqdn)
Exemple #44
0
    def process(self, i, router):
        if i.itype == 'fqdn':
            try:
                r = resolve_ns(i.indicator, t='CNAME')
            except Timeout:
                self.logger.info('timeout trying to resolve: {}'.format(i.indicator))
                r = []

            for rr in r:
                fqdn = Indicator(**i.__dict__)

                fqdn.indicator = str(rr).rstrip('.')
                fqdn.itype = 'fqdn'
                fqdn.confidence = (int(fqdn.confidence) / 2)
                x = router.indicators_create(fqdn)

            if i.is_subdomain():
                fqdn = Indicator(**i.__dict__)
                fqdn.indicator = i.is_subdomain()
                fqdn.confidence = (int(fqdn.confidence) / 3)
                x = router.indicators_create(fqdn)

            try:
                r = resolve_ns(i.indicator)
            except Timeout:
                self.logger.info('timeout trying to resolve: {}'.format(i.indicator))
                r = []

            for rr in r:
                ip = Indicator(**i.__dict__)
                ip.indicator = str(rr)
                ip.itype = 'ipv4'
                ip.rdata = i.indicator
                ip.confidence = (int(ip.confidence) / 4)
                x = router.indicators_create(ip)
                self.logger.debug(x)

            try:
                r = resolve_ns(i.indicator, t='NS')
            except Timeout:
                self.logger.info('timeout trying to resolve NS for: {}'.format(i.indicator))
                r = []

            for rr in r:
                ip = Indicator(**i.__dict__)
                ip.indicator = str(rr).rstrip('.')
                ip.itype = 'fqdn'
                ip.rdata = i.indicator
                ip.confidence = (int(ip.confidence) / 5)
                x = router.indicators_create(ip)
                self.logger.debug(x)

            try:
                r = resolve_ns(i.indicator, t='MX')
            except Timeout:
                self.logger.info('timeout trying to resolve MX for: {}'.format(i.indicator))
                r = []

            for rr in r:
                ip = Indicator(**i.__dict__)
                ip.indicator = str(rr).rstrip('.')
                ip.itype = 'fqdn'
                ip.rdata = i.indicator
                ip.confidence = (int(ip.confidence) / 6)
                x = router.indicators_create(ip)
                self.logger.debug(x)