def process(self, i, router):
        if i.itype == 'fqdn' and i.provider != 'spamhaus.org':
            try:
                r = self._resolve(i.indicator)

                try:
                    r = CODES[r]
                except Exception as e:
                    # https://www.spamhaus.org/faq/section/DNSBL%20Usage
                    self.logger.error(e)
                    self.logger.info('check spamhaus return codes')
                    r = None

                if r:
                    f = Indicator(**i.__dict__)

                    f.tags = [r['tags']]
                    f.description = r['description']
                    f.confidence = CONFIDENCE
                    f.provider = PROVIDER
                    f.reference_tlp = 'white'
                    f.reference = 'http://www.spamhaus.org/query/dbl?domain={}'.format(f.indicator)
                    x = router.indicators_create(f)
                    self.logger.debug(x)
            except KeyError as e:
                self.logger.error(e)
            except dns.resolver.NoAnswer:
                self.logger.info('no answer...')
            except dns.resolver.NXDOMAIN:
                self.logger.info('nxdomain...')
            except EmptyLabel:
                self.logger.error('empty label: {}'.format(i.indicator))
    def process(self, i, router, **kwargs):
        if i.itype != 'fqdn':
            return

        if 'whitelist' not in i.tags:
            return

        urls = []
        for p in ['http://', 'https://']:
            urls.append('{}{}'.format(p, i.indicator))
            if not i.indicator.startswith('www.'):
                urls.append('{}www.{}'.format(p, i.indicator))

        for u in urls:
            url = Indicator(**i.__dict__())
            url.indicator = u

            try:
                resolve_itype(url.indicator)
            except InvalidIndicator as e:
                self.logger.error(url)
                self.logger.error(e)
            else:
                url.tags = ['whitelist', 'hunter']
                url.itype = 'url'
                url.rdata = i.indicator
                url.lasttime = url.reporttime = arrow.utcnow()
                router.indicators_create(url)
    def process(self, i, router):
        if i.itype != 'ipv4' and i.itype != 'ipv6':
            return

        if i.provider == 'spamhaus.org' and not is_ipv4_net(i.indicator):
            return

        try:
            r = self._resolve(i.indicator)
            try:
                r = CODES.get(str(r), None)
            except Exception as e:
                # https://www.spamhaus.org/faq/section/DNSBL%20Usage
                self.logger.error(e)
                self.logger.info('check spamhaus return codes')
                r = None

            if r:
                f = Indicator(**i.__dict__())

                f.tags = [r['tags']]
                f.description = r['description']
                f.confidence = CONFIDENCE
                f.provider = PROVIDER
                f.reference_tlp = 'white'
                f.reference = 'http://www.spamhaus.org/query/bl?ip={}'.format(f.indicator)
                f.lasttime = arrow.utcnow()
                x = router.indicators_create(f)

        except Exception as e:
            self.logger.error(e)
            import traceback
            traceback.print_exc()
    def process(self, i, router):
        if i.itype != 'ipv4':
            return

        if 'whitelist' not in i.tags:
            return
        
        # only run this hunter if it's a single address (no CIDRs)
        if ipaddress.IPv4Network(i.indicator).prefixlen != 32:
            return

        prefix = i.indicator.split('.')
        prefix = prefix[:3]
        prefix.append('0/24')
        prefix = '.'.join(prefix)

        try:
            ii = Indicator(**i.__dict__())
        except InvalidIndicator as e:
            self.logger.error(e)
            return

        ii.lasttime = ii.reporttime = arrow.utcnow()

        ii.indicator = prefix
        ii.tags = ['whitelist', 'hunter']
        ii.confidence = (ii.confidence - 2) if ii.confidence >= 2 else 0
        router.indicators_create(ii)
    def process(self, i, router):
        if i.itype != 'ipv4' and i.itype != 'ipv6':
            return

        if i.provider == 'spamhaus.org' and not is_ipv4_net(i.indicator):
            return

        try:
            r = self._resolve(i.indicator)
            try:
                r = CODES.get(str(r), None)
            except Exception as e:
                # https://www.spamhaus.org/faq/section/DNSBL%20Usage
                self.logger.error(e)
                self.logger.info('check spamhaus return codes')
                r = None

            if r:
                f = Indicator(**i.__dict__())

                f.tags = [r['tags']]
                f.description = r['description']
                f.confidence = CONFIDENCE
                f.provider = PROVIDER
                f.reference_tlp = 'white'
                f.reference = 'http://www.spamhaus.org/query/bl?ip={}'.format(
                    f.indicator)
                x = router.indicators_create(f)

        except Exception as e:
            self.logger.error(e)
            import traceback
            traceback.print_exc()
    def process(self, i, router):
        if i.itype == 'fqdn' and i.provider != 'spamhaus.org':
            try:
                r = self._resolve(i.indicator)
                try:
                    r = CODES.get(str(r), None)
                except Exception as e:
                    # https://www.spamhaus.org/faq/section/DNSBL%20Usage
                    self.logger.error(e)
                    self.logger.info('check spamhaus return codes')
                    r = None

                if r:
                    confidence = CONFIDENCE
                    if ' legit ' in r['description']:
                        confidence = 6

                    f = Indicator(**i.__dict__())

                    f.tags = [r['tags']]
                    f.description = r['description']
                    f.confidence = confidence
                    f.provider = PROVIDER
                    f.reference_tlp = 'white'
                    f.reference = 'http://www.spamhaus.org/query/dbl?domain={}'.format(f.indicator)
                    f.lasttime = arrow.utcnow()
                    x = router.indicators_create(f)
                    self.logger.debug(x)
            except KeyError as e:
                self.logger.error(e)
Exemple #7
0
def process(i):
    if not ENABLED:
        return

    if i.itype != 'fqdn':
        return

    if i.provider == 'spamhaus.org':
        return

    r = _resolve(i.indicator)
    r = CODES.get(str(r), None)

    if not r:
        return

    confidence = CONFIDENCE
    if ' legit ' in r['description']:
        confidence = 1

    f = Indicator(**i.__dict__())

    f.tags = [r['tags']]
    f.description = r['description']
    f.confidence = confidence
    f.provider = PROVIDER
    f.reference_tlp = 'white'
    f.reference = 'http://www.spamhaus.org/query/dbl?domain={}'.format(
        f.indicator)
    f.lasttime = arrow.utcnow()
    f.probability = 0
    return f
Exemple #8
0
    def process(self, i, router):
        if i.itype != 'ipv4':
            return

        if 'whitelist' not in i.tags:
            return

        if i.indicator.endswith('/24'):
            return

        prefix = i.indicator.split('.')
        prefix = prefix[:3]
        prefix.append('0/24')
        prefix = '.'.join(prefix)

        try:
            ii = Indicator(**i.__dict__())
        except InvalidIndicator as e:
            self.logger.error(e)
            return

        ii.lasttime = arrow.utcnow()

        ii.indicator = prefix
        ii.tags = ['whitelist']
        ii.confidence = (ii.confidence - 2) if ii.confidence >= 2 else 0
        router.indicators_create(ii)
    def process(self, i, router):
        if (i.itype == 'ipv4' or i.itype == 'ipv6') and i.provider != 'spamhaus.org':
            try:
                r = self._resolve(i.indicator)
                try:
                    r = CODES.get(str(r), None)
                except Exception as e:
                    # https://www.spamhaus.org/faq/section/DNSBL%20Usage
                    self.logger.error(e)
                    self.logger.info('check spamhaus return codes')
                    r = None

                if r:
                    f = Indicator(**i.__dict__)

                    f.tags = [r['tags']]
                    f.description = r['description']
                    f.confidence = CONFIDENCE
                    f.provider = PROVIDER
                    f.reference_tlp = 'white'
                    f.reference = 'http://www.spamhaus.org/query/bl?ip={}'.format(f.indicator)
                    x = router.indicators_create(f)
                    self.logger.debug(x)
            except dns.resolver.NoAnswer:
                self.logger.info('no answer...')
            except dns.resolver.NXDOMAIN:
                self.logger.info('nxdomain...')
            except Exception as e:
                self.logger.error(e)
                import traceback
                traceback.print_exc()
    def process(self, i, router):
        if (i.itype == 'ipv4'
                or i.itype == 'ipv6') and i.provider != 'spamhaus.org':
            try:
                r = self._resolve(i.indicator)
                try:
                    r = CODES.get(str(r), None)
                except Exception as e:
                    # https://www.spamhaus.org/faq/section/DNSBL%20Usage
                    self.logger.error(e)
                    self.logger.info('check spamhaus return codes')
                    r = None

                if r:
                    f = Indicator(**i.__dict__)

                    f.tags = [r['tags']]
                    f.description = r['description']
                    f.confidence = CONFIDENCE
                    f.provider = PROVIDER
                    f.reference_tlp = 'white'
                    f.reference = 'http://www.spamhaus.org/query/bl?ip={}'.format(
                        f.indicator)
                    x = router.indicators_create(f)
                    self.logger.debug(x)
            except dns.resolver.NoAnswer:
                self.logger.info('no answer...')
            except dns.resolver.NXDOMAIN:
                self.logger.info('nxdomain...')
            except Exception as e:
                self.logger.error(e)
                import traceback
                traceback.print_exc()
Exemple #11
0
def _get_indicator(i):
    i2 = Indicator()
    timestamps = []
    ports = []

    # prioritize the various elements..
    for e in i:
        if i[e] == 'CC':
            i2.cc = e
            continue

        if i[e] == 'indicator':
            if i2.indicator:
                i2.reference = e
            else:
                i2.indicator = e
            continue

        if i[e] == 'timestamp':
            timestamps.append(get_ts(e))
            continue

        if i[e] == 'float':
            i2.asn = e
            continue

        if i[e] == 'int':
            ports.append(e)
            continue

        if i[e] == 'description':
            i2.description = e
            continue

        if i[e] == 'string':
            if re.match(r'[0-9A-Za-z\.\s\/]+', e) and i2.asn:
                i2.asn_desc = e
                continue

            if 4 <= len(e) <= 10 and re.match('[a-z-A-Z]+,?', e) \
                    and e not in ['ipv4', 'fqdn', 'url', 'ipv6']:
                i2.tags = [e]
                continue

            if ' ' in e and 5 <= len(e) and not i2.asn_desc:
                i2.description = e
                continue

    _calc_timestamps(i2, timestamps)
    _calc_ports(i2, ports)
    return i2
Exemple #12
0
def text_to_list(text, known_only=True):
    separator = find_seperator(text)
    t_tokens = top_tokens(text)
    top = set()
    for t in range(0, 9):
        top.add(t_tokens[t])

    if known_only:
        if separator not in KNOWN_SEPERATORS:

            pprint(top)
            raise SystemError('separator not in known list: {}'.format(separator))

    ret = []

    for l in text.split("\n"):
        if l == '':
            continue

        if l.startswith('#') or l.startswith(';'):
            continue

        cols = l.split(separator)
        cols = [x.strip() for x in cols]
        indicator = Indicator()
        for e in cols:
            if e:
                try:
                    i = resolve_itype(e)
                    if i:
                        indicator.indicator = e
                        indicator.itype = i
                except NotImplementedError:
                    pass

                try:
                    ts = arrow.get(e)
                    if ts:
                        indicator.lasttime = ts.datetime
                except (arrow.parser.ParserError, UnicodeDecodeError):
                    pass

                if e in top:
                    indicator.tags = [e]

        if indicator.itype and indicator.indicator:
            ret.append(indicator)

    return ret
Exemple #13
0
def text_to_list(text, known_only=True):
    separator = find_seperator(text)
    t_tokens = top_tokens(text)
    top = set()
    for t in range(0, 9):
        top.add(t_tokens[t])

    if known_only:
        if separator not in KNOWN_SEPERATORS:
            raise SystemError(
                'separator not in known list: {}'.format(separator))

    ret = []

    for l in text.split("\n"):
        if l == '':
            continue

        if l.startswith('#') or l.startswith(';'):
            continue

        cols = l.split(separator)
        cols = [x.strip() for x in cols]
        indicator = Indicator()
        for e in cols:
            if e:
                try:
                    i = resolve_itype(e)
                    if i:
                        indicator.indicator = e
                        indicator.itype = i
                except TypeError:
                    pass

                try:
                    ts = arrow.get(e)
                    if ts:
                        indicator.lasttime = ts.datetime
                except (arrow.parser.ParserError, UnicodeDecodeError):
                    pass

                if e in top:
                    indicator.tags = [e]

        if indicator.itype and indicator.indicator:
            ret.append(indicator)

    return ret
Exemple #14
0
def process(i):
    if not ENABLED:
        return

    if i.itype != 'fqdn':
        return

    try:
        r = resolve_ns(i.indicator)
        if not r:
            return
    except Timeout:
        return

    rv = []

    for rr in r:
        rr = str(rr)
        if rr in ["", 'localhost']:
            continue

        ip = Indicator(**i.__dict__())
        ip.lasttime = arrow.utcnow()

        ip.indicator = rr
        try:
            resolve_itype(ip.indicator)
        except:
            continue

        ip.itype = 'ipv4'
        ip.rdata = i.indicator
        ip.confidence = 1
        ip.probability = 0
        rv.append(ip)

        pdns = Indicator(**copy.deepcopy(i.__dict__()))

        # also create a passive dns tag
        pdns.tags = 'pdns'
        pdns.confidence = 4
        pdns.probability = i.probability
        pdns.indicator = ip.indicator
        pdns.rdata = i.indicator
        rv.append(pdns)

    return rv
    def process(self, i, router):
        if i.itype not in ['ipv4', 'ipv6']:
            return

        if 'whitelist' not in i.tags:
            return

        prefix = i.indicator.split('.')
        prefix = prefix[:3]
        prefix.append('0/24')
        prefix = '.'.join(prefix)

        ii = Indicator(**i.__dict__())
        ii.lasttime = arrow.utcnow()

        ii.indicator = prefix
        ii.tags = ['whitelist']
        ii.confidence = (ii.confidence - 2) if ii.confidence >= 2 else 0
        router.indicators_create(ii)
    def process(self, i, router):
        if i.itype not in ['ipv4', 'ipv6']:
            return

        if 'whitelist' not in i.tags:
            return

        prefix = i.indicator.split('.')
        prefix = prefix[:3]
        prefix.append('0/24')
        prefix = '.'.join(prefix)

        ii = Indicator(**i.__dict__())
        ii.lasttime = arrow.utcnow()

        ii.indicator = prefix
        ii.tags = ['whitelist']
        ii.confidence = (ii.confidence - 2) if ii.confidence >= 2 else 0
        router.indicators_create(ii)
Exemple #17
0
def process(i):
    if i.itype not in ['ipv4', 'ipv6']:
        return

    if 'whitelist' not in i.tags:
        return

    prefix = i.indicator.split('.')
    prefix = prefix[:3]
    prefix.append('0/24')
    prefix = '.'.join(prefix)

    ii = Indicator(**i.__dict__())
    ii.probability = 0
    ii.lasttime = arrow.utcnow()

    ii.indicator = prefix
    ii.tags = ['whitelist']
    ii.confidence = 2
    return ii
Exemple #18
0
    def process(self, i, router, **kwargs):
        if 'search' in i.tags:
            return

        if i.itype == 'fqdn' and i.provider != 'spamhaus.org':
            try:
                r = self._resolve(i.indicator)
                try:
                    r = CODES.get(str(r), None)
                except Exception as e:
                    # https://www.spamhaus.org/faq/section/DNSBL%20Usage
                    self.logger.error(e)
                    self.logger.info('check spamhaus return codes')
                    r = None

                if r:
                    confidence = CONFIDENCE
                    if ' legit ' in r['description']:
                        confidence = 6

                    f = Indicator(**i.__dict__())

                    f.tags = [r['tags']]
                    if 'hunter' not in f.tags:
                        f.tags.append('hunter')
                    f.description = r['description']
                    f.confidence = confidence
                    f.provider = PROVIDER
                    f.reference_tlp = 'white'
                    f.reference = 'http://www.spamhaus.org/query/dbl?domain={}'.format(f.indicator)
                    f.lasttime = f.reporttime = arrow.utcnow()
                    x = router.indicators_create(f)
                    self.logger.debug('Spamhaus FQDN: {}'.format(x))
            except KeyError as e:
                self.logger.error(e)
            except Exception as e:
                self.logger.error('[Hunter: SpamhausFqdn] {}: giving up on indicator {}'.format(e, i))
def process(i):
    if not ENABLED:
        return

    if i.itype != 'fqdn':
        return

    if i.probability:
        return

    if not predict(i.indicator):
        return

    fqdn = Indicator(**i.__dict__())
    fqdn.lasttime = arrow.utcnow()
    fqdn.confidence = 4
    fqdn.probability = 84
    fqdn.provider = 'csirtgadgets.com'
    fqdn.reference = 'https://github.com/csirtgadgets/csirtg-domainsml-py' + '#' + VERSION
    tags = set(fqdn.tags)
    tags.add('predicted')
    fqdn.tags = list(tags)

    return fqdn
Exemple #20
0
def main():
    p = get_argument_parser()
    p = ArgumentParser(
        description=textwrap.dedent('''\
            Env Variables:
                CSIRTG_RUNTIME_PATH

            example usage:
                $ csirtg-cef -f /var/log/foo.log
                $ ZYRE_GROUP=honeynet csirtg-cef -d -f /var/log/foo.log --client zyre
                $ csirtg-cef -f /var/log/foo.log --client csirtg --user wes --feed scanners -d
            '''),
        formatter_class=RawDescriptionHelpFormatter,
        prog='csirtg-cef',
        parents=[p],
    )

    p.add_argument('--no-verify-ssl', help='turn TLS/SSL verification OFF', action='store_true')
    p.add_argument('-f', '--file')
    p.add_argument('--client', default='stdout')
    p.add_argument('--user')
    p.add_argument('--feed')
    p.add_argument('--format', default='csv')
    p.add_argument('--tags', help='specify indicator tags [default %(default)s', default='scanner')
    p.add_argument('--provider', help='specify provider [default %(default)s]', default=PROVIDER)
    p.add_argument('--aggregate', help='specify how many seconds to aggregate batches before sending to client '
                                       '[default %(default)s]', default=60)

    p.add_argument('--tail-docker')

    args = p.parse_args()

    # setup logging
    setup_logging(args)

    verify_ssl = True
    if args.no_verify_ssl:
        verify_ssl = False

    if args.file:
        logger.debug('starting on: {}'.format(args.file))
        data_source = tail(args.file)
    elif args.tail_docker:
        logger.debug('starting on container: {}'.format(args.tail_docker))
        #data_source = subprocess.Popen(["docker", "logs", "-f", "--tail", "0", args.tail_docker], bufsize=1, stdout=subprocess.PIPE).stdout
        client = docker.from_env(version='auto')
        container = client.containers.get(args.tail_docker)
        data_source = container.logs(stream=True, follow=True, tail=0)
    else:
        logger.error('Missing --file or --tail-docker flag')
        raise SystemExit

    logger.info('sending data as: %s' % args.provider)

    s = Smrt(client=args.client, username=args.user, feed=args.feed, verify_ssl=verify_ssl)

    bucket = set()
    last_t = round_time(round=int(args.aggregate))
    try:
        for line in data_source:
            i = parse_line(line)

            if not i:
                logger.debug('skipping line')
                continue

            i = Indicator(**i)

            logger.debug(i)

            i.provider = args.provider
            i.tags = args.tags

            if args.aggregate:
                t = round_time(dt=datetime.now(), round=int(args.aggregate))
                if t != last_t:
                    bucket = set()
                
                last_t = t

                if i.indicator in bucket:
                    logger.info('skipping send {}'.format(i.indicator))
                    continue

                bucket.add(i.indicator)

            if args.client == 'stdout':
                print(FORMATS[args.format](data=[i]))
            else:
                try:
                    s.client.indicators_create(i)
                    logger.info('indicator created: {}'.format(i.indicator))
                except Exception as e:
                    logger.error(e)

    except Exception as e:
        logger.error(e)

    except KeyboardInterrupt:
        logger.info('SIGINT caught... stopping')
        if args.client != 'stdout':
            s.client.stop()

    logger.info('exiting...')