def process(self, i, router): if i.itype == 'fqdn' and i.provider != 'spamhaus.org': try: r = self._resolve(i.indicator) try: r = CODES[r] except Exception as e: # https://www.spamhaus.org/faq/section/DNSBL%20Usage self.logger.error(e) self.logger.info('check spamhaus return codes') r = None if r: f = Indicator(**i.__dict__) f.tags = [r['tags']] f.description = r['description'] f.confidence = CONFIDENCE f.provider = PROVIDER f.reference_tlp = 'white' f.reference = 'http://www.spamhaus.org/query/dbl?domain={}'.format(f.indicator) x = router.indicators_create(f) self.logger.debug(x) except KeyError as e: self.logger.error(e) except dns.resolver.NoAnswer: self.logger.info('no answer...') except dns.resolver.NXDOMAIN: self.logger.info('nxdomain...') except EmptyLabel: self.logger.error('empty label: {}'.format(i.indicator))
def process(self, i, router, **kwargs): if i.itype != 'fqdn': return if 'whitelist' not in i.tags: return urls = [] for p in ['http://', 'https://']: urls.append('{}{}'.format(p, i.indicator)) if not i.indicator.startswith('www.'): urls.append('{}www.{}'.format(p, i.indicator)) for u in urls: url = Indicator(**i.__dict__()) url.indicator = u try: resolve_itype(url.indicator) except InvalidIndicator as e: self.logger.error(url) self.logger.error(e) else: url.tags = ['whitelist', 'hunter'] url.itype = 'url' url.rdata = i.indicator url.lasttime = url.reporttime = arrow.utcnow() router.indicators_create(url)
def process(self, i, router): if i.itype != 'ipv4' and i.itype != 'ipv6': return if i.provider == 'spamhaus.org' and not is_ipv4_net(i.indicator): return try: r = self._resolve(i.indicator) try: r = CODES.get(str(r), None) except Exception as e: # https://www.spamhaus.org/faq/section/DNSBL%20Usage self.logger.error(e) self.logger.info('check spamhaus return codes') r = None if r: f = Indicator(**i.__dict__()) f.tags = [r['tags']] f.description = r['description'] f.confidence = CONFIDENCE f.provider = PROVIDER f.reference_tlp = 'white' f.reference = 'http://www.spamhaus.org/query/bl?ip={}'.format(f.indicator) f.lasttime = arrow.utcnow() x = router.indicators_create(f) except Exception as e: self.logger.error(e) import traceback traceback.print_exc()
def process(self, i, router): if i.itype != 'ipv4': return if 'whitelist' not in i.tags: return # only run this hunter if it's a single address (no CIDRs) if ipaddress.IPv4Network(i.indicator).prefixlen != 32: return prefix = i.indicator.split('.') prefix = prefix[:3] prefix.append('0/24') prefix = '.'.join(prefix) try: ii = Indicator(**i.__dict__()) except InvalidIndicator as e: self.logger.error(e) return ii.lasttime = ii.reporttime = arrow.utcnow() ii.indicator = prefix ii.tags = ['whitelist', 'hunter'] ii.confidence = (ii.confidence - 2) if ii.confidence >= 2 else 0 router.indicators_create(ii)
def process(self, i, router): if i.itype != 'ipv4' and i.itype != 'ipv6': return if i.provider == 'spamhaus.org' and not is_ipv4_net(i.indicator): return try: r = self._resolve(i.indicator) try: r = CODES.get(str(r), None) except Exception as e: # https://www.spamhaus.org/faq/section/DNSBL%20Usage self.logger.error(e) self.logger.info('check spamhaus return codes') r = None if r: f = Indicator(**i.__dict__()) f.tags = [r['tags']] f.description = r['description'] f.confidence = CONFIDENCE f.provider = PROVIDER f.reference_tlp = 'white' f.reference = 'http://www.spamhaus.org/query/bl?ip={}'.format( f.indicator) x = router.indicators_create(f) except Exception as e: self.logger.error(e) import traceback traceback.print_exc()
def process(self, i, router): if i.itype == 'fqdn' and i.provider != 'spamhaus.org': try: r = self._resolve(i.indicator) try: r = CODES.get(str(r), None) except Exception as e: # https://www.spamhaus.org/faq/section/DNSBL%20Usage self.logger.error(e) self.logger.info('check spamhaus return codes') r = None if r: confidence = CONFIDENCE if ' legit ' in r['description']: confidence = 6 f = Indicator(**i.__dict__()) f.tags = [r['tags']] f.description = r['description'] f.confidence = confidence f.provider = PROVIDER f.reference_tlp = 'white' f.reference = 'http://www.spamhaus.org/query/dbl?domain={}'.format(f.indicator) f.lasttime = arrow.utcnow() x = router.indicators_create(f) self.logger.debug(x) except KeyError as e: self.logger.error(e)
def process(i): if not ENABLED: return if i.itype != 'fqdn': return if i.provider == 'spamhaus.org': return r = _resolve(i.indicator) r = CODES.get(str(r), None) if not r: return confidence = CONFIDENCE if ' legit ' in r['description']: confidence = 1 f = Indicator(**i.__dict__()) f.tags = [r['tags']] f.description = r['description'] f.confidence = confidence f.provider = PROVIDER f.reference_tlp = 'white' f.reference = 'http://www.spamhaus.org/query/dbl?domain={}'.format( f.indicator) f.lasttime = arrow.utcnow() f.probability = 0 return f
def process(self, i, router): if i.itype != 'ipv4': return if 'whitelist' not in i.tags: return if i.indicator.endswith('/24'): return prefix = i.indicator.split('.') prefix = prefix[:3] prefix.append('0/24') prefix = '.'.join(prefix) try: ii = Indicator(**i.__dict__()) except InvalidIndicator as e: self.logger.error(e) return ii.lasttime = arrow.utcnow() ii.indicator = prefix ii.tags = ['whitelist'] ii.confidence = (ii.confidence - 2) if ii.confidence >= 2 else 0 router.indicators_create(ii)
def process(self, i, router): if (i.itype == 'ipv4' or i.itype == 'ipv6') and i.provider != 'spamhaus.org': try: r = self._resolve(i.indicator) try: r = CODES.get(str(r), None) except Exception as e: # https://www.spamhaus.org/faq/section/DNSBL%20Usage self.logger.error(e) self.logger.info('check spamhaus return codes') r = None if r: f = Indicator(**i.__dict__) f.tags = [r['tags']] f.description = r['description'] f.confidence = CONFIDENCE f.provider = PROVIDER f.reference_tlp = 'white' f.reference = 'http://www.spamhaus.org/query/bl?ip={}'.format(f.indicator) x = router.indicators_create(f) self.logger.debug(x) except dns.resolver.NoAnswer: self.logger.info('no answer...') except dns.resolver.NXDOMAIN: self.logger.info('nxdomain...') except Exception as e: self.logger.error(e) import traceback traceback.print_exc()
def process(self, i, router): if (i.itype == 'ipv4' or i.itype == 'ipv6') and i.provider != 'spamhaus.org': try: r = self._resolve(i.indicator) try: r = CODES.get(str(r), None) except Exception as e: # https://www.spamhaus.org/faq/section/DNSBL%20Usage self.logger.error(e) self.logger.info('check spamhaus return codes') r = None if r: f = Indicator(**i.__dict__) f.tags = [r['tags']] f.description = r['description'] f.confidence = CONFIDENCE f.provider = PROVIDER f.reference_tlp = 'white' f.reference = 'http://www.spamhaus.org/query/bl?ip={}'.format( f.indicator) x = router.indicators_create(f) self.logger.debug(x) except dns.resolver.NoAnswer: self.logger.info('no answer...') except dns.resolver.NXDOMAIN: self.logger.info('nxdomain...') except Exception as e: self.logger.error(e) import traceback traceback.print_exc()
def _get_indicator(i): i2 = Indicator() timestamps = [] ports = [] # prioritize the various elements.. for e in i: if i[e] == 'CC': i2.cc = e continue if i[e] == 'indicator': if i2.indicator: i2.reference = e else: i2.indicator = e continue if i[e] == 'timestamp': timestamps.append(get_ts(e)) continue if i[e] == 'float': i2.asn = e continue if i[e] == 'int': ports.append(e) continue if i[e] == 'description': i2.description = e continue if i[e] == 'string': if re.match(r'[0-9A-Za-z\.\s\/]+', e) and i2.asn: i2.asn_desc = e continue if 4 <= len(e) <= 10 and re.match('[a-z-A-Z]+,?', e) \ and e not in ['ipv4', 'fqdn', 'url', 'ipv6']: i2.tags = [e] continue if ' ' in e and 5 <= len(e) and not i2.asn_desc: i2.description = e continue _calc_timestamps(i2, timestamps) _calc_ports(i2, ports) return i2
def text_to_list(text, known_only=True): separator = find_seperator(text) t_tokens = top_tokens(text) top = set() for t in range(0, 9): top.add(t_tokens[t]) if known_only: if separator not in KNOWN_SEPERATORS: pprint(top) raise SystemError('separator not in known list: {}'.format(separator)) ret = [] for l in text.split("\n"): if l == '': continue if l.startswith('#') or l.startswith(';'): continue cols = l.split(separator) cols = [x.strip() for x in cols] indicator = Indicator() for e in cols: if e: try: i = resolve_itype(e) if i: indicator.indicator = e indicator.itype = i except NotImplementedError: pass try: ts = arrow.get(e) if ts: indicator.lasttime = ts.datetime except (arrow.parser.ParserError, UnicodeDecodeError): pass if e in top: indicator.tags = [e] if indicator.itype and indicator.indicator: ret.append(indicator) return ret
def text_to_list(text, known_only=True): separator = find_seperator(text) t_tokens = top_tokens(text) top = set() for t in range(0, 9): top.add(t_tokens[t]) if known_only: if separator not in KNOWN_SEPERATORS: raise SystemError( 'separator not in known list: {}'.format(separator)) ret = [] for l in text.split("\n"): if l == '': continue if l.startswith('#') or l.startswith(';'): continue cols = l.split(separator) cols = [x.strip() for x in cols] indicator = Indicator() for e in cols: if e: try: i = resolve_itype(e) if i: indicator.indicator = e indicator.itype = i except TypeError: pass try: ts = arrow.get(e) if ts: indicator.lasttime = ts.datetime except (arrow.parser.ParserError, UnicodeDecodeError): pass if e in top: indicator.tags = [e] if indicator.itype and indicator.indicator: ret.append(indicator) return ret
def process(i): if not ENABLED: return if i.itype != 'fqdn': return try: r = resolve_ns(i.indicator) if not r: return except Timeout: return rv = [] for rr in r: rr = str(rr) if rr in ["", 'localhost']: continue ip = Indicator(**i.__dict__()) ip.lasttime = arrow.utcnow() ip.indicator = rr try: resolve_itype(ip.indicator) except: continue ip.itype = 'ipv4' ip.rdata = i.indicator ip.confidence = 1 ip.probability = 0 rv.append(ip) pdns = Indicator(**copy.deepcopy(i.__dict__())) # also create a passive dns tag pdns.tags = 'pdns' pdns.confidence = 4 pdns.probability = i.probability pdns.indicator = ip.indicator pdns.rdata = i.indicator rv.append(pdns) return rv
def process(self, i, router): if i.itype not in ['ipv4', 'ipv6']: return if 'whitelist' not in i.tags: return prefix = i.indicator.split('.') prefix = prefix[:3] prefix.append('0/24') prefix = '.'.join(prefix) ii = Indicator(**i.__dict__()) ii.lasttime = arrow.utcnow() ii.indicator = prefix ii.tags = ['whitelist'] ii.confidence = (ii.confidence - 2) if ii.confidence >= 2 else 0 router.indicators_create(ii)
def process(i): if i.itype not in ['ipv4', 'ipv6']: return if 'whitelist' not in i.tags: return prefix = i.indicator.split('.') prefix = prefix[:3] prefix.append('0/24') prefix = '.'.join(prefix) ii = Indicator(**i.__dict__()) ii.probability = 0 ii.lasttime = arrow.utcnow() ii.indicator = prefix ii.tags = ['whitelist'] ii.confidence = 2 return ii
def process(self, i, router, **kwargs): if 'search' in i.tags: return if i.itype == 'fqdn' and i.provider != 'spamhaus.org': try: r = self._resolve(i.indicator) try: r = CODES.get(str(r), None) except Exception as e: # https://www.spamhaus.org/faq/section/DNSBL%20Usage self.logger.error(e) self.logger.info('check spamhaus return codes') r = None if r: confidence = CONFIDENCE if ' legit ' in r['description']: confidence = 6 f = Indicator(**i.__dict__()) f.tags = [r['tags']] if 'hunter' not in f.tags: f.tags.append('hunter') f.description = r['description'] f.confidence = confidence f.provider = PROVIDER f.reference_tlp = 'white' f.reference = 'http://www.spamhaus.org/query/dbl?domain={}'.format(f.indicator) f.lasttime = f.reporttime = arrow.utcnow() x = router.indicators_create(f) self.logger.debug('Spamhaus FQDN: {}'.format(x)) except KeyError as e: self.logger.error(e) except Exception as e: self.logger.error('[Hunter: SpamhausFqdn] {}: giving up on indicator {}'.format(e, i))
def process(i): if not ENABLED: return if i.itype != 'fqdn': return if i.probability: return if not predict(i.indicator): return fqdn = Indicator(**i.__dict__()) fqdn.lasttime = arrow.utcnow() fqdn.confidence = 4 fqdn.probability = 84 fqdn.provider = 'csirtgadgets.com' fqdn.reference = 'https://github.com/csirtgadgets/csirtg-domainsml-py' + '#' + VERSION tags = set(fqdn.tags) tags.add('predicted') fqdn.tags = list(tags) return fqdn
def main(): p = get_argument_parser() p = ArgumentParser( description=textwrap.dedent('''\ Env Variables: CSIRTG_RUNTIME_PATH example usage: $ csirtg-cef -f /var/log/foo.log $ ZYRE_GROUP=honeynet csirtg-cef -d -f /var/log/foo.log --client zyre $ csirtg-cef -f /var/log/foo.log --client csirtg --user wes --feed scanners -d '''), formatter_class=RawDescriptionHelpFormatter, prog='csirtg-cef', parents=[p], ) p.add_argument('--no-verify-ssl', help='turn TLS/SSL verification OFF', action='store_true') p.add_argument('-f', '--file') p.add_argument('--client', default='stdout') p.add_argument('--user') p.add_argument('--feed') p.add_argument('--format', default='csv') p.add_argument('--tags', help='specify indicator tags [default %(default)s', default='scanner') p.add_argument('--provider', help='specify provider [default %(default)s]', default=PROVIDER) p.add_argument('--aggregate', help='specify how many seconds to aggregate batches before sending to client ' '[default %(default)s]', default=60) p.add_argument('--tail-docker') args = p.parse_args() # setup logging setup_logging(args) verify_ssl = True if args.no_verify_ssl: verify_ssl = False if args.file: logger.debug('starting on: {}'.format(args.file)) data_source = tail(args.file) elif args.tail_docker: logger.debug('starting on container: {}'.format(args.tail_docker)) #data_source = subprocess.Popen(["docker", "logs", "-f", "--tail", "0", args.tail_docker], bufsize=1, stdout=subprocess.PIPE).stdout client = docker.from_env(version='auto') container = client.containers.get(args.tail_docker) data_source = container.logs(stream=True, follow=True, tail=0) else: logger.error('Missing --file or --tail-docker flag') raise SystemExit logger.info('sending data as: %s' % args.provider) s = Smrt(client=args.client, username=args.user, feed=args.feed, verify_ssl=verify_ssl) bucket = set() last_t = round_time(round=int(args.aggregate)) try: for line in data_source: i = parse_line(line) if not i: logger.debug('skipping line') continue i = Indicator(**i) logger.debug(i) i.provider = args.provider i.tags = args.tags if args.aggregate: t = round_time(dt=datetime.now(), round=int(args.aggregate)) if t != last_t: bucket = set() last_t = t if i.indicator in bucket: logger.info('skipping send {}'.format(i.indicator)) continue bucket.add(i.indicator) if args.client == 'stdout': print(FORMATS[args.format](data=[i])) else: try: s.client.indicators_create(i) logger.info('indicator created: {}'.format(i.indicator)) except Exception as e: logger.error(e) except Exception as e: logger.error(e) except KeyboardInterrupt: logger.info('SIGINT caught... stopping') if args.client != 'stdout': s.client.stop() logger.info('exiting...')