def process(i): if not ENABLED: return if i.itype != 'url': return if i.probability: return for t in i.tags: if t == 'predicted': return if not predict(i.indicator): return i = Indicator(**i.__dict__()) i.lasttime = arrow.utcnow() i.confidence = 4 i.probability = 84 i.provider = 'csirtgadgets.com' i.reference = 'https://github.com/csirtgadgets/csirtg-urlsml-py' + '#' + VERSION tags = set(i.tags) tags.add('predicted') i.tags = list(tags) return i
def process(self, i, router): if i.itype == 'fqdn' and i.provider != 'spamhaus.org': try: r = self._resolve(i.indicator) self.logger.debug(r) try: r = CODES[r] except Exception as e: # https://www.spamhaus.org/faq/section/DNSBL%20Usage self.logger.error(e) self.logger.info('check spamhaus return codes') r = None if r: f = Indicator(**i.__dict__) f.tags = [r['tags']] f.description = r['description'] f.confidence = CONFIDENCE f.provider = PROVIDER f.reference_tlp = 'white' f.reference = 'http://www.spamhaus.org/query/dbl?domain={}'.format( f.indicator) x = router.indicators_create(f) self.logger.debug(x) except KeyError as e: self.logger.error(e) except dns.resolver.NoAnswer: self.logger.debug('no answer...') except dns.resolver.NXDOMAIN: self.logger.debug('nxdomain...') except EmptyLabel: self.logger.error('empty label: {}'.format(i.indicator))
def process(i): if not ENABLED: return if i.itype not in ['ipv4', 'ipv6']: return if i.provider == 'spamhaus.org' and not is_ipv4_net(i.indicator): return try: r = _resolve(i.indicator) except Exception as e: return r = CODES.get(str(r), None) if not r: return f = Indicator(**i.__dict__()) f.tags = [r['tags']] f.description = r['description'] f.confidence = CONFIDENCE f.provider = PROVIDER f.reference_tlp = 'white' f.reference = 'http://www.spamhaus.org/query/bl?ip={}'.format(f.indicator) f.lasttime = arrow.utcnow() f.probability = 0 return f
def process(i): if not ENABLED: return if i.itype != 'fqdn': return if i.provider == 'spamhaus.org': return r = _resolve(i.indicator) r = CODES.get(str(r), None) if not r: return confidence = CONFIDENCE if ' legit ' in r['description']: confidence = 1 f = Indicator(**i.__dict__()) f.tags = [r['tags']] f.description = r['description'] f.confidence = confidence f.provider = PROVIDER f.reference_tlp = 'white' f.reference = 'http://www.spamhaus.org/query/dbl?domain={}'.format( f.indicator) f.lasttime = arrow.utcnow() f.probability = 0 return f
def process(self, i, router): if i.itype != 'ipv4' and i.itype != 'ipv6': return if i.provider == 'spamhaus.org' and not is_ipv4_net(i.indicator): return try: r = self._resolve(i.indicator) try: r = CODES.get(str(r), None) except Exception as e: # https://www.spamhaus.org/faq/section/DNSBL%20Usage self.logger.error(e) self.logger.info('check spamhaus return codes') r = None if r: f = Indicator(**i.__dict__()) f.tags = [r['tags']] f.description = r['description'] f.confidence = CONFIDENCE f.provider = PROVIDER f.reference_tlp = 'white' f.reference = 'http://www.spamhaus.org/query/bl?ip={}'.format( f.indicator) x = router.indicators_create(f) except Exception as e: self.logger.error(e) import traceback traceback.print_exc()
def process(self, i, router): if i.itype == 'fqdn' and i.provider != 'spamhaus.org': try: r = self._resolve(i.indicator) try: r = CODES.get(str(r), None) except Exception as e: # https://www.spamhaus.org/faq/section/DNSBL%20Usage self.logger.error(e) self.logger.info('check spamhaus return codes') r = None if r: confidence = CONFIDENCE if ' legit ' in r['description']: confidence = 6 f = Indicator(**i.__dict__()) f.tags = [r['tags']] f.description = r['description'] f.confidence = confidence f.provider = PROVIDER f.reference_tlp = 'white' f.reference = 'http://www.spamhaus.org/query/dbl?domain={}'.format(f.indicator) f.lasttime = arrow.utcnow() x = router.indicators_create(f) self.logger.debug(x) except KeyError as e: self.logger.error(e)
def process(self, i, router): if i.itype == 'fqdn' and i.provider != 'spamhaus.org': try: r = self._resolve(i.indicator) try: r = CODES[r] except Exception as e: # https://www.spamhaus.org/faq/section/DNSBL%20Usage self.logger.error(e) self.logger.info('check spamhaus return codes') r = None if r: f = Indicator(**i.__dict__) f.tags = [r['tags']] f.description = r['description'] f.confidence = CONFIDENCE f.provider = PROVIDER f.reference_tlp = 'white' f.reference = 'http://www.spamhaus.org/query/dbl?domain={}'.format(f.indicator) x = router.indicators_create(f) self.logger.debug(x) except KeyError as e: self.logger.error(e) except dns.resolver.NoAnswer: self.logger.info('no answer...') except dns.resolver.NXDOMAIN: self.logger.info('nxdomain...') except EmptyLabel: self.logger.error('empty label: {}'.format(i.indicator))
def process(self, i, router): if i.itype != 'ipv4' and i.itype != 'ipv6': return if i.provider == 'spamhaus.org' and not is_ipv4_net(i.indicator): return try: r = self._resolve(i.indicator) try: r = CODES.get(str(r), None) except Exception as e: # https://www.spamhaus.org/faq/section/DNSBL%20Usage self.logger.error(e) self.logger.info('check spamhaus return codes') r = None if r: f = Indicator(**i.__dict__()) f.tags = [r['tags']] f.description = r['description'] f.confidence = CONFIDENCE f.provider = PROVIDER f.reference_tlp = 'white' f.reference = 'http://www.spamhaus.org/query/bl?ip={}'.format(f.indicator) f.lasttime = arrow.utcnow() x = router.indicators_create(f) except Exception as e: self.logger.error(e) import traceback traceback.print_exc()
def process(self, i, router): if (i.itype == 'ipv4' or i.itype == 'ipv6') and i.provider != 'spamhaus.org': try: r = self._resolve(i.indicator) try: r = CODES.get(str(r), None) except Exception as e: # https://www.spamhaus.org/faq/section/DNSBL%20Usage self.logger.error(e) self.logger.info('check spamhaus return codes') r = None if r: f = Indicator(**i.__dict__) f.tags = [r['tags']] f.description = r['description'] f.confidence = CONFIDENCE f.provider = PROVIDER f.reference_tlp = 'white' f.reference = 'http://www.spamhaus.org/query/bl?ip={}'.format(f.indicator) x = router.indicators_create(f) self.logger.debug(x) except dns.resolver.NoAnswer: self.logger.info('no answer...') except dns.resolver.NXDOMAIN: self.logger.info('nxdomain...') except Exception as e: self.logger.error(e) import traceback traceback.print_exc()
def process(self, i, router): if (i.itype == 'ipv4' or i.itype == 'ipv6') and i.provider != 'spamhaus.org': try: r = self._resolve(i.indicator) try: r = CODES.get(str(r), None) except Exception as e: # https://www.spamhaus.org/faq/section/DNSBL%20Usage self.logger.error(e) self.logger.info('check spamhaus return codes') r = None if r: f = Indicator(**i.__dict__) f.tags = [r['tags']] f.description = r['description'] f.confidence = CONFIDENCE f.provider = PROVIDER f.reference_tlp = 'white' f.reference = 'http://www.spamhaus.org/query/bl?ip={}'.format( f.indicator) x = router.indicators_create(f) self.logger.debug(x) except dns.resolver.NoAnswer: self.logger.info('no answer...') except dns.resolver.NXDOMAIN: self.logger.info('nxdomain...') except Exception as e: self.logger.error(e) import traceback traceback.print_exc()
def process(self, i, router, **kwargs): if 'search' in i.tags: return if i.itype != 'ipv4' and i.itype != 'ipv6': return if i.provider == 'spamhaus.org' and not is_ipv4_net(i.indicator): return try: r = self._resolve(i.indicator) try: r = CODES.get(str(r), None) except Exception as e: # https://www.spamhaus.org/faq/section/DNSBL%20Usage self.logger.error(e) self.logger.info('check spamhaus return codes') r = None if r: f = Indicator(**i.__dict__()) f.tags = [r['tags']] if 'hunter' not in f.tags: f.tags.append('hunter') f.description = r['description'] f.confidence = CONFIDENCE f.provider = PROVIDER f.reference_tlp = 'white' f.reference = 'http://www.spamhaus.org/query/bl?ip={}'.format( f.indicator) f.lasttime = f.reporttime = arrow.utcnow() x = router.indicators_create(f) self.logger.debug("Spamhaus IP: {}".format(x)) except Exception as e: self.logger.error( "[Hunter: SpamhausIp] {}: giving up on indicator {}".format( e, i)) import traceback traceback.print_exc()
def process(self, i, router, **kwargs): if 'search' in i.tags: return if i.itype == 'fqdn' and i.provider != 'spamhaus.org': try: r = self._resolve(i.indicator) try: r = CODES.get(str(r), None) except Exception as e: # https://www.spamhaus.org/faq/section/DNSBL%20Usage self.logger.error(e) self.logger.info('check spamhaus return codes') r = None if r: confidence = CONFIDENCE if ' legit ' in r['description']: confidence = 6 f = Indicator(**i.__dict__()) f.tags = [r['tags']] if 'hunter' not in f.tags: f.tags.append('hunter') f.description = r['description'] f.confidence = confidence f.provider = PROVIDER f.reference_tlp = 'white' f.reference = 'http://www.spamhaus.org/query/dbl?domain={}'.format(f.indicator) f.lasttime = f.reporttime = arrow.utcnow() x = router.indicators_create(f) self.logger.debug('Spamhaus FQDN: {}'.format(x)) except KeyError as e: self.logger.error(e) except Exception as e: self.logger.error('[Hunter: SpamhausFqdn] {}: giving up on indicator {}'.format(e, i))
def process(i): if not ENABLED: return if i.itype != 'fqdn': return if i.probability: return if not predict(i.indicator): return fqdn = Indicator(**i.__dict__()) fqdn.lasttime = arrow.utcnow() fqdn.confidence = 4 fqdn.probability = 84 fqdn.provider = 'csirtgadgets.com' fqdn.reference = 'https://github.com/csirtgadgets/csirtg-domainsml-py' + '#' + VERSION tags = set(fqdn.tags) tags.add('predicted') fqdn.tags = list(tags) return fqdn
def main(): p = get_argument_parser() p = ArgumentParser( description=textwrap.dedent('''\ Env Variables: CSIRTG_RUNTIME_PATH example usage: $ csirtg-ufw -f /var/log/ufw.log $ ZYRE_GROUP=honeynet csirtg-ufw -d -f /var/log/ufw.log --client zyre $ csirtg-ufw -f /var/log/ufw.log --client csirtg --user wes --feed scanners -d '''), formatter_class=RawDescriptionHelpFormatter, prog='csirtg-ufw', parents=[p], ) p.add_argument('--no-verify-ssl', help='turn TLS/SSL verification OFF', action='store_true') p.add_argument('-f', '--file', default=FILENAME) p.add_argument('--client', default='stdout') p.add_argument('--user') p.add_argument('--feed') p.add_argument('--format', default='csv') p.add_argument('--provider', help='specify provider [default %(default)s]', default=PROVIDER) p.add_argument('--ignore-client-errors', help='skip when client errors out (eg: HTTP 5XX, etc)', action='store_true') p.add_argument('--aggregate', help='specify how many seconds to aggregate batches before sending to client ' '[default %(default)s]', default=60) args = p.parse_args() if not args.provider: raise RuntimeError('Missing --provider flag') # setup logging setup_logging(args) logger.debug('starting on: {}'.format(args.file)) verify_ssl = True if args.no_verify_ssl: verify_ssl = False from csirtg_smrt import Smrt s = Smrt(client=args.client, username=args.user, feed=args.feed, verify_ssl=verify_ssl) bucket = set() last_t = round_time(round=int(args.aggregate)) try: for line in tail(args.file): if 'csirtg-ufw' in line: continue if '[UFW BLOCK]' not in line: continue if ' SYN ' not in line: continue logger.debug(line) try: i = parse_line(line) except AttributeError: logger.debug("line not matched: \n{}".format(line)) continue i = Indicator(**i) i.provider = args.provider u_indicator = ':'.join([i.indicator,'/'.join([i.portlist,i.protocol])]) if args.aggregate: t = round_time(dt=datetime.now(), round=int(args.aggregate)) if t != last_t: bucket = set() last_t = t if u_indicator in bucket: logger.info('skipping send {}'.format(u_indicator)) continue bucket.add(u_indicator) if args.client == 'stdout': print(FORMATS[args.format](data=[i])) continue try: s.client.indicators_create(i) logger.info('indicator created: {}'.format(u_indicator)) except Exception as e: logger.error(e) if args.ignore_client_errors: pass except KeyboardInterrupt: logger.info('SIGINT caught... stopping') if args.client != 'stdout': try: s.client.stop() except AttributeError: pass logger.info('exiting...')
def main(): p = get_argument_parser() p = ArgumentParser( description=textwrap.dedent('''\ Env Variables: CSIRTG_RUNTIME_PATH example usage: $ csirtg-cef -f /var/log/foo.log $ ZYRE_GROUP=honeynet csirtg-cef -d -f /var/log/foo.log --client zyre $ csirtg-cef -f /var/log/foo.log --client csirtg --user wes --feed scanners -d '''), formatter_class=RawDescriptionHelpFormatter, prog='csirtg-cef', parents=[p], ) p.add_argument('--no-verify-ssl', help='turn TLS/SSL verification OFF', action='store_true') p.add_argument('-f', '--file') p.add_argument('--client', default='stdout') p.add_argument('--user') p.add_argument('--feed') p.add_argument('--format', default='csv') p.add_argument('--tags', help='specify indicator tags [default %(default)s', default='scanner') p.add_argument('--provider', help='specify provider [default %(default)s]', default=PROVIDER) p.add_argument('--tail-docker') args = p.parse_args() if not args.provider: raise RuntimeError('Missing --provider flag') if not args.file: raise RuntimeError('Missing --file flag') # setup logging setup_logging(args) logger.debug('starting on: {}'.format(args.file)) verify_ssl = True if args.no_verify_ssl: verify_ssl = False f = open(args.file) from csirtg_smrt import Smrt s = Smrt(client=args.client, username=args.user, feed=args.feed, verify_ssl=verify_ssl) try: for line in tailer.follow(f): i = parse_line(line) if not i: logger.debug('skipping line') continue i = Indicator(**i) logger.debug(i) i.provider = args.provider i.tags = args.tags if args.client == 'stdout': print(FORMATS[args.format](data=[i])) else: s.client.indicators_create(i) logger.info('indicator created: {}'.format(i.indicator)) except KeyboardInterrupt: logger.info('SIGINT caught... stopping') if args.client != 'stdout': s.client.stop() logger.info('exiting...')
def main(): p = get_argument_parser() p = ArgumentParser( description=textwrap.dedent('''\ Env Variables: CSIRTG_RUNTIME_PATH example usage: $ csirtg-cef -f /var/log/foo.log $ ZYRE_GROUP=honeynet csirtg-cef -d -f /var/log/foo.log --client zyre $ csirtg-cef -f /var/log/foo.log --client csirtg --user wes --feed scanners -d '''), formatter_class=RawDescriptionHelpFormatter, prog='csirtg-cef', parents=[p], ) p.add_argument('--no-verify-ssl', help='turn TLS/SSL verification OFF', action='store_true') p.add_argument('-f', '--file') p.add_argument('--client', default='stdout') p.add_argument('--user') p.add_argument('--feed') p.add_argument('--format', default='csv') p.add_argument('--tags', help='specify indicator tags [default %(default)s', default='scanner') p.add_argument('--provider', help='specify provider [default %(default)s]', default=PROVIDER) p.add_argument('--aggregate', help='specify how many seconds to aggregate batches before sending to client ' '[default %(default)s]', default=60) p.add_argument('--tail-docker') args = p.parse_args() # setup logging setup_logging(args) verify_ssl = True if args.no_verify_ssl: verify_ssl = False if args.file: logger.debug('starting on: {}'.format(args.file)) data_source = tail(args.file) elif args.tail_docker: logger.debug('starting on container: {}'.format(args.tail_docker)) #data_source = subprocess.Popen(["docker", "logs", "-f", "--tail", "0", args.tail_docker], bufsize=1, stdout=subprocess.PIPE).stdout client = docker.from_env(version='auto') container = client.containers.get(args.tail_docker) data_source = container.logs(stream=True, follow=True, tail=0) else: logger.error('Missing --file or --tail-docker flag') raise SystemExit logger.info('sending data as: %s' % args.provider) s = Smrt(client=args.client, username=args.user, feed=args.feed, verify_ssl=verify_ssl) bucket = set() last_t = round_time(round=int(args.aggregate)) try: for line in data_source: i = parse_line(line) if not i: logger.debug('skipping line') continue i = Indicator(**i) logger.debug(i) i.provider = args.provider i.tags = args.tags if args.aggregate: t = round_time(dt=datetime.now(), round=int(args.aggregate)) if t != last_t: bucket = set() last_t = t if i.indicator in bucket: logger.info('skipping send {}'.format(i.indicator)) continue bucket.add(i.indicator) if args.client == 'stdout': print(FORMATS[args.format](data=[i])) else: try: s.client.indicators_create(i) logger.info('indicator created: {}'.format(i.indicator)) except Exception as e: logger.error(e) except Exception as e: logger.error(e) except KeyboardInterrupt: logger.info('SIGINT caught... stopping') if args.client != 'stdout': s.client.stop() logger.info('exiting...')