def process(i):
if not ENABLED:
return
if i.itype != 'url':
return
if i.probability:
return
for t in i.tags:
if t == 'predicted':
return
if not predict(i.indicator):
return
i = Indicator(**i.__dict__())
i.lasttime = arrow.utcnow()
i.confidence = 4
i.probability = 84
i.provider = 'csirtgadgets.com'
i.reference = 'https://github.com/csirtgadgets/csirtg-urlsml-py' + '#' + VERSION
tags = set(i.tags)
tags.add('predicted')
i.tags = list(tags)
return i
def process(self, i, router):
if i.itype == 'fqdn' and i.provider != 'spamhaus.org':
try:
r = self._resolve(i.indicator)
self.logger.debug(r)
try:
r = CODES[r]
except Exception as e:
# https://www.spamhaus.org/faq/section/DNSBL%20Usage
self.logger.error(e)
self.logger.info('check spamhaus return codes')
r = None
if r:
f = Indicator(**i.__dict__)
f.tags = [r['tags']]
f.description = r['description']
f.confidence = CONFIDENCE
f.provider = PROVIDER
f.reference_tlp = 'white'
f.reference = 'http://www.spamhaus.org/query/dbl?domain={}'.format(
f.indicator)
x = router.indicators_create(f)
self.logger.debug(x)
except KeyError as e:
self.logger.error(e)
except dns.resolver.NoAnswer:
self.logger.debug('no answer...')
except dns.resolver.NXDOMAIN:
self.logger.debug('nxdomain...')
except EmptyLabel:
self.logger.error('empty label: {}'.format(i.indicator))
def process(i):
if not ENABLED:
return
if i.itype not in ['ipv4', 'ipv6']:
return
if i.provider == 'spamhaus.org' and not is_ipv4_net(i.indicator):
return
try:
r = _resolve(i.indicator)
except Exception as e:
return
r = CODES.get(str(r), None)
if not r:
return
f = Indicator(**i.__dict__())
f.tags = [r['tags']]
f.description = r['description']
f.confidence = CONFIDENCE
f.provider = PROVIDER
f.reference_tlp = 'white'
f.reference = 'http://www.spamhaus.org/query/bl?ip={}'.format(f.indicator)
f.lasttime = arrow.utcnow()
f.probability = 0
return f
def process(i):
if not ENABLED:
return
if i.itype != 'fqdn':
return
if i.provider == 'spamhaus.org':
return
r = _resolve(i.indicator)
r = CODES.get(str(r), None)
if not r:
return
confidence = CONFIDENCE
if ' legit ' in r['description']:
confidence = 1
f = Indicator(**i.__dict__())
f.tags = [r['tags']]
f.description = r['description']
f.confidence = confidence
f.provider = PROVIDER
f.reference_tlp = 'white'
f.reference = 'http://www.spamhaus.org/query/dbl?domain={}'.format(
f.indicator)
f.lasttime = arrow.utcnow()
f.probability = 0
return f
def process(self, i, router):
if i.itype != 'ipv4' and i.itype != 'ipv6':
return
if i.provider == 'spamhaus.org' and not is_ipv4_net(i.indicator):
return
try:
r = self._resolve(i.indicator)
try:
r = CODES.get(str(r), None)
except Exception as e:
# https://www.spamhaus.org/faq/section/DNSBL%20Usage
self.logger.error(e)
self.logger.info('check spamhaus return codes')
r = None
if r:
f = Indicator(**i.__dict__())
f.tags = [r['tags']]
f.description = r['description']
f.confidence = CONFIDENCE
f.provider = PROVIDER
f.reference_tlp = 'white'
f.reference = 'http://www.spamhaus.org/query/bl?ip={}'.format(
f.indicator)
x = router.indicators_create(f)
except Exception as e:
self.logger.error(e)
import traceback
traceback.print_exc()
def process(self, i, router):
if i.itype == 'fqdn' and i.provider != 'spamhaus.org':
try:
r = self._resolve(i.indicator)
try:
r = CODES.get(str(r), None)
except Exception as e:
# https://www.spamhaus.org/faq/section/DNSBL%20Usage
self.logger.error(e)
self.logger.info('check spamhaus return codes')
r = None
if r:
confidence = CONFIDENCE
if ' legit ' in r['description']:
confidence = 6
f = Indicator(**i.__dict__())
f.tags = [r['tags']]
f.description = r['description']
f.confidence = confidence
f.provider = PROVIDER
f.reference_tlp = 'white'
f.reference = 'http://www.spamhaus.org/query/dbl?domain={}'.format(f.indicator)
f.lasttime = arrow.utcnow()
x = router.indicators_create(f)
self.logger.debug(x)
except KeyError as e:
self.logger.error(e)
def process(self, i, router):
if i.itype == 'fqdn' and i.provider != 'spamhaus.org':
try:
r = self._resolve(i.indicator)
try:
r = CODES[r]
except Exception as e:
# https://www.spamhaus.org/faq/section/DNSBL%20Usage
self.logger.error(e)
self.logger.info('check spamhaus return codes')
r = None
if r:
f = Indicator(**i.__dict__)
f.tags = [r['tags']]
f.description = r['description']
f.confidence = CONFIDENCE
f.provider = PROVIDER
f.reference_tlp = 'white'
f.reference = 'http://www.spamhaus.org/query/dbl?domain={}'.format(f.indicator)
x = router.indicators_create(f)
self.logger.debug(x)
except KeyError as e:
self.logger.error(e)
except dns.resolver.NoAnswer:
self.logger.info('no answer...')
except dns.resolver.NXDOMAIN:
self.logger.info('nxdomain...')
except EmptyLabel:
self.logger.error('empty label: {}'.format(i.indicator))
def process(self, i, router):
if i.itype != 'ipv4' and i.itype != 'ipv6':
return
if i.provider == 'spamhaus.org' and not is_ipv4_net(i.indicator):
return
try:
r = self._resolve(i.indicator)
try:
r = CODES.get(str(r), None)
except Exception as e:
# https://www.spamhaus.org/faq/section/DNSBL%20Usage
self.logger.error(e)
self.logger.info('check spamhaus return codes')
r = None
if r:
f = Indicator(**i.__dict__())
f.tags = [r['tags']]
f.description = r['description']
f.confidence = CONFIDENCE
f.provider = PROVIDER
f.reference_tlp = 'white'
f.reference = 'http://www.spamhaus.org/query/bl?ip={}'.format(f.indicator)
f.lasttime = arrow.utcnow()
x = router.indicators_create(f)
except Exception as e:
self.logger.error(e)
import traceback
traceback.print_exc()
def process(self, i, router):
if (i.itype == 'ipv4' or i.itype == 'ipv6') and i.provider != 'spamhaus.org':
try:
r = self._resolve(i.indicator)
try:
r = CODES.get(str(r), None)
except Exception as e:
# https://www.spamhaus.org/faq/section/DNSBL%20Usage
self.logger.error(e)
self.logger.info('check spamhaus return codes')
r = None
if r:
f = Indicator(**i.__dict__)
f.tags = [r['tags']]
f.description = r['description']
f.confidence = CONFIDENCE
f.provider = PROVIDER
f.reference_tlp = 'white'
f.reference = 'http://www.spamhaus.org/query/bl?ip={}'.format(f.indicator)
x = router.indicators_create(f)
self.logger.debug(x)
except dns.resolver.NoAnswer:
self.logger.info('no answer...')
except dns.resolver.NXDOMAIN:
self.logger.info('nxdomain...')
except Exception as e:
self.logger.error(e)
import traceback
traceback.print_exc()
def process(self, i, router):
if (i.itype == 'ipv4'
or i.itype == 'ipv6') and i.provider != 'spamhaus.org':
try:
r = self._resolve(i.indicator)
try:
r = CODES.get(str(r), None)
except Exception as e:
# https://www.spamhaus.org/faq/section/DNSBL%20Usage
self.logger.error(e)
self.logger.info('check spamhaus return codes')
r = None
if r:
f = Indicator(**i.__dict__)
f.tags = [r['tags']]
f.description = r['description']
f.confidence = CONFIDENCE
f.provider = PROVIDER
f.reference_tlp = 'white'
f.reference = 'http://www.spamhaus.org/query/bl?ip={}'.format(
f.indicator)
x = router.indicators_create(f)
self.logger.debug(x)
except dns.resolver.NoAnswer:
self.logger.info('no answer...')
except dns.resolver.NXDOMAIN:
self.logger.info('nxdomain...')
except Exception as e:
self.logger.error(e)
import traceback
traceback.print_exc()
def process(self, i, router, **kwargs):
if 'search' in i.tags:
return
if i.itype != 'ipv4' and i.itype != 'ipv6':
return
if i.provider == 'spamhaus.org' and not is_ipv4_net(i.indicator):
return
try:
r = self._resolve(i.indicator)
try:
r = CODES.get(str(r), None)
except Exception as e:
# https://www.spamhaus.org/faq/section/DNSBL%20Usage
self.logger.error(e)
self.logger.info('check spamhaus return codes')
r = None
if r:
f = Indicator(**i.__dict__())
f.tags = [r['tags']]
if 'hunter' not in f.tags:
f.tags.append('hunter')
f.description = r['description']
f.confidence = CONFIDENCE
f.provider = PROVIDER
f.reference_tlp = 'white'
f.reference = 'http://www.spamhaus.org/query/bl?ip={}'.format(
f.indicator)
f.lasttime = f.reporttime = arrow.utcnow()
x = router.indicators_create(f)
self.logger.debug("Spamhaus IP: {}".format(x))
except Exception as e:
self.logger.error(
"[Hunter: SpamhausIp] {}: giving up on indicator {}".format(
e, i))
import traceback
traceback.print_exc()
def process(self, i, router, **kwargs):
if 'search' in i.tags:
return
if i.itype == 'fqdn' and i.provider != 'spamhaus.org':
try:
r = self._resolve(i.indicator)
try:
r = CODES.get(str(r), None)
except Exception as e:
# https://www.spamhaus.org/faq/section/DNSBL%20Usage
self.logger.error(e)
self.logger.info('check spamhaus return codes')
r = None
if r:
confidence = CONFIDENCE
if ' legit ' in r['description']:
confidence = 6
f = Indicator(**i.__dict__())
f.tags = [r['tags']]
if 'hunter' not in f.tags:
f.tags.append('hunter')
f.description = r['description']
f.confidence = confidence
f.provider = PROVIDER
f.reference_tlp = 'white'
f.reference = 'http://www.spamhaus.org/query/dbl?domain={}'.format(f.indicator)
f.lasttime = f.reporttime = arrow.utcnow()
x = router.indicators_create(f)
self.logger.debug('Spamhaus FQDN: {}'.format(x))
except KeyError as e:
self.logger.error(e)
except Exception as e:
self.logger.error('[Hunter: SpamhausFqdn] {}: giving up on indicator {}'.format(e, i))
def process(i):
if not ENABLED:
return
if i.itype != 'fqdn':
return
if i.probability:
return
if not predict(i.indicator):
return
fqdn = Indicator(**i.__dict__())
fqdn.lasttime = arrow.utcnow()
fqdn.confidence = 4
fqdn.probability = 84
fqdn.provider = 'csirtgadgets.com'
fqdn.reference = 'https://github.com/csirtgadgets/csirtg-domainsml-py' + '#' + VERSION
tags = set(fqdn.tags)
tags.add('predicted')
fqdn.tags = list(tags)
return fqdn
def main():
p = get_argument_parser()
p = ArgumentParser(
description=textwrap.dedent('''\
Env Variables:
CSIRTG_RUNTIME_PATH
example usage:
$ csirtg-ufw -f /var/log/ufw.log
$ ZYRE_GROUP=honeynet csirtg-ufw -d -f /var/log/ufw.log --client zyre
$ csirtg-ufw -f /var/log/ufw.log --client csirtg --user wes --feed scanners -d
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='csirtg-ufw',
parents=[p],
)
p.add_argument('--no-verify-ssl', help='turn TLS/SSL verification OFF', action='store_true')
p.add_argument('-f', '--file', default=FILENAME)
p.add_argument('--client', default='stdout')
p.add_argument('--user')
p.add_argument('--feed')
p.add_argument('--format', default='csv')
p.add_argument('--provider', help='specify provider [default %(default)s]', default=PROVIDER)
p.add_argument('--ignore-client-errors', help='skip when client errors out (eg: HTTP 5XX, etc)', action='store_true')
p.add_argument('--aggregate', help='specify how many seconds to aggregate batches before sending to client '
'[default %(default)s]', default=60)
args = p.parse_args()
if not args.provider:
raise RuntimeError('Missing --provider flag')
# setup logging
setup_logging(args)
logger.debug('starting on: {}'.format(args.file))
verify_ssl = True
if args.no_verify_ssl:
verify_ssl = False
from csirtg_smrt import Smrt
s = Smrt(client=args.client, username=args.user, feed=args.feed, verify_ssl=verify_ssl)
bucket = set()
last_t = round_time(round=int(args.aggregate))
try:
for line in tail(args.file):
if 'csirtg-ufw' in line:
continue
if '[UFW BLOCK]' not in line:
continue
if ' SYN ' not in line:
continue
logger.debug(line)
try:
i = parse_line(line)
except AttributeError:
logger.debug("line not matched: \n{}".format(line))
continue
i = Indicator(**i)
i.provider = args.provider
u_indicator = ':'.join([i.indicator,'/'.join([i.portlist,i.protocol])])
if args.aggregate:
t = round_time(dt=datetime.now(), round=int(args.aggregate))
if t != last_t:
bucket = set()
last_t = t
if u_indicator in bucket:
logger.info('skipping send {}'.format(u_indicator))
continue
bucket.add(u_indicator)
if args.client == 'stdout':
print(FORMATS[args.format](data=[i]))
continue
try:
s.client.indicators_create(i)
logger.info('indicator created: {}'.format(u_indicator))
except Exception as e:
logger.error(e)
if args.ignore_client_errors:
pass
except KeyboardInterrupt:
logger.info('SIGINT caught... stopping')
if args.client != 'stdout':
try:
s.client.stop()
except AttributeError:
pass
logger.info('exiting...')
def main():
p = get_argument_parser()
p = ArgumentParser(
description=textwrap.dedent('''\
Env Variables:
CSIRTG_RUNTIME_PATH
example usage:
$ csirtg-cef -f /var/log/foo.log
$ ZYRE_GROUP=honeynet csirtg-cef -d -f /var/log/foo.log --client zyre
$ csirtg-cef -f /var/log/foo.log --client csirtg --user wes --feed scanners -d
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='csirtg-cef',
parents=[p],
)
p.add_argument('--no-verify-ssl',
help='turn TLS/SSL verification OFF',
action='store_true')
p.add_argument('-f', '--file')
p.add_argument('--client', default='stdout')
p.add_argument('--user')
p.add_argument('--feed')
p.add_argument('--format', default='csv')
p.add_argument('--tags',
help='specify indicator tags [default %(default)s',
default='scanner')
p.add_argument('--provider',
help='specify provider [default %(default)s]',
default=PROVIDER)
p.add_argument('--tail-docker')
args = p.parse_args()
if not args.provider:
raise RuntimeError('Missing --provider flag')
if not args.file:
raise RuntimeError('Missing --file flag')
# setup logging
setup_logging(args)
logger.debug('starting on: {}'.format(args.file))
verify_ssl = True
if args.no_verify_ssl:
verify_ssl = False
f = open(args.file)
from csirtg_smrt import Smrt
s = Smrt(client=args.client,
username=args.user,
feed=args.feed,
verify_ssl=verify_ssl)
try:
for line in tailer.follow(f):
i = parse_line(line)
if not i:
logger.debug('skipping line')
continue
i = Indicator(**i)
logger.debug(i)
i.provider = args.provider
i.tags = args.tags
if args.client == 'stdout':
print(FORMATS[args.format](data=[i]))
else:
s.client.indicators_create(i)
logger.info('indicator created: {}'.format(i.indicator))
except KeyboardInterrupt:
logger.info('SIGINT caught... stopping')
if args.client != 'stdout':
s.client.stop()
logger.info('exiting...')
def main():
p = get_argument_parser()
p = ArgumentParser(
description=textwrap.dedent('''\
Env Variables:
CSIRTG_RUNTIME_PATH
example usage:
$ csirtg-cef -f /var/log/foo.log
$ ZYRE_GROUP=honeynet csirtg-cef -d -f /var/log/foo.log --client zyre
$ csirtg-cef -f /var/log/foo.log --client csirtg --user wes --feed scanners -d
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='csirtg-cef',
parents=[p],
)
p.add_argument('--no-verify-ssl', help='turn TLS/SSL verification OFF', action='store_true')
p.add_argument('-f', '--file')
p.add_argument('--client', default='stdout')
p.add_argument('--user')
p.add_argument('--feed')
p.add_argument('--format', default='csv')
p.add_argument('--tags', help='specify indicator tags [default %(default)s', default='scanner')
p.add_argument('--provider', help='specify provider [default %(default)s]', default=PROVIDER)
p.add_argument('--aggregate', help='specify how many seconds to aggregate batches before sending to client '
'[default %(default)s]', default=60)
p.add_argument('--tail-docker')
args = p.parse_args()
# setup logging
setup_logging(args)
verify_ssl = True
if args.no_verify_ssl:
verify_ssl = False
if args.file:
logger.debug('starting on: {}'.format(args.file))
data_source = tail(args.file)
elif args.tail_docker:
logger.debug('starting on container: {}'.format(args.tail_docker))
#data_source = subprocess.Popen(["docker", "logs", "-f", "--tail", "0", args.tail_docker], bufsize=1, stdout=subprocess.PIPE).stdout
client = docker.from_env(version='auto')
container = client.containers.get(args.tail_docker)
data_source = container.logs(stream=True, follow=True, tail=0)
else:
logger.error('Missing --file or --tail-docker flag')
raise SystemExit
logger.info('sending data as: %s' % args.provider)
s = Smrt(client=args.client, username=args.user, feed=args.feed, verify_ssl=verify_ssl)
bucket = set()
last_t = round_time(round=int(args.aggregate))
try:
for line in data_source:
i = parse_line(line)
if not i:
logger.debug('skipping line')
continue
i = Indicator(**i)
logger.debug(i)
i.provider = args.provider
i.tags = args.tags
if args.aggregate:
t = round_time(dt=datetime.now(), round=int(args.aggregate))
if t != last_t:
bucket = set()
last_t = t
if i.indicator in bucket:
logger.info('skipping send {}'.format(i.indicator))
continue
bucket.add(i.indicator)
if args.client == 'stdout':
print(FORMATS[args.format](data=[i]))
else:
try:
s.client.indicators_create(i)
logger.info('indicator created: {}'.format(i.indicator))
except Exception as e:
logger.error(e)
except Exception as e:
logger.error(e)
except KeyboardInterrupt:
logger.info('SIGINT caught... stopping')
if args.client != 'stdout':
s.client.stop()
logger.info('exiting...')
