def main():
p = get_argument_parser()
p = ArgumentParser(
description=textwrap.dedent('''\
Env Variables:
CSIRTG_RUNTIME_PATH
example usage:
$ csirtg-ufw -f /var/log/ufw.log
$ ZYRE_GROUP=honeynet csirtg-ufw -d -f /var/log/ufw.log --client zyre
$ csirtg-ufw -f /var/log/ufw.log --client csirtg --user wes --feed scanners -d
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='csirtg-ufw',
parents=[p],
)
p.add_argument('--no-verify-ssl', help='turn TLS/SSL verification OFF', action='store_true')
p.add_argument('-f', '--file', default=FILENAME)
p.add_argument('--client', default='stdout')
p.add_argument('--user')
p.add_argument('--feed')
p.add_argument('--format', default='csv')
p.add_argument('--provider', help='specify provider [default %(default)s]', default=PROVIDER)
p.add_argument('--ignore-client-errors', help='skip when client errors out (eg: HTTP 5XX, etc)', action='store_true')
p.add_argument('--aggregate', help='specify how many seconds to aggregate batches before sending to client '
'[default %(default)s]', default=60)
args = p.parse_args()
if not args.provider:
raise RuntimeError('Missing --provider flag')
# setup logging
setup_logging(args)
logger.debug('starting on: {}'.format(args.file))
verify_ssl = True
if args.no_verify_ssl:
verify_ssl = False
from csirtg_smrt import Smrt
s = Smrt(client=args.client, username=args.user, feed=args.feed, verify_ssl=verify_ssl)
bucket = set()
last_t = round_time(round=int(args.aggregate))
try:
for line in tail(args.file):
if 'csirtg-ufw' in line:
continue
if '[UFW BLOCK]' not in line:
continue
if ' SYN ' not in line:
continue
logger.debug(line)
try:
i = parse_line(line)
except AttributeError:
logger.debug("line not matched: \n{}".format(line))
continue
i = Indicator(**i)
i.provider = args.provider
u_indicator = ':'.join([i.indicator,'/'.join([i.portlist,i.protocol])])
if args.aggregate:
t = round_time(dt=datetime.now(), round=int(args.aggregate))
if t != last_t:
bucket = set()
last_t = t
if u_indicator in bucket:
logger.info('skipping send {}'.format(u_indicator))
continue
bucket.add(u_indicator)
if args.client == 'stdout':
print(FORMATS[args.format](data=[i]))
continue
try:
s.client.indicators_create(i)
logger.info('indicator created: {}'.format(u_indicator))
except Exception as e:
logger.error(e)
if args.ignore_client_errors:
pass
except KeyboardInterrupt:
logger.info('SIGINT caught... stopping')
if args.client != 'stdout':
try:
s.client.stop()
except AttributeError:
pass
logger.info('exiting...')
def main():
p = get_argument_parser()
p = ArgumentParser(
description=textwrap.dedent('''\
Env Variables:
CSIRTG_RUNTIME_PATH
CSIRTG_SMRT_TOKEN
example usage:
$ csirtg-smrt --rule rules/default
$ csirtg-smrt --rule default/csirtg.yml --feed port-scanners --remote http://localhost:5000
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='csirtg-smrt',
parents=[p],
)
p.add_argument(
"-r",
"--rule",
help=
"specify the rules directory or specific rules file [default: %(default)s",
default=SMRT_RULES_PATH)
p.add_argument("-f", "--feed", help="specify the feed to process")
p.add_argument("--remote", help="specify the remote api url")
p.add_argument(
'--remote-type',
help="specify remote type [cif, csirtg, elasticsearch, syslog, etc]")
p.add_argument('--client', default='stdout')
p.add_argument('--cache',
help="specify feed cache [default %(default)s]",
default=SMRT_CACHE)
p.add_argument(
"--limit",
help="limit the number of records processed [default: %(default)s]",
default=None)
p.add_argument("--token",
help="specify token [default: %(default)s]",
default=TOKEN)
p.add_argument('--service',
action='store_true',
help="start in service mode")
p.add_argument('--service-interval',
help='set run interval [in minutes, default %(default)s]',
default=SERVICE_INTERVAL)
p.add_argument('--ignore-unknown', action='store_true')
p.add_argument('--config',
help='specify csirtg-smrt config path [default %(default)s',
default=CONFIG_PATH)
p.add_argument('--user')
p.add_argument('--delay',
help='specify initial delay',
default=randint(5, 55))
p.add_argument('--remember-path',
help='specify remember db path [default: %(default)s',
default=ARCHIVE_PATH)
p.add_argument('--remember',
help='remember what has been already processed',
action='store_true')
p.add_argument('--format',
help='specify output format [default: %(default)s]"',
default=FORMAT,
choices=FORMATS.keys())
p.add_argument('--filter-indicator',
help='filter for specific indicator, useful in testing')
p.add_argument('--fireball',
help='run in fireball mode, bulk+async magic',
action='store_true')
p.add_argument('--no-fetch',
help='do not re-fetch if the cache exists',
action='store_true')
p.add_argument('--no-verify-ssl',
help='turn TLS/SSL verification OFF',
action='store_true')
p.add_argument(
'--goback',
help=
'specify default number of days to start out at [default %(default)s]',
default=GOBACK_DAYS)
p.add_argument('--fields',
help='specify fields for stdout [default %(default)s]"',
default=','.join(STDOUT_FIELDS))
p.add_argument('--skip-invalid',
help="skip invalid indicators in DEBUG (-d) mode",
action="store_true")
p.add_argument('--skip-broken',
help='skip seemingly broken feeds',
action='store_true')
p.add_argument(
'--send-retries',
help=
'specify how many times to re-try sending indicators after a failure '
'[default: %(default)s',
default=5)
p.add_argument(
'--send-retries-wait',
help='how many seconds to wait between retries [default: %(default)s',
default=30)
args = p.parse_args()
o = read_config(args)
options = vars(args)
for v in options:
if options[v] is None:
options[v] = o.get(v)
setup_logging(args)
logger.info('loglevel is: {}'.format(
logging.getLevelName(logger.getEffectiveLevel())))
setup_runtime_path(args.runtime_path)
verify_ssl = True
if options.get('no_verify_ssl') or o.get('no_verify_ssl'):
verify_ssl = False
goback = args.goback
if goback:
goback = arrow.utcnow().replace(days=-int(goback))
if not args.service:
data = None
if select.select([
sys.stdin,
], [], [], 0.0)[0]:
data = sys.stdin.read()
try:
_run_smrt(
options, **{
'args': args,
'data': data,
'verify_ssl': verify_ssl,
'goback': goback
})
except KeyboardInterrupt:
logger.info('exiting..')
raise SystemExit
# we're running as a service
setup_signals(__name__)
service_interval = int(args.service_interval)
r = int(args.delay)
logger.info(
"random delay is {}, then running every {} min after that".format(
r, service_interval))
if r != 0:
try:
sleep((r * 60))
except KeyboardInterrupt:
logger.info('shutting down')
raise SystemExit
except Exception as e:
logger.error(e)
raise SystemExit
logger.info('starting...')
def _run():
logger.debug('forking process...')
p = Process(target=_run_smrt,
args=(options, ),
kwargs={
'args': args,
'verify_ssl': verify_ssl,
'goback': goback,
'service_mode': True
})
p.daemon = False
p.start()
p.join()
logger.debug('child process re-joined')
# first run, PeriodicCallback has builtin wait..
_run()
main_loop = ioloop.IOLoop()
service_interval = (service_interval * 60000)
loop = ioloop.PeriodicCallback(_run, service_interval)
try:
loop.start()
main_loop.start()
except KeyboardInterrupt:
logger.info('exiting..')
pass
except Exception as e:
logger.error(e)
pass
def main():
p = get_argument_parser()
p = ArgumentParser(
description=textwrap.dedent('''\
Env Variables:
CSIRTG_RUNTIME_PATH
example usage:
$ csirtg-cef -f /var/log/foo.log
$ ZYRE_GROUP=honeynet csirtg-cef -d -f /var/log/foo.log --client zyre
$ csirtg-cef -f /var/log/foo.log --client csirtg --user wes --feed scanners -d
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='csirtg-cef',
parents=[p],
)
p.add_argument('--no-verify-ssl',
help='turn TLS/SSL verification OFF',
action='store_true')
p.add_argument('-f', '--file')
p.add_argument('--client', default='stdout')
p.add_argument('--user')
p.add_argument('--feed')
p.add_argument('--format', default='csv')
p.add_argument('--tags',
help='specify indicator tags [default %(default)s',
default='scanner')
p.add_argument('--provider',
help='specify provider [default %(default)s]',
default=PROVIDER)
p.add_argument('--tail-docker')
args = p.parse_args()
if not args.provider:
raise RuntimeError('Missing --provider flag')
if not args.file:
raise RuntimeError('Missing --file flag')
# setup logging
setup_logging(args)
logger.debug('starting on: {}'.format(args.file))
verify_ssl = True
if args.no_verify_ssl:
verify_ssl = False
f = open(args.file)
from csirtg_smrt import Smrt
s = Smrt(client=args.client,
username=args.user,
feed=args.feed,
verify_ssl=verify_ssl)
try:
for line in tailer.follow(f):
i = parse_line(line)
if not i:
logger.debug('skipping line')
continue
i = Indicator(**i)
logger.debug(i)
i.provider = args.provider
i.tags = args.tags
if args.client == 'stdout':
print(FORMATS[args.format](data=[i]))
else:
s.client.indicators_create(i)
logger.info('indicator created: {}'.format(i.indicator))
except KeyboardInterrupt:
logger.info('SIGINT caught... stopping')
if args.client != 'stdout':
s.client.stop()
logger.info('exiting...')
def main():
p = get_argument_parser()
p = ArgumentParser(
description=textwrap.dedent('''\
Env Variables:
CSIRTG_RUNTIME_PATH
CSIRTG_SMRT_TOKEN
example usage:
$ csirtg-smrt --rule rules/default
$ csirtg-smrt --rule default/csirtg.yml --feed port-scanners --remote http://localhost:5000
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='csirtg-smrt',
parents=[p],
)
p.add_argument("-r", "--rule", help="specify the rules directory or specific rules file [default: %(default)s",
default=SMRT_RULES_PATH)
p.add_argument("-f", "--feed", help="specify the feed to process")
p.add_argument("--remote", help="specify the remote api url")
p.add_argument('--remote-type', help="specify remote type [cif, csirtg, elasticsearch, syslog, etc]")
p.add_argument('--client', default='stdout')
p.add_argument('--cache', help="specify feed cache [default %(default)s]", default=SMRT_CACHE)
p.add_argument("--limit", help="limit the number of records processed [default: %(default)s]",
default=None)
p.add_argument("--token", help="specify token [default: %(default)s]", default=TOKEN)
p.add_argument('--service', action='store_true', help="start in service mode")
p.add_argument('--service-interval', help='set run interval [in minutes, default %(default)s]',
default=SERVICE_INTERVAL)
p.add_argument('--ignore-unknown', action='store_true')
p.add_argument('--config', help='specify csirtg-smrt config path [default %(default)s', default=CONFIG_PATH)
p.add_argument('--user')
p.add_argument('--delay', help='specify initial delay', default=randint(5, 55))
p.add_argument('--remember-path', help='specify remember db path [default: %(default)s', default=ARCHIVE_PATH)
p.add_argument('--remember', help='remember what has been already processed', action='store_true')
p.add_argument('--format', help='specify output format [default: %(default)s]"', default=FORMAT,
choices=FORMATS.keys())
p.add_argument('--filter-indicator', help='filter for specific indicator, useful in testing')
p.add_argument('--fireball', help='run in fireball mode, bulk+async magic', action='store_true')
p.add_argument('--no-fetch', help='do not re-fetch if the cache exists', action='store_true')
p.add_argument('--no-verify-ssl', help='turn TLS/SSL verification OFF', action='store_true')
p.add_argument('--goback', help='specify default number of days to start out at [default %(default)s]',
default=GOBACK_DAYS)
p.add_argument('--fields', help='specify fields for stdout [default %(default)s]"', default=','.join(STDOUT_FIELDS))
p.add_argument('--skip-invalid', help="skip invalid indicators in DEBUG (-d) mode", action="store_true")
p.add_argument('--skip-broken', help='skip seemingly broken feeds', action='store_true')
p.add_argument('--send-retries', help='specify how many times to re-try sending indicators after a failure '
'[default: %(default)s', default=5)
p.add_argument('--send-retries-wait', help='how many seconds to wait between retries [default: %(default)s',
default=30)
args = p.parse_args()
o = read_config(args)
options = vars(args)
for v in options:
if options[v] is None:
options[v] = o.get(v)
setup_logging(args)
logger.info('loglevel is: {}'.format(logging.getLevelName(logger.getEffectiveLevel())))
setup_runtime_path(args.runtime_path)
verify_ssl = True
if options.get('no_verify_ssl') or o.get('no_verify_ssl'):
verify_ssl = False
goback = args.goback
if goback:
goback = arrow.utcnow().replace(days=-int(goback))
if not args.service:
data = None
if select.select([sys.stdin, ], [], [], 0.0)[0]:
data = sys.stdin.read()
try:
_run_smrt(options, **{
'args': args,
'data': data,
'verify_ssl': verify_ssl,
'goback': goback
})
except KeyboardInterrupt:
logger.info('exiting..')
raise SystemExit
# we're running as a service
setup_signals(__name__)
service_interval = int(args.service_interval)
r = int(args.delay)
logger.info("random delay is {}, then running every {} min after that".format(r, service_interval))
if r != 0:
try:
sleep((r * 60))
except KeyboardInterrupt:
logger.info('shutting down')
raise SystemExit
except Exception as e:
logger.error(e)
raise SystemExit
logger.info('starting...')
def _run():
logger.debug('forking process...')
p = Process(target=_run_smrt, args=(options,), kwargs={
'args': args,
'verify_ssl': verify_ssl,
'goback': goback,
'service_mode': True
})
p.daemon = False
p.start()
p.join()
logger.debug('child process re-joined')
# first run, PeriodicCallback has builtin wait..
_run()
main_loop = ioloop.IOLoop()
service_interval = (service_interval * 60000)
loop = ioloop.PeriodicCallback(_run, service_interval)
try:
loop.start()
main_loop.start()
except KeyboardInterrupt:
logger.info('exiting..')
pass
except Exception as e:
logger.error(e)
pass
def main():
p = get_argument_parser()
p = ArgumentParser(
description=textwrap.dedent('''\
Env Variables:
CSIRTG_RUNTIME_PATH
example usage:
$ csirtg-cef -f /var/log/foo.log
$ ZYRE_GROUP=honeynet csirtg-cef -d -f /var/log/foo.log --client zyre
$ csirtg-cef -f /var/log/foo.log --client csirtg --user wes --feed scanners -d
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='csirtg-cef',
parents=[p],
)
p.add_argument('--no-verify-ssl', help='turn TLS/SSL verification OFF', action='store_true')
p.add_argument('-f', '--file')
p.add_argument('--client', default='stdout')
p.add_argument('--user')
p.add_argument('--feed')
p.add_argument('--format', default='csv')
p.add_argument('--tags', help='specify indicator tags [default %(default)s', default='scanner')
p.add_argument('--provider', help='specify provider [default %(default)s]', default=PROVIDER)
p.add_argument('--aggregate', help='specify how many seconds to aggregate batches before sending to client '
'[default %(default)s]', default=60)
p.add_argument('--tail-docker')
args = p.parse_args()
# setup logging
setup_logging(args)
verify_ssl = True
if args.no_verify_ssl:
verify_ssl = False
if args.file:
logger.debug('starting on: {}'.format(args.file))
data_source = tail(args.file)
elif args.tail_docker:
logger.debug('starting on container: {}'.format(args.tail_docker))
#data_source = subprocess.Popen(["docker", "logs", "-f", "--tail", "0", args.tail_docker], bufsize=1, stdout=subprocess.PIPE).stdout
client = docker.from_env(version='auto')
container = client.containers.get(args.tail_docker)
data_source = container.logs(stream=True, follow=True, tail=0)
else:
logger.error('Missing --file or --tail-docker flag')
raise SystemExit
logger.info('sending data as: %s' % args.provider)
s = Smrt(client=args.client, username=args.user, feed=args.feed, verify_ssl=verify_ssl)
bucket = set()
last_t = round_time(round=int(args.aggregate))
try:
for line in data_source:
i = parse_line(line)
if not i:
logger.debug('skipping line')
continue
i = Indicator(**i)
logger.debug(i)
i.provider = args.provider
i.tags = args.tags
if args.aggregate:
t = round_time(dt=datetime.now(), round=int(args.aggregate))
if t != last_t:
bucket = set()
last_t = t
if i.indicator in bucket:
logger.info('skipping send {}'.format(i.indicator))
continue
bucket.add(i.indicator)
if args.client == 'stdout':
print(FORMATS[args.format](data=[i]))
else:
try:
s.client.indicators_create(i)
logger.info('indicator created: {}'.format(i.indicator))
except Exception as e:
logger.error(e)
except Exception as e:
logger.error(e)
except KeyboardInterrupt:
logger.info('SIGINT caught... stopping')
if args.client != 'stdout':
s.client.stop()
logger.info('exiting...')
def main():
p = get_argument_parser()
p = ArgumentParser(
description=textwrap.dedent('''\
Env Variables:
CSIRTG_RUNTIME_PATH
CSIRTG_TOKEN
example usage:
$ csirtg-smrt --rule rules/default
$ csirtg-smrt --rule default/csirtg.yml --feed port-scanners --remote http://localhost:5000
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='cif-smrt',
parents=[p],
)
p.add_argument("-r", "--rule", help="specify the rules directory or specific rules file [default: %(default)s",
default=SMRT_RULES_PATH)
p.add_argument("-f", "--feed", help="specify the feed to process")
p.add_argument("--remote", dest="remote", help="specify the remote api url [default: %(default)s",
default=REMOTE_ADDR)
p.add_argument('--cache', help="specify feed cache [default %(default)s]", default=SMRT_CACHE)
p.add_argument("--limit", dest="limit", help="limit the number of records processed [default: %(default)s]",
default=None)
p.add_argument("--token", dest="token", help="specify token [default: %(default)s]", default=TOKEN)
p.add_argument('--test', action='store_true')
p.add_argument('--sleep', default=60)
p.add_argument('--ignore-unknown', action='store_true')
p.add_argument('--config', help='specify csirtg-smrt config path [default %(default)s', default=CONFIG_PATH)
p.add_argument('--client', default='cif')
p.add_argument('--user')
args = p.parse_args()
o = read_config(args)
options = vars(args)
for v in options:
if options[v] is None:
options[v] = o.get(v)
setup_logging(args)
logger = logging.getLogger(__name__)
logger.info('loglevel is: {}'.format(logging.getLevelName(logger.getEffectiveLevel())))
setup_signals(__name__)
setup_runtime_path(args.runtime_path)
stop = False
r = False
if not args.test:
r = randint(5, 55)
logger.info("random delay is {}, then running every 60min after that".format(r))
sleep((r * 60))
while not stop:
if args.test:
stop = True
logger.info('starting...')
try:
with Smrt(options.get('remote'), options.get('token'), client=args.client, user=args.user,
feed=args.feed) as s:
logger.info('staring up...')
logger.info('testing router connection...')
s.ping_router()
x = s.process(args.rule, feed=args.feed, limit=args.limit)
logger.info('complete')
if not args.test:
logger.info('sleeping for 1 hour')
sleep((60 * 60))
except AuthError as e:
logger.error(e)
stop = True
except RuntimeError as e:
logger.error(e)
if str(e).startswith('submission failed'):
stop = True
else:
logging.exception('Got exception on main handler')
except TimeoutError as e:
logger.error(e)
stop = True
except KeyboardInterrupt:
logger.info('shutting down')
stop = True
logger.info('completed')