def main():
p = get_argument_parser()
p = ArgumentParser(
description=textwrap.dedent('''\
Env Variables:
CSIRTG_RUNTIME_PATH
CSIRTG_SMRT_TOKEN
example usage:
$ csirtg-smrt --rule rules/default
$ csirtg-smrt --rule default/csirtg.yml --feed port-scanners --remote http://localhost:5000
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='csirtg-smrt',
parents=[p],
)
p.add_argument("-r", "--rule", help="specify the rules directory or specific rules file [default: %(default)s",
default=SMRT_RULES_PATH)
p.add_argument("-f", "--feed", help="specify the feed to process")
p.add_argument("--remote", help="specify the remote api url")
p.add_argument('--remote-type', help="specify remote type [cif, csirtg, elasticsearch, syslog, etc]")
p.add_argument('--client', default='stdout')
p.add_argument('--cache', help="specify feed cache [default %(default)s]", default=SMRT_CACHE)
p.add_argument("--limit", help="limit the number of records processed [default: %(default)s]",
default=None)
p.add_argument("--token", help="specify token [default: %(default)s]", default=TOKEN)
p.add_argument('--service', action='store_true', help="start in service mode")
p.add_argument('--service-interval', help='set run interval [in minutes, default %(default)s]',
default=SERVICE_INTERVAL)
p.add_argument('--ignore-unknown', action='store_true')
p.add_argument('--config', help='specify csirtg-smrt config path [default %(default)s', default=CONFIG_PATH)
p.add_argument('--user')
p.add_argument('--delay', help='specify initial delay', default=randint(5, 55))
p.add_argument('--remember-path', help='specify remember db path [default: %(default)s', default=ARCHIVE_PATH)
p.add_argument('--remember', help='remember what has been already processed', action='store_true')
p.add_argument('--format', help='specify output format [default: %(default)s]"', default=FORMAT,
choices=FORMATS.keys())
p.add_argument('--filter-indicator', help='filter for specific indicator, useful in testing')
p.add_argument('--fireball', help='run in fireball mode, bulk+async magic', action='store_true')
p.add_argument('--no-fetch', help='do not re-fetch if the cache exists', action='store_true')
p.add_argument('--no-verify-ssl', help='turn TLS/SSL verification OFF', action='store_true')
p.add_argument('--goback', help='specify default number of days to start out at [default %(default)s]',
default=GOBACK_DAYS)
p.add_argument('--fields', help='specify fields for stdout [default %(default)s]"', default=','.join(STDOUT_FIELDS))
p.add_argument('--skip-invalid', help="skip invalid indicators in DEBUG (-d) mode", action="store_true")
p.add_argument('--skip-broken', help='skip seemingly broken feeds', action='store_true')
p.add_argument('--send-retries', help='specify how many times to re-try sending indicators after a failure '
'[default: %(default)s', default=5)
p.add_argument('--send-retries-wait', help='how many seconds to wait between retries [default: %(default)s',
default=30)
args = p.parse_args()
o = read_config(args)
options = vars(args)
for v in options:
if options[v] is None:
options[v] = o.get(v)
setup_logging(args)
logger.info('loglevel is: {}'.format(logging.getLevelName(logger.getEffectiveLevel())))
setup_runtime_path(args.runtime_path)
verify_ssl = True
if options.get('no_verify_ssl') or o.get('no_verify_ssl'):
verify_ssl = False
goback = args.goback
if goback:
goback = arrow.utcnow().replace(days=-int(goback))
if not args.service:
data = None
if select.select([sys.stdin, ], [], [], 0.0)[0]:
data = sys.stdin.read()
try:
_run_smrt(options, **{
'args': args,
'data': data,
'verify_ssl': verify_ssl,
'goback': goback
})
except KeyboardInterrupt:
logger.info('exiting..')
raise SystemExit
# we're running as a service
setup_signals(__name__)
service_interval = int(args.service_interval)
r = int(args.delay)
logger.info("random delay is {}, then running every {} min after that".format(r, service_interval))
if r != 0:
try:
sleep((r * 60))
except KeyboardInterrupt:
logger.info('shutting down')
raise SystemExit
except Exception as e:
logger.error(e)
raise SystemExit
logger.info('starting...')
def _run():
logger.debug('forking process...')
p = Process(target=_run_smrt, args=(options,), kwargs={
'args': args,
'verify_ssl': verify_ssl,
'goback': goback,
'service_mode': True
})
p.daemon = False
p.start()
p.join()
logger.debug('child process re-joined')
# first run, PeriodicCallback has builtin wait..
_run()
main_loop = ioloop.IOLoop()
service_interval = (service_interval * 60000)
loop = ioloop.PeriodicCallback(_run, service_interval)
try:
loop.start()
main_loop.start()
except KeyboardInterrupt:
logger.info('exiting..')
pass
except Exception as e:
logger.error(e)
pass
def main():
p = get_argument_parser()
p = ArgumentParser(
description=textwrap.dedent('''\
Env Variables:
CSIRTG_RUNTIME_PATH
CSIRTG_SMRT_TOKEN
example usage:
$ csirtg-smrt --rule rules/default
$ csirtg-smrt --rule default/csirtg.yml --feed port-scanners --remote http://localhost:5000
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='csirtg-smrt',
parents=[p],
)
p.add_argument(
"-r",
"--rule",
help=
"specify the rules directory or specific rules file [default: %(default)s",
default=SMRT_RULES_PATH)
p.add_argument("-f", "--feed", help="specify the feed to process")
p.add_argument("--remote", help="specify the remote api url")
p.add_argument(
'--remote-type',
help="specify remote type [cif, csirtg, elasticsearch, syslog, etc]")
p.add_argument('--client', default='stdout')
p.add_argument('--cache',
help="specify feed cache [default %(default)s]",
default=SMRT_CACHE)
p.add_argument(
"--limit",
help="limit the number of records processed [default: %(default)s]",
default=None)
p.add_argument("--token",
help="specify token [default: %(default)s]",
default=TOKEN)
p.add_argument('--service',
action='store_true',
help="start in service mode")
p.add_argument('--service-interval',
help='set run interval [in minutes, default %(default)s]',
default=SERVICE_INTERVAL)
p.add_argument('--ignore-unknown', action='store_true')
p.add_argument('--config',
help='specify csirtg-smrt config path [default %(default)s',
default=CONFIG_PATH)
p.add_argument('--user')
p.add_argument('--delay',
help='specify initial delay',
default=randint(5, 55))
p.add_argument('--remember-path',
help='specify remember db path [default: %(default)s',
default=ARCHIVE_PATH)
p.add_argument('--remember',
help='remember what has been already processed',
action='store_true')
p.add_argument('--format',
help='specify output format [default: %(default)s]"',
default=FORMAT,
choices=FORMATS.keys())
p.add_argument('--filter-indicator',
help='filter for specific indicator, useful in testing')
p.add_argument('--fireball',
help='run in fireball mode, bulk+async magic',
action='store_true')
p.add_argument('--no-fetch',
help='do not re-fetch if the cache exists',
action='store_true')
p.add_argument('--no-verify-ssl',
help='turn TLS/SSL verification OFF',
action='store_true')
p.add_argument(
'--goback',
help=
'specify default number of days to start out at [default %(default)s]',
default=GOBACK_DAYS)
p.add_argument('--fields',
help='specify fields for stdout [default %(default)s]"',
default=','.join(STDOUT_FIELDS))
p.add_argument('--skip-invalid',
help="skip invalid indicators in DEBUG (-d) mode",
action="store_true")
p.add_argument('--skip-broken',
help='skip seemingly broken feeds',
action='store_true')
p.add_argument(
'--send-retries',
help=
'specify how many times to re-try sending indicators after a failure '
'[default: %(default)s',
default=5)
p.add_argument(
'--send-retries-wait',
help='how many seconds to wait between retries [default: %(default)s',
default=30)
args = p.parse_args()
o = read_config(args)
options = vars(args)
for v in options:
if options[v] is None:
options[v] = o.get(v)
setup_logging(args)
logger.info('loglevel is: {}'.format(
logging.getLevelName(logger.getEffectiveLevel())))
setup_runtime_path(args.runtime_path)
verify_ssl = True
if options.get('no_verify_ssl') or o.get('no_verify_ssl'):
verify_ssl = False
goback = args.goback
if goback:
goback = arrow.utcnow().replace(days=-int(goback))
if not args.service:
data = None
if select.select([
sys.stdin,
], [], [], 0.0)[0]:
data = sys.stdin.read()
try:
_run_smrt(
options, **{
'args': args,
'data': data,
'verify_ssl': verify_ssl,
'goback': goback
})
except KeyboardInterrupt:
logger.info('exiting..')
raise SystemExit
# we're running as a service
setup_signals(__name__)
service_interval = int(args.service_interval)
r = int(args.delay)
logger.info(
"random delay is {}, then running every {} min after that".format(
r, service_interval))
if r != 0:
try:
sleep((r * 60))
except KeyboardInterrupt:
logger.info('shutting down')
raise SystemExit
except Exception as e:
logger.error(e)
raise SystemExit
logger.info('starting...')
def _run():
logger.debug('forking process...')
p = Process(target=_run_smrt,
args=(options, ),
kwargs={
'args': args,
'verify_ssl': verify_ssl,
'goback': goback,
'service_mode': True
})
p.daemon = False
p.start()
p.join()
logger.debug('child process re-joined')
# first run, PeriodicCallback has builtin wait..
_run()
main_loop = ioloop.IOLoop()
service_interval = (service_interval * 60000)
loop = ioloop.PeriodicCallback(_run, service_interval)
try:
loop.start()
main_loop.start()
except KeyboardInterrupt:
logger.info('exiting..')
pass
except Exception as e:
logger.error(e)
pass
def main():
p = get_argument_parser()
p = ArgumentParser(
description=textwrap.dedent('''\
Env Variables:
CSIRTG_RUNTIME_PATH
CSIRTG_TOKEN
example usage:
$ csirtg-smrt --rule rules/default
$ csirtg-smrt --rule default/csirtg.yml --feed port-scanners --remote http://localhost:5000
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='cif-smrt',
parents=[p],
)
p.add_argument("-r", "--rule", help="specify the rules directory or specific rules file [default: %(default)s",
default=SMRT_RULES_PATH)
p.add_argument("-f", "--feed", help="specify the feed to process")
p.add_argument("--remote", dest="remote", help="specify the remote api url [default: %(default)s",
default=REMOTE_ADDR)
p.add_argument('--cache', help="specify feed cache [default %(default)s]", default=SMRT_CACHE)
p.add_argument("--limit", dest="limit", help="limit the number of records processed [default: %(default)s]",
default=None)
p.add_argument("--token", dest="token", help="specify token [default: %(default)s]", default=TOKEN)
p.add_argument('--test', action='store_true')
p.add_argument('--sleep', default=60)
p.add_argument('--ignore-unknown', action='store_true')
p.add_argument('--config', help='specify csirtg-smrt config path [default %(default)s', default=CONFIG_PATH)
p.add_argument('--client', default='cif')
p.add_argument('--user')
args = p.parse_args()
o = read_config(args)
options = vars(args)
for v in options:
if options[v] is None:
options[v] = o.get(v)
setup_logging(args)
logger = logging.getLogger(__name__)
logger.info('loglevel is: {}'.format(logging.getLevelName(logger.getEffectiveLevel())))
setup_signals(__name__)
setup_runtime_path(args.runtime_path)
stop = False
r = False
if not args.test:
r = randint(5, 55)
logger.info("random delay is {}, then running every 60min after that".format(r))
sleep((r * 60))
while not stop:
if args.test:
stop = True
logger.info('starting...')
try:
with Smrt(options.get('remote'), options.get('token'), client=args.client, user=args.user,
feed=args.feed) as s:
logger.info('staring up...')
logger.info('testing router connection...')
s.ping_router()
x = s.process(args.rule, feed=args.feed, limit=args.limit)
logger.info('complete')
if not args.test:
logger.info('sleeping for 1 hour')
sleep((60 * 60))
except AuthError as e:
logger.error(e)
stop = True
except RuntimeError as e:
logger.error(e)
if str(e).startswith('submission failed'):
stop = True
else:
logging.exception('Got exception on main handler')
except TimeoutError as e:
logger.error(e)
stop = True
except KeyboardInterrupt:
logger.info('shutting down')
stop = True
logger.info('completed')