def main():
p = get_argument_parser()
p = ArgumentParser(description=textwrap.dedent('''\
Env Variables:
CIF_RUNTIME_PATH
example usage:
$ cif-store -d
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='cif-store',
parents=[p])
p.add_argument("--store",
help="store type {} [default: %(default)s]".format(
', '.join(STORE_PLUGINS)),
default=STORE_DEFAULT)
p.add_argument('--remote', help='specify remote')
args = p.parse_args()
setup_logging(args)
setup_signals(__name__)
def main():
p = get_argument_parser()
p = ArgumentParser(description=textwrap.dedent('''\
Env Variables:
example usage:
$ cif-hunter -d
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='cif-hunter',
parents=[p])
p.add_argument('-w', '--workers', help='Number of workers', default=1)
args = p.parse_args()
setup_logging(args)
if args.verbose:
logger.setLevel(logging.INFO)
if args.debug:
logger.setLevel(logging.DEBUG)
logger.info(f"loglevel: {logger.getEffectiveLevel()}")
m = Manager(target=Hunter, threads=args.workers)
m.start()
def main():
p = get_argument_parser()
p = ArgumentParser(description=textwrap.dedent('''\
Env Variables:
CIF_ROUTER_STREAM_ADDR
CIF_STREAM_ADDR
example usage:
$ cif-streamer -d
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='cif-streamer',
parents=[p])
args = p.parse_args()
setup_logging(args)
logger.info('loglevel is: {}'.format(
logging.getLevelName(logger.getEffectiveLevel())))
setup_runtime_path(args.runtime_path)
# setup_signals(__name__)
s = Streamer()
try:
s.start()
except KeyboardInterrupt:
s.stop()
def main():
p = get_argument_parser()
p = ArgumentParser(description=textwrap.dedent('''\
Env Variables:
example usage:
$ cif-hunter -d
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='cif-router',
parents=[p])
args = p.parse_args()
setup_logging(args)
logger = logging.getLogger(__name__)
logger.info('loglevel is: {}'.format(
logging.getLevelName(logger.getEffectiveLevel())))
if args.logging_ignore:
to_ignore = args.logging_ignore.split(',')
for i in to_ignore:
logging.getLogger(i).setLevel(logging.WARNING)
setup_runtime_path(args.runtime_path)
# setup_signals(__name__)
h = Hunter()
try:
h.start()
except KeyboardInterrupt or SystemExit:
h.stop()
def main():
p = get_argument_parser()
p = ArgumentParser(description=textwrap.dedent('''\
Env Variables:
example usage:
$ cif-enricher -d
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='cif-enricher',
parents=[p])
p.add_argument('-E', '--threads', help='Number of workers', default=1)
args = p.parse_args()
setup_logging(args)
if args.verbose:
logger.setLevel(logging.INFO)
if args.debug:
logger.setLevel(logging.DEBUG)
logger.info(f"loglevel: {logger.getEffectiveLevel()}")
m = EnrichmentManager(threads=args.threads, target=Enricher)
try:
m.start()
except KeyboardInterrupt:
m.stop()
def main():
p = get_argument_parser()
p = ArgumentParser(description=textwrap.dedent('''\
Env Variables:
CIF_RUNTIME_PATH
CIF_ROUTER_ADDR
CIF_HUNTER_ADDR
CIF_STORE_ADDR
example usage:
$ cif-router --listen 0.0.0.0 -d
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='cif-router',
parents=[p])
p.add_argument('--listen',
help='address to listen on [default: %(default)s]',
default=ROUTER_ADDR)
p.add_argument('--hunter',
help='address hunters listen on on [default: %(default)s]',
default=HUNTER_ADDR)
p.add_argument("--store",
help="specify a store address [default: %(default)s]",
default=STORE_ADDR)
p.add_argument('--p2p',
action='store_true',
help='enable experimental p2p support')
args = p.parse_args()
setup_logging(args)
logger = logging.getLogger(__name__)
logger.info('loglevel is: {}'.format(
logging.getLevelName(logger.getEffectiveLevel())))
setup_signals(__name__)
setup_runtime_path(args.runtime_path)
with Router(listen=args.listen,
hunter=args.hunter,
store=args.store,
p2p=args.p2p) as r:
try:
logger.info('starting router..')
r.run()
except KeyboardInterrupt:
logger.info('shutting down...')
logger.info('Shutting down')
def main():
p = get_argument_parser()
p = ArgumentParser(description=textwrap.dedent('''\
example usage:
$ cif-httpd -d
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='cif-httpd',
parents=[p])
router_address = app.config.get('CIF_ROUTER_ADDR', ROUTER_ADDR)
p.add_argument("--router",
help="specify router frontend [default %(default)s]",
default=router_address)
p.add_argument('--token',
help="specify cif-httpd token [default %(default)s]",
default=TOKEN)
p.add_argument(
'--listen',
help='specify the interface to listen on [default %(default)s]',
default=HTTP_LISTEN)
p.add_argument('--listen-port',
help='specify the port to listen on [default %(default)s]',
default=HTTP_LISTEN_PORT)
p.add_argument('--fdebug', action='store_true')
args = p.parse_args()
setup_logging(args)
logger = logging.getLogger(__name__)
logger.info('loglevel is: {}'.format(
logging.getLevelName(logger.getEffectiveLevel())))
setup_signals(__name__)
setup_runtime_path(args.runtime_path)
try:
logger.info('pinging router...')
app.config["SECRET_KEY"] = os.urandom(1024)
logger.info('starting up...')
app.run(host=args.listen,
port=args.listen_port,
debug=args.fdebug,
threaded=True)
except KeyboardInterrupt:
logger.info('shutting down...')
raise SystemExit
def main():
p = get_argument_parser()
p = ArgumentParser(
description=textwrap.dedent('''\
example usage:
$ cif-hunter -d
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='cif-hunter',
parents=[p],
)
p.add_argument('--remote',
help="cif-router hunter address [default %(default)s]",
default=HUNTER_ADDR)
p.add_argument('--router',
help='cif-router front end address [default %(default)s]',
default=ROUTER_ADDR)
p.add_argument('--token',
help='specify cif-hunter token [default %(default)s]',
default=TOKEN)
p.add_argument('--config', default=CONFIG_PATH)
args = p.parse_args()
setup_logging(args)
o = read_config(args)
options = vars(args)
for v in options:
if options[v] is None:
options[v] = o.get(v)
logger = logging.getLogger(__name__)
logger.info('loglevel is: {}'.format(
logging.getLevelName(logger.getEffectiveLevel())))
setup_signals(__name__)
with Hunter(remote=options.get('remote'),
router=args.router,
token=options.get('token')) as h:
try:
logger.info('starting up...')
h.start()
except KeyboardInterrupt:
logging.info("shutting down...")
h.stop()
def main():
p = get_argument_parser()
p = ArgumentParser(
description=textwrap.dedent('''\
Env Variables:
CIF_RUNTIME_PATH
CIF_ROUTER_ADDR
CIF_HUNTER_ADDR
CIF_STORE_ADDR
example usage:
$ cif-router --listen 0.0.0.0 -d
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='cif-router',
parents=[p]
)
p.add_argument('--listen', help='address to listen on [default: %(default)s]', default=ROUTER_ADDR)
p.add_argument('--hunter', help='address hunters listen on on [default: %(default)s]', default=HUNTER_ADDR)
p.add_argument("--store", help="specify a store address [default: %(default)s]",
default=STORE_ADDR)
p.add_argument('--p2p', action='store_true', help='enable experimental p2p support')
args = p.parse_args()
setup_logging(args)
logger = logging.getLogger(__name__)
logger.info('loglevel is: {}'.format(logging.getLevelName(logger.getEffectiveLevel())))
setup_signals(__name__)
setup_runtime_path(args.runtime_path)
with Router(listen=args.listen, hunter=args.hunter, store=args.store, p2p=args.p2p) as r:
try:
logger.info('starting router..')
r.run()
except KeyboardInterrupt:
logger.info('shutting down...')
logger.info('Shutting down')
def main():
p = get_argument_parser()
p = ArgumentParser(
description=textwrap.dedent('''\
example usage:
$ cif-httpd -d
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='cif-httpd',
parents=[p]
)
router_address = app.config.get('CIF_ROUTER_ADDR', ROUTER_ADDR)
p.add_argument("--router", help="specify router frontend [default %(default)s]", default=router_address)
p.add_argument('--token', help="specify cif-httpd token [default %(default)s]", default=TOKEN)
p.add_argument('--listen', help='specify the interface to listen on [default %(default)s]', default=HTTP_LISTEN)
p.add_argument('--listen-port', help='specify the port to listen on [default %(default)s]',
default=HTTP_LISTEN_PORT)
p.add_argument('--fdebug', action='store_true')
args = p.parse_args()
setup_logging(args)
logger = logging.getLogger(__name__)
logger.info('loglevel is: {}'.format(logging.getLevelName(logger.getEffectiveLevel())))
setup_signals(__name__)
setup_runtime_path(args.runtime_path)
try:
logger.info('pinging router...')
app.config["SECRET_KEY"] = os.urandom(1024)
logger.info('starting up...')
app.run(host=args.listen, port=args.listen_port, debug=args.fdebug, threaded=True)
except KeyboardInterrupt:
logger.info('shutting down...')
raise SystemExit
def main():
p = get_argument_parser()
p = ArgumentParser(description=textwrap.dedent('''\
example usage:
$ cif-httpd -d
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='cif-httpd',
parents=[p])
p.add_argument('--fdebug', action='store_true')
args = p.parse_args()
setup_logging(args)
logger.info('loglevel is: {}'.format(
logging.getLevelName(logger.getEffectiveLevel())))
setup_runtime_path(args.runtime_path)
if not args.fdebug:
# http://stackoverflow.com/a/789383/7205341
pid = str(os.getpid())
logger.debug("pid: %s" % pid)
try:
logger.info('pinging router...')
logger.info('starting up...')
app.run(host=HTTP_LISTEN, port=HTTP_LISTEN_PORT, debug=args.fdebug)
except KeyboardInterrupt:
logger.info('shutting down...')
except Exception as e:
logger.critical(e)
traceback.print_exc()
def main():
p = get_argument_parser()
p = ArgumentParser(
description=textwrap.dedent('''\
example usage:
$ cif-hunter -d
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='cif-hunter',
parents=[p],
)
p.add_argument('--remote', help="cif-router hunter address [default %(default)s]", default=HUNTER_ADDR)
p.add_argument('--router', help='cif-router front end address [default %(default)s]', default=ROUTER_ADDR)
p.add_argument('--token', help='specify cif-hunter token [default %(default)s]', default=TOKEN)
p.add_argument('--config', default=CONFIG_PATH)
args = p.parse_args()
setup_logging(args)
o = read_config(args)
options = vars(args)
for v in options:
if options[v] is None:
options[v] = o.get(v)
logger = logging.getLogger(__name__)
logger.info('loglevel is: {}'.format(logging.getLevelName(logger.getEffectiveLevel())))
setup_signals(__name__)
with Hunter(remote=options.get('remote'), router=args.router, token=options.get('token')) as h:
try:
logger.info('starting up...')
h.start()
except KeyboardInterrupt:
logging.info("shutting down...")
h.stop()
def main():
p = get_argument_parser()
p = ArgumentParser(
description=textwrap.dedent('''\
Env Variables:
CIF_RUNTIME_PATH
CIF_ROUTER_CONFIG_PATH
CIF_ROUTER_ADDR
CIF_HUNTER_ADDR
CIF_HUNTER_TOKEN
CIF_HUNTER_THREADS
CIF_GATHERER_THREADS
CIF_STORE_ADDR
example usage:
$ cif-router --listen 0.0.0.0 -d
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='cif-router',
parents=[p]
)
p.add_argument('--config', help='specify config path [default: %(default)s', default=CONFIG_PATH)
p.add_argument('--listen', help='address to listen on [default: %(default)s]', default=ROUTER_ADDR)
p.add_argument('--gatherer-threads', help='specify number of gatherer threads to use [default: %(default)s]',
default=GATHERER_THREADS)
p.add_argument('--hunter', help='address hunters listen on on [default: %(default)s]', default=HUNTER_ADDR)
p.add_argument('--hunter-token', help='specify token for hunters to use [default: %(default)s]',
default=HUNTER_TOKEN)
p.add_argument('--hunter-threads', help='specify number of hunter threads to use [default: %(default)s]',
default=HUNTER_THREADS)
p.add_argument("--store-address", help="specify the store address cif-router is listening on[default: %("
"default)s]", default=STORE_ADDR)
p.add_argument("--store", help="specify a store type {} [default: %(default)s]".format(', '.join(STORE_PLUGINS)),
default=STORE_DEFAULT)
p.add_argument('--store-nodes', help='specify storage nodes address [default: %(default)s]', default=STORE_NODES)
p.add_argument('--logging-ignore', help='set logging to WARNING for specific modules')
p.add_argument('--pidfile', help='specify pidfile location [default: %(default)s]', default=PIDFILE)
args = p.parse_args()
setup_logging(args)
logger = logging.getLogger(__name__)
logger.info('loglevel is: {}'.format(logging.getLevelName(logger.getEffectiveLevel())))
if args.logging_ignore:
to_ignore = args.logging_ignore.split(',')
for i in to_ignore:
logging.getLogger(i).setLevel(logging.WARNING)
o = read_config(args)
options = vars(args)
for v in options:
if options[v] is None:
options[v] = o.get(v)
setup_runtime_path(args.runtime_path)
setup_signals(__name__)
# http://stackoverflow.com/a/789383/7205341
pid = str(os.getpid())
logger.debug("pid: %s" % pid)
if os.path.isfile(args.pidfile):
logger.critical("%s already exists, exiting" % args.pidfile)
raise SystemExit
try:
pidfile = open(args.pidfile, 'w')
pidfile.write(pid)
pidfile.close()
except PermissionError as e:
logger.error('unable to create pid %s' % args.pidfile)
with Router(listen=args.listen, hunter=args.hunter, store_type=args.store, store_address=args.store_address,
store_nodes=args.store_nodes, hunter_token=args.hunter_token, hunter_threads=args.hunter_threads,
gatherer_threads=args.gatherer_threads) as r:
try:
logger.info('starting router..')
r.start()
except KeyboardInterrupt:
# todo - signal to threads to shut down and wait for them to finish
logger.info('shutting down via SIGINT...')
except SystemExit:
logger.info('shutting down via SystemExit...')
except Exception as e:
logger.critical(e)
traceback.print_exc()
r.stop()
logger.info('Shutting down')
if os.path.isfile(args.pidfile):
os.unlink(args.pidfile)
def main():
p = get_argument_parser()
p = ArgumentParser(
description=textwrap.dedent('''\
Env Variables:
CIF_RUNTIME_PATH
CIF_STORE_ADDR
example usage:
$ cif-store -d
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='cif-store',
parents=[p]
)
p.add_argument("--store-address", help="specify the store address cif-router is listening on[default: %("
"default)s]", default=STORE_ADDR)
p.add_argument("--store", help="specify a store type {} [default: %(default)s]".format(', '.join(STORE_PLUGINS)),
default=STORE_DEFAULT)
p.add_argument('--store-type')
p.add_argument('--nodes')
p.add_argument('--config', help='specify config path [default %(default)s]', default=CONFIG_PATH)
p.add_argument('--token-create-admin', help='generate an admin token')
p.add_argument('--token-create-smrt')
p.add_argument('--token-create-smrt-remote', default=REMOTE_ADDR)
p.add_argument('--token-create-hunter')
p.add_argument('--remote', help='specify remote')
args = p.parse_args()
setup_logging(args)
logger = logging.getLogger(__name__)
logger.info('loglevel is: {}'.format(logging.getLevelName(logger.getEffectiveLevel())))
setup_signals(__name__)
if args.token_create_smrt:
with Store(store_type=args.store, nodes=args.nodes) as s:
t = s.token_create_smrt()
if t:
data = {
'token': t.decode('utf-8'),
}
if args.remote:
data['remote'] = args.remote
with open(args.token_create_smrt, 'w') as f:
f.write(yaml.dump(data, default_flow_style=False))
logger.info('token config generated: {}'.format(args.token_create_smrt))
else:
logger.error('token not created')
if args.token_create_hunter:
with Store(store_type=args.store, nodes=args.nodes) as s:
t = s.token_create_hunter()
if t:
data = {
'hunter_token': t.decode('utf-8'),
}
with open(args.token_create_hunter, 'w') as f:
f.write(yaml.dump(data, default_flow_style=False))
logger.info('token config generated: {}'.format(args.token_create_hunter))
else:
logger.error('token not created')
if args.token_create_admin:
with Store(store_type=args.store, nodes=args.nodes) as s:
t = s.token_create_admin()
if t:
data = {
'token': t.decode('utf-8'),
}
with open(args.token_create_admin, 'w') as f:
f.write(yaml.dump(data, default_flow_style=False))
logger.info('token config generated: {}'.format(args.token_create_admin))
else:
logger.error('token not created')
def main():
p = get_argument_parser()
p = ArgumentParser(description=textwrap.dedent('''\
example usage:
$ cif-httpd -d
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='cif-httpd',
parents=[p])
router_address = app.config.get('CIF_ROUTER_ADDR', ROUTER_ADDR)
p.add_argument("--router",
help="specify router frontend [default %(default)s]",
default=router_address)
p.add_argument(
'--listen',
help='specify the interface to listen on [default %(default)s]',
default=HTTP_LISTEN)
p.add_argument('--listen-port',
help='specify the port to listen on [default %(default)s]',
default=HTTP_LISTEN_PORT)
p.add_argument('--pidfile',
help='specify pidfile location [default: %(default)s]',
default=PIDFILE)
p.add_argument('--fdebug', action='store_true')
args = p.parse_args()
setup_logging(args)
logger = logging.getLogger(__name__)
if TRACE:
logger.setLevel(logging.DEBUG)
logger.info('loglevel is: {}'.format(
logging.getLevelName(logger.getEffectiveLevel())))
setup_signals(__name__)
setup_runtime_path(args.runtime_path)
if not args.fdebug:
# http://stackoverflow.com/a/789383/7205341
pid = str(os.getpid())
logger.debug("pid: %s" % pid)
if os.path.isfile(args.pidfile):
logger.critical("%s already exists, exiting" % args.pidfile)
raise SystemExit
try:
pidfile = open(args.pidfile, 'w')
pidfile.write(pid)
pidfile.close()
except PermissionError as e:
logger.error('unable to create pid %s' % args.pidfile)
try:
logger.info('pinging router...')
logger.info('starting up...')
app.run(host=args.listen,
port=args.listen_port,
debug=args.fdebug,
threaded=True,
extra_files=extra_files)
except KeyboardInterrupt:
logger.info('shutting down...')
except Exception as e:
logger.critical(e)
traceback.print_exc()
if os.path.isfile(args.pidfile):
os.unlink(args.pidfile)
def main():
p = get_argument_parser()
p = ArgumentParser(
description=textwrap.dedent('''\
example usage:
$ cif -q example.org -d
$ cif --search 1.2.3.0/24
$ cif --ping
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='cif',
parents=[p]
)
p.add_argument('--token', help='specify api token', default=TOKEN)
p.add_argument('--remote', help='specify API remote [default %(default)s]', default=REMOTE_ADDR)
p.add_argument('-p', '--ping', action="store_true") # meg?
p.add_argument('-q', '--search', help="search")
p.add_argument('--itype', help='filter by indicator type') ## need to fix sqlite for non-ascii stuff first
p.add_argument("--submit", action="store_true", help="submit an indicator")
p.add_argument('--limit', help='limit results [default %(default)s]', default=SEARCH_LIMIT)
p.add_argument('-n', '--nolog', help='do not log search', action='store_true')
p.add_argument('-f', '--format', help='specify output format [default: %(default)s]"', default=FORMAT, choices=FORMATS.keys())
p.add_argument('--indicator')
p.add_argument('--tags', nargs='+')
p.add_argument('--provider')
p.add_argument("--zmq", help="use zmq as a transport instead of http", action="store_true")
p.add_argument('--config', help='specify config file [default %(default)s]', default=CONFIG_PATH)
args = p.parse_args()
setup_logging(args)
logger = logging.getLogger(__name__)
o = read_config(args)
options = vars(args)
for v in options:
if options[v] is None:
options[v] = o.get(v)
if not options.get('token'):
raise RuntimeError('missing --token')
verify_ssl = True
if o.get('no_verify_ssl') or options.get('no_verify_ssl'):
verify_ssl = False
options = vars(args)
if options.get("zmq"):
from cifsdk.client.zeromq import ZMQ as ZMQClient
cli = ZMQClient(**options)
else:
from cifsdk.client.http import HTTP as HTTPClient
cli = HTTPClient(args.remote, args.token, verify_ssl=verify_ssl)
if options.get('ping'):
logger.info('running ping')
for num in range(0, 4):
ret = cli.ping()
if ret != 0:
print("roundtrip: {} ms".format(ret))
select.select([], [], [], 1)
else:
logger.error('ping failed')
raise RuntimeError
elif options.get('itype'):
logger.info('searching for {}'.format(options['itype']))
try:
rv = cli.search({
'itype': options['itype'],
'limit': options['limit'],
'provider': options.get('provider')
})
except AuthError as e:
logger.error('unauthorized')
except RuntimeError as e:
import traceback
traceback.print_exc()
logger.error(e)
else:
print(FORMATS[options.get('format')](data=rv))
elif options.get('search'):
logger.info("searching for {0}".format(options.get("search")))
try:
rv = cli.indicators_search({
'indicator': options['search'],
'limit': options['limit'],
'nolog': options['nolog']
}
)
except RuntimeError as e:
import traceback
traceback.print_exc()
logger.error(e)
except AuthError as e:
logger.error('unauthorized')
else:
print(FORMATS[options.get('format')](data=rv))
elif options.get("submit"):
logger.info("submitting {0}".format(options.get("submit")))
rv = cli.submit(indicator=args.indicator, tags=args.tags)
def main():
p = get_argument_parser()
p = ArgumentParser(description=textwrap.dedent('''\
example usage:
$ cif -q example.org -d
$ cif --search 1.2.3.0/24
$ cif --ping
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='cif',
parents=[p])
p.add_argument('--token', help='specify api token', default=TOKEN)
p.add_argument('--remote',
help='specify API remote [default %(default)s]',
default=REMOTE_ADDR)
p.add_argument('-p', '--ping', action="store_true") # meg?
p.add_argument('-q', '--search', help="search")
p.add_argument('--itype', help='filter by indicator type'
) ## need to fix sqlite for non-ascii stuff first
p.add_argument("--submit", action="store_true", help="submit an indicator")
p.add_argument('--limit',
help='limit results [default %(default)s]',
default=SEARCH_LIMIT)
p.add_argument('--reporttime', help='specify reporttime filter')
p.add_argument('-n',
'--nolog',
help='do not log search',
action='store_true')
p.add_argument('-f',
'--format',
help='specify output format [default: %(default)s]"',
default=FORMAT,
choices=FORMATS.keys())
p.add_argument('--indicator')
p.add_argument('--tags', nargs='+')
p.add_argument('--provider')
p.add_argument('--confidence', help="specify confidence level")
p.add_argument("--zmq",
help="use zmq as a transport instead of http",
action="store_true")
p.add_argument('--config',
help='specify config file [default %(default)s]',
default=CONFIG_PATH)
p.add_argument('--feed', action='store_true')
p.add_argument('--no-verify-ssl', action='store_true')
p.add_argument('--last-day',
action="store_true",
help='auto-sets reporttime to 23 hours and 59 seconds ago '
'(current time UTC) and reporttime-end to "now"')
p.add_argument(
'--last-hour',
action='store_true',
help='auto-sets reporttime to the beginning of the previous full'
' hour and reporttime-end to end of previous full hour')
p.add_argument('--days', help='filter results within last X days')
p.add_argument('--today',
help='auto-sets reporttime to today, 00:00:00Z (UTC)',
action='store_true')
args = p.parse_args()
setup_logging(args)
logger = logging.getLogger(__name__)
o = read_config(args)
options = vars(args)
for v in options:
if v == 'remote' and options[v] == REMOTE_ADDR and o.get('remote'):
options[v] = o['remote']
if options[v] is None:
options[v] = o.get(v)
if not options.get('token'):
raise RuntimeError('missing --token')
verify_ssl = True
if o.get('no_verify_ssl') or options.get('no_verify_ssl'):
verify_ssl = False
if options.get("zmq"):
from cifsdk.client.zeromq import ZMQ as ZMQClient
cli = ZMQClient(**options)
else:
from cifsdk.client.http import HTTP as HTTPClient
if args.remote == 'https://localhost':
verify_ssl = False
cli = HTTPClient(args.remote, args.token, verify_ssl=verify_ssl)
if options.get('ping'):
logger.info('running ping')
for num in range(0, 4):
ret = cli.ping()
if ret != 0:
print("roundtrip: {} ms".format(ret))
select.select([], [], [], 1)
else:
logger.error('ping failed')
raise RuntimeError
raise SystemExit
if options.get("submit"):
logger.info("submitting {0}".format(options.get("submit")))
i = Indicator(indicator=args.indicator,
tags=args.tags,
confidence=args.confidence)
rv = cli.indicators_create(i)
logger.info('success id: {}'.format(rv))
raise SystemExit
filters = {
'itype': options['itype'],
'limit': options['limit'],
'provider': options.get('provider'),
'indicator': options.get('search'),
'nolog': options['nolog'],
'tags': options['tags'],
'confidence': options.get('confidence')
}
if args.last_day:
filters['days'] = '1'
if args.last_hour:
filters['hours'] = '1'
if args.days:
filters['days'] = args.days
if args.today:
now = arrow.utcnow()
filters['reporttime'] = '{0}Z'.format(
now.format('YYYY-MM-DDT00:00:00'))
if options.get('feed'):
if not filters.get('confidence'):
filters['confidence'] = 8
if not filters.get('reporttime') and not filters.get(
'day') and not filters.get('hour'):
filters['days'] = FEED_DAYS_LIMIT
if args.limit == SEARCH_LIMIT:
filters['limit'] = FEED_LIMIT
try:
rv = cli.feed(filters=filters)
except AuthError as e:
logger.error('unauthorized')
else:
print(FORMATS[options.get('format')](data=rv))
raise SystemExit
try:
rv = cli.search(filters)
except AuthError as e:
logger.error('unauthorized')
else:
print(FORMATS[options.get('format')](data=rv))
def main(): # pragma: no cover
p = get_argument_parser()
p = ArgumentParser(description=textwrap.dedent('''\
example usage:
$ cif -q example.org -d
$ cif --search 1.2.3.0/24
$ cif --profile zeek
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='cif',
parents=[p])
p.add_argument('--remote',
help='specify API remote [default %(default)s]',
default=REMOTE)
p.add_argument('--no-verify-ssl', action='store_true')
p.add_argument("--create", action="store_true", help="create an indicator")
p.add_argument('--delete', action='store_true')
p.add_argument('-q', '--search', help="search")
p.add_argument('--itype', help='filter by indicator type')
p.add_argument('--reported_at', help='specify reported_at filter')
p.add_argument('-n',
'--nolog',
help='do not log search',
action='store_true')
p.add_argument('-f',
'--format',
help='specify output format [default: %(default)s]"',
default=FORMAT,
choices=FORMATS.keys())
p.add_argument('--indicator',
help='indicator (ip, url, etc..) '
'to search for')
p.add_argument('--confidence', help="specify confidence level")
p.add_argument('--tags', nargs='+', help='filter by tag(s)')
p.add_argument('--provider', help='provider to filter by')
p.add_argument('--asn', help='filter by asn')
p.add_argument('--cc', help='filter by country code')
p.add_argument('--asn-name', help='filter by asn name')
p.add_argument('--rdata', help='filter by rdata')
p.add_argument('--groups', help='filter by group(s)')
p.add_argument('--days', help='filter results within last X days')
p.add_argument('--today',
action='store_true',
help='auto-sets reported_at to today, 00:00:00Z (UTC)')
p.add_argument('--limit',
help='limit results [default %(default)s]',
default=SEARCH_LIMIT)
p.add_argument('--columns',
default=','.join(COLUMNS),
help='specify output columns [default %(default)s]')
p.add_argument('--no-feed',
action='store_true',
help='return a non-filtered dataset (no whitelist applied)')
p.add_argument('--profile',
help='specify feed profile',
choices=PROFILES.keys())
args = p.parse_args()
setup_logging(args)
opts = vars(args)
options = {}
for k, v in opts.items():
if v:
options[k] = v
if args.remote.startswith('http'):
verify_ssl = True
if args.no_verify_ssl:
verify_ssl = False
if args.remote == 'https://localhost':
verify_ssl = False
cli = Client(verify_ssl=verify_ssl)
else:
from cifsdk.client.zeromq import ZMQ
cli = ZMQ()
filters = {e: options.get(e) for e in VALID_FILTERS}
if args.search:
filters['indicator'] = args.search
for k, v in filters.items():
if v is True:
filters[k] = 1
if v is False:
filters[k] = 0
if options.get("create"):
_create(cli, args, filters)
raise SystemExit
if options.get("delete"):
_delete(cli, args, filters)
raise SystemExit
if not sys.stdin.isatty():
buffer = sys.stdin.read().rstrip("\n").split("\n")
filters = [{'indicator': i} for i in buffer]
_search(cli, args, options, filters)
def main():
p = get_argument_parser()
p = ArgumentParser(
description=textwrap.dedent('''\
Env Variables:
CIF_TOKEN
CIF_REMOTE
example usage:
$ cif-tail
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='cif-tail',
parents=[p],
)
p.add_argument('--no-verify-ssl',
help='turn TLS/SSL verification OFF',
action='store_true')
p.add_argument('--format', default='table')
p.add_argument('--cycle',
help='specify a cycle in which to run',
default=5)
p.add_argument('--filters',
help='specify data filters to use',
default='itype=ipv4,confidence=7,limit=10')
p.add_argument('--remote', default=REMOTE_ADDR)
p.add_argument('--token', default=TOKEN)
p.add_argument('--start',
default=arrow.get((arrow.utcnow().timestamp - 420)))
args = p.parse_args()
# setup logging
setup_logging(args)
verify_ssl = True
if args.no_verify_ssl:
verify_ssl = False
filters = {}
for k in args.filters.split(','):
kk, v = k.split('=')
filters[kk] = v
remote = args.remote
token = args.token
client = Client(remote, token, verify_ssl=verify_ssl)
start = args.start
start = arrow.get(start)
cycle = (int(args.cycle) * 60)
# we want a 120s buffer for things that are being generated "now"
end = arrow.get((arrow.utcnow().timestamp - 120))
while True:
logger.debug('now: %s' % arrow.utcnow())
start = start.strftime('%Y-%m-%dT%H:%M:%S') + 'Z'
end = end.strftime('%Y-%m-%dT%H:%M:%S') + 'Z'
filters['reporttime'] = '{},{}'.format(start, end)
logger.debug('searching {} - {}'.format(start, end))
resp = client.indicators_search(filters)
if args.format == 'csv':
for l in get_lines_csv(resp):
print(l)
else:
for l in get_lines_table(resp):
print(l)
logger.debug('sleeping for {}m'.format(args.cycle))
sleep(cycle)
# todo- this needs some work, maybe use last record if there was one?
# what if there wasn't?
start = arrow.get(arrow.get(end).timestamp + 1)
end = arrow.get((arrow.utcnow().timestamp - 120))
def main():
p = get_argument_parser()
p = ArgumentParser(description=textwrap.dedent('''\
example usage:
$ cif -q example.org -d
$ cif --search 1.2.3.0/24
$ cif --ping
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='cif',
parents=[p])
p.add_argument('--token', help='specify api token', default=TOKEN)
p.add_argument('--remote',
help='specify API remote [default %(default)s]',
default=REMOTE_ADDR)
p.add_argument('-p', '--ping', action="store_true") # meg?
p.add_argument('--ping-indef', action="store_true")
p.add_argument('-q', '--search', help="search")
p.add_argument('--itype', help='filter by indicator type'
) ## need to fix sqlite for non-ascii stuff first
p.add_argument("--submit", action="store_true", help="submit an indicator")
p.add_argument('--limit',
help='limit results [default %(default)s]',
default=SEARCH_LIMIT)
p.add_argument('--reporttime', help='specify reporttime filter')
p.add_argument('-n',
'--nolog',
help='do not log search',
action='store_true')
p.add_argument('-f',
'--format',
help='specify output format [default: %(default)s]"',
default=FORMAT,
choices=FORMATS.keys())
p.add_argument('--indicator')
p.add_argument('--tags', nargs='+')
p.add_argument('--provider')
p.add_argument('--confidence', help="specify confidence level")
p.add_argument('--tlp', help="specify traffic light protocol")
p.add_argument("--zmq",
help="use zmq as a transport instead of http",
action="store_true")
p.add_argument('--config',
help='specify config file [default %(default)s]',
default=CONFIG_PATH)
p.add_argument('--feed', action='store_true')
p.add_argument('--no-verify-ssl', action='store_true')
p.add_argument('--last-day',
action="store_true",
help='auto-sets reporttime to 23 hours and 59 seconds ago '
'(current time UTC) and reporttime-end to "now"')
p.add_argument(
'--last-hour',
action='store_true',
help='auto-sets reporttime to the beginning of the previous full'
' hour and reporttime-end to end of previous full hour')
p.add_argument('--days', help='filter results within last X days')
p.add_argument('--today',
help='auto-sets reporttime to today, 00:00:00Z (UTC)',
action='store_true')
p.add_argument('--columns',
help='specify output columns [default %(default)s]',
default=','.join(COLUMNS))
p.add_argument('--asn')
p.add_argument('--cc')
p.add_argument('--asn-desc')
p.add_argument('--rdata')
p.add_argument('--no-feed', action='store_true')
p.add_argument('--region')
p.add_argument('--groups', help='specify groups filter (csv)')
p.add_argument('--delete', action='store_true')
p.add_argument('--id')
args = p.parse_args()
setup_logging(args)
logger = logging.getLogger(__name__)
o = read_config(args)
options = vars(args)
for v in options:
if v == 'remote' and options[v] == REMOTE_ADDR and o.get('remote'):
options[v] = o['remote']
if v == 'token' and o.get('token'):
options[v] = o['token']
if options[v] is None or options[v] == '':
options[v] = o.get(v)
if not options.get('token'):
raise RuntimeError('missing --token')
verify_ssl = True
if o.get('no_verify_ssl') or options.get('no_verify_ssl'):
verify_ssl = False
if options.get("zmq"):
from cifsdk.client.zeromq import ZMQ as ZMQClient
cli = ZMQClient(**options)
else:
from cifsdk.client.http import HTTP as HTTPClient
if args.remote == 'https://localhost':
verify_ssl = False
cli = HTTPClient(args.remote, args.token, verify_ssl=verify_ssl)
if options.get('ping') or options.get('ping_indef'):
logger.info('running ping')
n = 4
if args.ping_indef:
n = 999
try:
for num in range(0, n):
ret = cli.ping()
if ret != 0:
print("roundtrip: {} ms".format(ret))
select.select([], [], [], 1)
from time import sleep
sleep(1)
else:
logger.error('ping failed')
raise RuntimeError
except KeyboardInterrupt:
pass
raise SystemExit
if options.get("submit"):
print("submitting {0}".format(options.get("submit")))
i = Indicator(indicator=args.indicator,
tags=args.tags,
confidence=args.confidence,
group=args.groups,
tlp=args.tlp,
provider=args.provider)
rv = cli.indicators_create(i)
print('success id: {}\n'.format(rv))
raise SystemExit
filters = {
'itype': options['itype'],
'limit': options['limit'],
'provider': options.get('provider'),
'indicator': options.get('search') or options.get('indicator'),
'nolog': options['nolog'],
'tags': options['tags'],
'confidence': options.get('confidence'),
'asn': options.get('asn'),
'asn_desc': options.get('asn_desc'),
'cc': options.get('cc'),
'region': options.get('region'),
'rdata': options.get('rdata'),
'reporttime': options.get('reporttime'),
'groups': options.get('groups'),
'tlp': options.get('tlp')
}
if args.last_day:
filters['days'] = '1'
del filters['reporttime']
if args.last_hour:
filters['hours'] = '1'
del filters['reporttime']
if args.days:
filters['days'] = args.days
del filters['reporttime']
if args.today:
now = arrow.utcnow()
filters['reporttime'] = '{0}Z'.format(
now.format('YYYY-MM-DDT00:00:00'))
if filters.get('itype') and not filters.get('search') and not args.no_feed:
logger.info('setting feed flag by default, use --no-feed to override')
options['feed'] = True
if options.get("delete"):
if args.id:
filters = {'id': args.id}
filters = {f: filters[f] for f in filters if filters.get(f)}
print("deleting {0}".format(filters))
rv = cli.indicators_delete(filters)
print('deleted: {}'.format(rv))
raise SystemExit
if options.get('feed'):
if not filters.get('itype') and not ADVANCED:
print('\nmissing --itype\n\n')
raise SystemExit
if not filters.get('tags') and not ADVANCED:
print(
'\nmissing --tags [phishing|malware|botnet|scanner|pdns|whitelist|...]\n\n'
)
raise SystemExit
if not filters.get('confidence'):
filters['confidence'] = 8
if args.limit == SEARCH_LIMIT:
filters['limit'] = FEED_LIMIT
try:
rv = cli.feed(filters=filters)
except AuthError as e:
logger.error('unauthorized')
except KeyboardInterrupt:
pass
except Exception as e:
logger.error(e)
else:
print(FORMATS[options.get('format')](data=rv,
cols=args.columns.split(',')))
raise SystemExit
try:
rv = cli.search(filters)
except AuthError as e:
logger.error('unauthorized')
except KeyboardInterrupt:
pass
except Exception as e:
import traceback
traceback.print_exc()
logger.error(e)
else:
print(FORMATS[options.get('format')](data=rv,
cols=args.columns.split(',')))
def main():
p = get_argument_parser()
p = ArgumentParser(
description=textwrap.dedent('''\
Env Variables:
CIF_TOKEN
CIF_REMOTE
example usage:
$ cif-tail
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='cif-tail',
parents=[p],
)
p.add_argument('--no-verify-ssl', help='turn TLS/SSL verification OFF', action='store_true')
p.add_argument('--format', default='table')
p.add_argument('--cycle', help='specify a cycle in which to run', default=5)
p.add_argument('--filters', help='specify data filters to use', default='itype=ipv4,confidence=7,limit=10')
p.add_argument('--remote', default=REMOTE_ADDR)
p.add_argument('--token', default=TOKEN)
p.add_argument('--start', default=arrow.get((arrow.utcnow().timestamp - 420)))
args = p.parse_args()
# setup logging
setup_logging(args)
verify_ssl = not args.no_verify_ssl
filters = {}
for k in args.filters.split(','):
kk, v = k.split('=')
filters[kk] = v
remote = args.remote
token = args.token
client = Client(remote, token, verify_ssl=verify_ssl)
start = args.start
start = arrow.get(start)
cycle = (int(args.cycle) * 60)
delay = Delayer(upper=cycle)
# we want a 120s buffer for things that are being generated "now"
end = arrow.get((arrow.utcnow().timestamp - 120))
while True:
logger.debug('now: %s' % arrow.utcnow())
start = start.strftime('%Y-%m-%dT%H:%M:%S') + 'Z'
end = end.strftime('%Y-%m-%dT%H:%M:%S') + 'Z'
filters['reporttime'] = '{},{}'.format(start, end)
logger.debug('searching {} - {}'.format(start, end))
try:
resp = client.indicators_search(filters)
except Exception:
logger.exception("CIF API Error")
resp = []
if resp:
delay.reset()
if args.format == 'csv':
for l in get_lines_csv(resp):
print(l)
else:
for l in get_lines_table(resp):
print(l)
delay.sleep()
# todo- this needs some work, maybe use last record if there was one?
# what if there wasn't?
start = arrow.get(arrow.get(end).timestamp + 1)
end = arrow.get((arrow.utcnow().timestamp - 120))
def main():
p = get_argument_parser()
p = ArgumentParser(
description=textwrap.dedent('''\
Env Variables:
CIF_RUNTIME_PATH
CIF_ROUTER_CONFIG_PATH
CIF_ROUTER_ADDR
CIF_HUNTER_ADDR
CIF_HUNTER_TOKEN
CIF_HUNTER_THREADS
CIF_GATHERER_THREADS
CIF_STORE_ADDR
example usage:
$ cif-router --listen 0.0.0.0 -d
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='cif-router',
parents=[p]
)
p.add_argument('--config', help='specify config path [default: %(default)s', default=CONFIG_PATH)
p.add_argument('--listen', help='address to listen on [default: %(default)s]', default=ROUTER_ADDR)
p.add_argument('--gatherer-threads', help='specify number of gatherer threads to use [default: %(default)s]',
default=GATHERER_THREADS)
p.add_argument('--hunter', help='address hunters listen on on [default: %(default)s]', default=HUNTER_ADDR)
p.add_argument('--hunter-token', help='specify token for hunters to use [default: %(default)s]',
default=HUNTER_TOKEN)
p.add_argument('--hunter-threads', help='specify number of hunter threads to use [default: %(default)s]',
default=HUNTER_THREADS)
p.add_argument("--store-address", help="specify the store address cif-router is listening on[default: %("
"default)s]", default=STORE_ADDR)
p.add_argument("--store", help="specify a store type {} [default: %(default)s]".format(', '.join(STORE_PLUGINS)),
default=STORE_DEFAULT)
p.add_argument('--store-nodes', help='specify storage nodes address [default: %(default)s]', default=STORE_NODES)
p.add_argument('--p2p', action='store_true', help='enable experimental p2p support')
args = p.parse_args()
setup_logging(args)
logger = logging.getLogger(__name__)
logger.info('loglevel is: {}'.format(logging.getLevelName(logger.getEffectiveLevel())))
o = read_config(args)
options = vars(args)
for v in options:
if options[v] is None:
options[v] = o.get(v)
setup_signals(__name__)
setup_runtime_path(args.runtime_path)
with Router(listen=args.listen, hunter=args.hunter, store_type=args.store, store_address=args.store_address,
store_nodes=args.store_nodes, p2p=args.p2p, hunter_token=args.hunter_token, hunter_threads=args.hunter_threads,
gatherer_threads=args.gatherer_threads) as r:
try:
logger.info('starting router..')
r.start()
except KeyboardInterrupt:
# todo - signal to threads to shut down and wait for them to finish
logger.info('shutting down...')
logger.info('Shutting down')
def main():
p = get_argument_parser()
p = ArgumentParser(description=textwrap.dedent('''\
Env Variables:
CIF_RUNTIME_PATH
CIF_STORE_ADDR
example usage:
$ cif-store -d
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='cif-store',
parents=[p])
p.add_argument(
"--store-address",
help="specify the store address cif-router is listening on[default: %("
"default)s]",
default=STORE_ADDR)
p.add_argument(
"--store",
help="specify a store type {} [default: %(default)s]".format(
', '.join(STORE_PLUGINS)),
default=STORE_DEFAULT)
p.add_argument('--nodes')
p.add_argument('--config',
help='specify config path [default %(default)s]',
default=CONFIG_PATH)
p.add_argument('--token-create-admin',
help='generate an admin token',
action="store_true")
p.add_argument('--token-create-smrt', action="store_true")
p.add_argument('--token-create-smrt-remote', default=REMOTE_ADDR)
p.add_argument('--token-create-hunter', action="store_true")
p.add_argument('--token-create-httpd', action="store_true")
p.add_argument('--config-path', help='store the token as a config')
p.add_argument('--token', help='specify the token to use', default=None)
p.add_argument(
'--token-groups',
help="specify groups associated with token [default %(default)s]'",
default='everyone')
p.add_argument('--remote', help='specify remote')
args = p.parse_args()
groups = args.token_groups.split(',')
setup_logging(args)
logger = logging.getLogger(__name__)
logger.info('loglevel is: {}'.format(
logging.getLevelName(logger.getEffectiveLevel())))
setup_signals(__name__)
if not args.token_create_smrt and not args.token_create_admin and not args.token_create_hunter and not \
args.token_create_httpd:
logger.error('missing required arguments, see -h for more information')
raise SystemExit
if args.token_create_smrt:
with Store(store_type=args.store, nodes=args.nodes) as s:
s._load_plugin(store_type=args.store, nodes=args.nodes)
t = s.token_create_smrt(token=args.token, groups=groups)
if t:
if PYVERSION == 2:
t = t.encode('utf-8')
data = {
'token': t,
}
if args.remote:
data['remote'] = args.remote
if args.config_path:
with open(args.config_path, 'w') as f:
f.write(yaml.dump(data, default_flow_style=False))
logger.info('token config generated: {}'.format(
args.token_create_smrt))
else:
logger.error('token not created')
if args.token_create_hunter:
with Store(store_type=args.store, nodes=args.nodes) as s:
s._load_plugin(store_type=args.store, nodes=args.nodes)
t = s.token_create_hunter(token=args.token, groups=groups)
if t:
if PYVERSION == 2:
t = t.encode('utf-8')
data = {
'hunter_token': t,
}
if args.config_path:
with open(args.config_path, 'w') as f:
f.write(yaml.dump(data, default_flow_style=False))
logger.info('token config generated: {}'.format(
args.token_create_hunter))
else:
logger.error('token not created')
if args.token_create_admin:
with Store(store_type=args.store, nodes=args.nodes) as s:
s._load_plugin(store_type=args.store, nodes=args.nodes)
t = s.token_create_admin(token=args.token, groups=groups)
if t:
if PYVERSION == 2:
t = t.encode('utf-8')
data = {
'token': t,
}
if args.config_path:
with open(args.config_path, 'w') as f:
f.write(yaml.dump(data, default_flow_style=False))
logger.info('token config generated: {}'.format(
args.token_create_admin))
else:
logger.error('token not created')
if args.token_create_httpd:
with Store(store_type=args.store, nodes=args.nodes) as s:
s._load_plugin(store_type=args.store, nodes=args.nodes)
t = s.token_create_httpd(token=args.token, groups=groups)
if t:
if PYVERSION == 2:
t = t.encode('utf-8')
data = {
'token': t,
}
if args.config_path:
with open(args.config_path, 'w') as f:
f.write(yaml.dump(data, default_flow_style=False))
logger.info('token config generated: {}'.format(
args.token_create_httpd))
else:
logger.error('token not created')
def main():
p = get_argument_parser()
p = ArgumentParser(
description=textwrap.dedent('''\
example usage:
$ cif -q example.org -d
$ cif --search 1.2.3.0/24
$ cif --ping
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='cif',
parents=[p]
)
p.add_argument('--token', help='specify api token', default=TOKEN)
p.add_argument('--remote', help='specify API remote [default %(default)s]', default=REMOTE_ADDR)
p.add_argument('-p', '--ping', action="store_true") # meg?
p.add_argument('--ping-indef', action="store_true")
p.add_argument('-q', '--search', help="search")
p.add_argument('--itype', help='filter by indicator type') ## need to fix sqlite for non-ascii stuff first
p.add_argument("--submit", action="store_true", help="submit an indicator")
p.add_argument('--limit', help='limit results [default %(default)s]', default=SEARCH_LIMIT)
p.add_argument('--reporttime', help='specify reporttime filter')
p.add_argument('-n', '--nolog', help='do not log search', action='store_true')
p.add_argument('-f', '--format', help='specify output format [default: %(default)s]"', default=FORMAT, choices=FORMATS.keys())
p.add_argument('--indicator')
p.add_argument('--tags', nargs='+')
p.add_argument('--provider')
p.add_argument('--confidence', help="specify confidence level")
p.add_argument('--tlp', help="specify traffic light protocol")
p.add_argument("--zmq", help="use zmq as a transport instead of http", action="store_true")
p.add_argument('--config', help='specify config file [default %(default)s]', default=CONFIG_PATH)
p.add_argument('--feed', action='store_true')
p.add_argument('--no-verify-ssl', action='store_true')
p.add_argument('--last-day', action="store_true", help='auto-sets reporttime to 23 hours and 59 seconds ago '
'(current time UTC) and reporttime-end to "now"')
p.add_argument('--last-hour', action='store_true', help='auto-sets reporttime to the beginning of the previous full'
' hour and reporttime-end to end of previous full hour')
p.add_argument('--days', help='filter results within last X days')
p.add_argument('--today', help='auto-sets reporttime to today, 00:00:00Z (UTC)', action='store_true')
p.add_argument('--columns', help='specify output columns [default %(default)s]', default=','.join(COLUMNS))
p.add_argument('--fields', help='same as --columns [default %(default)s]', default=','.join(COLUMNS))
p.add_argument('--asn')
p.add_argument('--cc')
p.add_argument('--asn-desc')
p.add_argument('--rdata')
p.add_argument('--no-feed', action='store_true')
p.add_argument('--region')
p.add_argument('--groups', help='specify groups filter (csv)')
p.add_argument('--delete', action='store_true')
p.add_argument('--id')
args = p.parse_args()
if args.fields != ','.join(COLUMNS):
args.columns = args.fields
setup_logging(args)
logger = logging.getLogger(__name__)
o = read_config(args)
options = vars(args)
# support for separate read and write tokens
if o.get('write_token') and options.get('submit'):
o['token'] = o['write_token']
elif o.get('read_token'):
o['token'] = o['read_token']
for v in options:
if v == 'remote' and options[v] == REMOTE_ADDR and o.get('remote'):
options[v] = o['remote']
if v == 'token' and o.get('token'):
options[v] = o['token']
if options[v] is None or options[v] == '':
options[v] = o.get(v)
if not options.get('token'):
raise RuntimeError('missing --token')
verify_ssl = True
if o.get('no_verify_ssl') or options.get('no_verify_ssl'):
verify_ssl = False
if options.get("zmq"):
from cifsdk.client.zeromq import ZMQ as ZMQClient
cli = ZMQClient(**options)
else:
from cifsdk.client.http import HTTP as HTTPClient
if args.remote == 'https://localhost':
verify_ssl = False
cli = HTTPClient(args.remote, args.token, verify_ssl=verify_ssl)
if options.get('ping') or options.get('ping_indef'):
logger.info('running ping')
n = 4
if args.ping_indef:
n = 999
try:
for num in range(0, n):
ret = cli.ping()
if ret != 0:
print("roundtrip: {} ms".format(ret))
select.select([], [], [], 1)
from time import sleep
sleep(1)
else:
logger.error('ping failed')
raise RuntimeError
except KeyboardInterrupt:
pass
raise SystemExit
if options.get("submit"):
print("submitting {0}".format(options.get("submit")))
i = Indicator(indicator=args.indicator, tags=args.tags, confidence=args.confidence, group=args.groups, tlp=args.tlp, provider=args.provider)
rv = cli.indicators_create(i)
print('success id: {}\n'.format(rv))
raise SystemExit
filters = {
'itype': options['itype'],
'limit': options['limit'],
'provider': options.get('provider'),
'indicator': options.get('search') or options.get('indicator'),
'nolog': options['nolog'],
'tags': options['tags'],
'confidence': options.get('confidence'),
'asn': options.get('asn'),
'asn_desc': options.get('asn_desc'),
'cc': options.get('cc'),
'region': options.get('region'),
'rdata': options.get('rdata'),
'reporttime': options.get('reporttime'),
'groups': options.get('groups'),
'tlp': options.get('tlp')
}
if args.last_day:
filters['days'] = '1'
del filters['reporttime']
if args.last_hour:
filters['hours'] = '1'
del filters['reporttime']
if args.days:
filters['days'] = args.days
del filters['reporttime']
if args.today:
now = arrow.utcnow()
filters['reporttime'] = '{0}Z'.format(now.format('YYYY-MM-DDT00:00:00'))
if filters.get('itype') and not filters.get('search') and not args.no_feed:
logger.info('setting feed flag by default, use --no-feed to override')
options['feed'] = True
if options.get("delete"):
if args.id:
filters = {'id': args.id}
filters = {f: filters[f] for f in filters if filters.get(f)}
print("deleting {0}".format(filters))
rv = cli.indicators_delete(filters)
print('deleted: {}'.format(rv))
raise SystemExit
if options.get('feed'):
if not filters.get('itype') and not ADVANCED:
print('\nmissing --itype\n\n')
raise SystemExit
if not filters.get('tags') and not ADVANCED:
print('\nmissing --tags [phishing|malware|botnet|scanner|pdns|whitelist|...]\n\n')
raise SystemExit
if not filters.get('confidence'):
filters['confidence'] = 8
if args.limit == SEARCH_LIMIT:
filters['limit'] = FEED_LIMIT
try:
rv = cli.feed(filters=filters)
except AuthError as e:
logger.error('unauthorized')
except KeyboardInterrupt:
pass
except Exception as e:
logger.error(e)
else:
print(FORMATS[options.get('format')](data=rv, cols=args.columns.split(',')))
raise SystemExit
try:
rv = cli.search(filters)
except AuthError as e:
logger.error('unauthorized')
except KeyboardInterrupt:
pass
except Exception as e:
import traceback
traceback.print_exc()
logger.error(e)
else:
print(FORMATS[options.get('format')](data=rv, cols=args.columns.split(',')))
def main():
p = get_argument_parser()
p = ArgumentParser(
description=textwrap.dedent('''\
Env Variables:
CIF_RUNTIME_PATH
CIF_STORE_ADDR
example usage:
$ cif-store -d
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='cif-store',
parents=[p]
)
p.add_argument("--store-address", help="specify the store address cif-router is listening on[default: %("
"default)s]", default=STORE_ADDR)
p.add_argument("--store", help="specify a store type {} [default: %(default)s]".format(', '.join(STORE_PLUGINS)),
default=STORE_DEFAULT)
p.add_argument('--nodes')
p.add_argument('--config', help='specify config path [default %(default)s]', default=CONFIG_PATH)
p.add_argument('--token-create-admin', help='generate an admin token')
p.add_argument('--token-create-smrt')
p.add_argument('--token-create-smrt-remote', default=REMOTE_ADDR)
p.add_argument('--token-create-hunter')
p.add_argument('--remote', help='specify remote')
args = p.parse_args()
setup_logging(args)
logger = logging.getLogger(__name__)
logger.info('loglevel is: {}'.format(logging.getLevelName(logger.getEffectiveLevel())))
setup_signals(__name__)
if args.token_create_smrt:
with Store(store_type=args.store, nodes=args.nodes) as s:
s._load_plugin(store_type=args.store, nodes=args.nodes)
t = s.token_create_smrt()
if t:
data = {
'token': t.encode('utf-8'),
}
if args.remote:
data['remote'] = args.remote
with open(args.token_create_smrt, 'w') as f:
f.write(yaml.dump(data, default_flow_style=False))
logger.info('token config generated: {}'.format(args.token_create_smrt))
else:
logger.error('token not created')
if args.token_create_hunter:
with Store(store_type=args.store, nodes=args.nodes) as s:
s._load_plugin(store_type=args.store, nodes=args.nodes)
t = s.token_create_hunter()
if t:
data = {
'hunter_token': t.encode('utf-8'),
}
with open(args.token_create_hunter, 'w') as f:
f.write(yaml.dump(data, default_flow_style=False))
logger.info('token config generated: {}'.format(args.token_create_hunter))
else:
logger.error('token not created')
if args.token_create_admin:
with Store(store_type=args.store, nodes=args.nodes) as s:
s._load_plugin(store_type=args.store, nodes=args.nodes)
t = s.token_create_admin()
if t:
data = {
'token': t.encode('utf-8'),
}
with open(args.token_create_admin, 'w') as f:
f.write(yaml.dump(data, default_flow_style=False))
logger.info('token config generated: {}'.format(args.token_create_admin))
else:
logger.error('token not created')
def main():
p = get_argument_parser()
p = ArgumentParser(description=textwrap.dedent('''\
Env Variables:
CIF_RUNTIME_PATH
CIF_ROUTER_CONFIG_PATH
CIF_ROUTER_ADDR
CIF_HUNTER_ADDR
CIF_HUNTER_TOKEN
CIF_HUNTER_THREADS
CIF_GATHERER_THREADS
CIF_STORE_ADDR
example usage:
$ cif-router --listen 0.0.0.0 -d
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='cif-router',
parents=[p])
p.add_argument('--config',
help='specify config path [default: %(default)s',
default=CONFIG_PATH)
p.add_argument('--listen',
help='address to listen on [default: %(default)s]',
default=ROUTER_ADDR)
p.add_argument(
'--gatherer-threads',
help='specify number of gatherer threads to use [default: %(default)s]',
default=GATHERER_THREADS)
p.add_argument('--hunter',
help='address hunters listen on on [default: %(default)s]',
default=HUNTER_ADDR)
p.add_argument(
'--hunter-token',
help='specify token for hunters to use [default: %(default)s]',
default=HUNTER_TOKEN)
p.add_argument(
'--hunter-threads',
help='specify number of hunter threads to use [default: %(default)s]',
default=HUNTER_THREADS)
p.add_argument(
"--store-address",
help="specify the store address cif-router is listening on[default: %("
"default)s]",
default=STORE_ADDR)
p.add_argument(
"--store",
help="specify a store type {} [default: %(default)s]".format(
', '.join(STORE_PLUGINS)),
default=STORE_DEFAULT)
p.add_argument('--store-nodes',
help='specify storage nodes address [default: %(default)s]',
default=STORE_NODES)
p.add_argument('--p2p',
action='store_true',
help='enable experimental p2p support')
p.add_argument('--logging-ignore',
help='set logging to WARNING for specific modules')
args = p.parse_args()
setup_logging(args)
logger = logging.getLogger(__name__)
logger.info('loglevel is: {}'.format(
logging.getLevelName(logger.getEffectiveLevel())))
if args.logging_ignore:
to_ignore = args.logging_ignore.split(',')
for i in to_ignore:
logging.getLogger(i).setLevel(logging.WARNING)
o = read_config(args)
options = vars(args)
for v in options:
if options[v] is None:
options[v] = o.get(v)
setup_signals(__name__)
setup_runtime_path(args.runtime_path)
with Router(listen=args.listen,
hunter=args.hunter,
store_type=args.store,
store_address=args.store_address,
store_nodes=args.store_nodes,
p2p=args.p2p,
hunter_token=args.hunter_token,
hunter_threads=args.hunter_threads,
gatherer_threads=args.gatherer_threads) as r:
try:
logger.info('starting router..')
r.start()
except KeyboardInterrupt:
# todo - signal to threads to shut down and wait for them to finish
logger.info('shutting down...')
logger.info('Shutting down')
def main():
p = get_argument_parser()
p = ArgumentParser(description=textwrap.dedent('''\
Env Variables:
CIF_RUNTIME_PATH
CIF_ROUTER_CONFIG_PATH
CIF_ROUTER_ADDR
CIF_HUNTER_ADDR
CIF_HUNTER_TOKEN
CIF_HUNTER_THREADS
CIF_GATHERER_THREADS
CIF_STORE_ADDR
example usage:
$ cif-router --listen 0.0.0.0 -d
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='cif-router',
parents=[p])
p.add_argument('--config',
help='specify config path [default: %(default)s',
default=CONFIG_PATH)
p.add_argument('--listen',
help='address to listen on [default: %(default)s]',
default=ROUTER_ADDR)
p.add_argument(
'--gatherer-threads',
help='specify number of gatherer threads to use [default: %(default)s]',
default=GATHERER_THREADS)
p.add_argument('--hunter',
help='address hunters listen on on [default: %(default)s]',
default=HUNTER_ADDR)
p.add_argument(
'--hunter-token',
help='specify token for hunters to use [default: %(default)s]',
default=HUNTER_TOKEN)
p.add_argument(
'--hunter-threads',
help='specify number of hunter threads to use [default: %(default)s]',
default=HUNTER_THREADS)
p.add_argument(
"--store-address",
help="specify the store address cif-router is listening on[default: %("
"default)s]",
default=STORE_ADDR)
p.add_argument(
"--store",
help="specify a store type {} [default: %(default)s]".format(
', '.join(STORE_PLUGINS)),
default=STORE_DEFAULT)
p.add_argument('--store-nodes',
help='specify storage nodes address [default: %(default)s]',
default=STORE_NODES)
p.add_argument('--logging-ignore',
help='set logging to WARNING for specific modules')
p.add_argument('--pidfile',
help='specify pidfile location [default: %(default)s]',
default=PIDFILE)
args = p.parse_args()
setup_logging(args)
logger = logging.getLogger(__name__)
logger.info('loglevel is: {}'.format(
logging.getLevelName(logger.getEffectiveLevel())))
if args.logging_ignore:
to_ignore = args.logging_ignore.split(',')
for i in to_ignore:
logging.getLogger(i).setLevel(logging.WARNING)
o = read_config(args)
options = vars(args)
for v in options:
if options[v] is None:
options[v] = o.get(v)
setup_runtime_path(args.runtime_path)
setup_signals(__name__)
# http://stackoverflow.com/a/789383/7205341
pid = str(os.getpid())
logger.debug("pid: %s" % pid)
if os.path.isfile(args.pidfile):
logger.critical("%s already exists, exiting" % args.pidfile)
raise SystemExit
try:
pidfile = open(args.pidfile, 'w')
pidfile.write(pid)
pidfile.close()
except PermissionError as e:
logger.error('unable to create pid %s' % args.pidfile)
with Router(listen=args.listen,
hunter=args.hunter,
store_type=args.store,
store_address=args.store_address,
store_nodes=args.store_nodes,
hunter_token=args.hunter_token,
hunter_threads=args.hunter_threads,
gatherer_threads=args.gatherer_threads) as r:
try:
logger.info('starting router..')
r.start()
except KeyboardInterrupt:
# todo - signal to threads to shut down and wait for them to finish
logger.info('shutting down via SIGINT...')
except SystemExit:
logger.info('shutting down via SystemExit...')
except Exception as e:
logger.critical(e)
traceback.print_exc()
r.stop()
logger.info('Shutting down')
if os.path.isfile(args.pidfile):
os.unlink(args.pidfile)
def main():
p = get_argument_parser()
p = ArgumentParser(description=textwrap.dedent('''\
example usage:
$ cif -q example.org -d
$ cif --search 1.2.3.0/24
$ cif --ping
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='cif',
parents=[p])
p.add_argument('--token', help='specify api token', default=TOKEN)
p.add_argument('--remote',
help='specify API remote [default %(default)s]',
default=REMOTE_ADDR)
p.add_argument('--no-verify-ssl', action='store_true')
p.add_argument('-p', '--ping', action="store_true")
p.add_argument('--ping-indef', action="store_true")
p.add_argument("--submit", action="store_true", help="submit an indicator")
p.add_argument('--delete', action='store_true')
p.add_argument('-q', '--search', help="search")
p.add_argument('--id')
p.add_argument('--itype', help='filter by indicator type')
p.add_argument('--reported_at', help='specify reported_at filter')
p.add_argument('-n',
'--nolog',
help='do not log search',
action='store_true')
p.add_argument('-f',
'--format',
help='specify output format [default: %(default)s]"',
default=FORMAT,
choices=FORMATS.keys())
p.add_argument('--indicator')
p.add_argument('--confidence', help="specify confidence level")
p.add_argument('--probability')
p.add_argument('--tags', nargs='+')
p.add_argument('--provider')
p.add_argument('--asn')
p.add_argument('--cc')
p.add_argument('--asn-desc')
p.add_argument('--rdata')
p.add_argument('--region')
p.add_argument('--groups', help='specify groups filter (csv)')
p.add_argument('--days', help='filter results within last X days')
p.add_argument('--today',
help='auto-sets reporttime to today, 00:00:00Z (UTC)',
action='store_true')
p.add_argument('--limit',
help='limit results [default %(default)s]',
default=SEARCH_LIMIT)
p.add_argument('--columns',
help='specify output columns [default %(default)s]',
default=','.join(COLUMNS))
p.add_argument('--no-feed', action='store_true')
p.add_argument('--graph', help='dump the graph', action='store_true')
p.add_argument('--profile',
help='specify feed profile',
choices=PROFILES.keys())
args = p.parse_args()
setup_logging(args)
opts = vars(args)
options = {}
for k, v in opts.items():
if v:
options[k] = v
verify_ssl = True
if args.no_verify_ssl:
verify_ssl = False
if args.remote == 'https://localhost':
verify_ssl = False
cli = Client(args.remote, args.token, verify_ssl=verify_ssl)
if args.ping or args.ping_indef:
_ping(cli, args)
filters = {
'itype': options.get('itype'),
'limit': options.get('limit'),
'provider': options.get('provider'),
'indicator': options.get('search') or options.get('indicator'),
'nolog': options.get('nolog'),
'tags': options.get('tags'),
'confidence': options.get('confidence'),
'asn': options.get('asn'),
'asn_desc': options.get('asn_desc'),
'cc': options.get('cc'),
'region': options.get('region'),
'rdata': options.get('rdata'),
'reported_at': options.get('reported_at'),
'groups': options.get('groups'),
'hours': options.get('hours'),
'days': options.get('days'),
'today': options.get('today'),
'no_feed': options.get('no_feed'),
'probability': options.get('probability')
}
for k, v in filters.items():
if v is True:
filters[k] = 1
if v is False:
filters[k] = 0
if options.get("submit"):
_submit(cli, args, options)
if options.get("delete"):
_delete(cli, args, options, filters)
if args.graph:
_graph(cli, args, options, filters)
if not sys.stdin.isatty():
buffer = sys.stdin.read().rstrip("\n").split("\n")
filters = [{'indicator': i} for i in buffer]
_search(cli, args, options, filters)
def main():
p = get_argument_parser()
p = ArgumentParser(
description=textwrap.dedent('''\
example usage:
$ cif-httpd -d
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='cif-httpd',
parents=[p]
)
router_address = app.config.get('CIF_ROUTER_ADDR', ROUTER_ADDR)
p.add_argument("--router", help="specify router frontend [default %(default)s]", default=router_address)
p.add_argument('--listen', help='specify the interface to listen on [default %(default)s]', default=HTTP_LISTEN)
p.add_argument('--listen-port', help='specify the port to listen on [default %(default)s]',
default=HTTP_LISTEN_PORT)
p.add_argument('--pidfile', help='specify pidfile location [default: %(default)s]', default=PIDFILE)
p.add_argument('--fdebug', action='store_true')
args = p.parse_args()
setup_logging(args)
logger = logging.getLogger(__name__)
if TRACE:
logger.setLevel(logging.DEBUG)
logger.info('loglevel is: {}'.format(logging.getLevelName(logger.getEffectiveLevel())))
setup_signals(__name__)
setup_runtime_path(args.runtime_path)
if not args.fdebug:
# http://stackoverflow.com/a/789383/7205341
pid = str(os.getpid())
logger.debug("pid: %s" % pid)
if os.path.isfile(args.pidfile):
logger.critical("%s already exists, exiting" % args.pidfile)
raise SystemExit
try:
pidfile = open(args.pidfile, 'w')
pidfile.write(pid)
pidfile.close()
except PermissionError as e:
logger.error('unable to create pid %s' % args.pidfile)
try:
logger.info('pinging router...')
logger.info('starting up...')
app.run(host=args.listen, port=args.listen_port, debug=args.fdebug, threaded=True, extra_files=extra_files)
except KeyboardInterrupt:
logger.info('shutting down...')
except Exception as e:
logger.critical(e)
traceback.print_exc()
if os.path.isfile(args.pidfile):
os.unlink(args.pidfile)
def main():
p = get_argument_parser()
p = ArgumentParser(description=textwrap.dedent('''\
example usage:
$ cif-tokens --name [email protected] --create --admin
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='cif',
parents=[p])
p.add_argument('--token',
help='specify api token [default %(default)s]',
default=TOKEN)
p.add_argument('--remote',
help='specify API remote [default %(default)s]',
default=REMOTE_ADDR)
p.add_argument('--create',
help='create token (requires admin token',
action='store_true')
p.add_argument('--delete',
help='delete token (requires admin token)',
action='store_true')
p.add_argument('--delete-token', help='specify the token to delete')
p.add_argument('--username', help='specify username')
p.add_argument('--name', help='specify username')
p.add_argument('--admin', action='store_true')
p.add_argument('--expires', help='set a token expiration timestamp')
p.add_argument('--read',
help='set the token read flag',
action='store_true')
p.add_argument('--write',
help='set the token write flag',
action='store_true')
p.add_argument('--revoked',
help='set the token revoked flag',
action='store_true')
p.add_argument(
'--groups',
help=
'specify token groups (eg: everyone,group1,group2) [default %(default)s]',
default='everyone')
p.add_argument('--no-everyone',
help="do not create key in the 'everyone' group",
action='store_true')
p.add_argument('--acl',
help='set the token itype acls (eg: ipv4,ipv6)',
default='')
p.add_argument(
'--columns',
help='specify columns to print when searching [default %(default)s]',
default=','.join(COLS))
p.add_argument('--config-generate', help='generate configuration file')
p.add_argument('--config',
help='specify configuration file [default %(default)s]',
default=CONFIG_PATH)
p.add_argument('--no-verify-ssl',
help='Turn OFF TLS verification',
action='store_true')
p.add_argument('--update', help='update a token')
args = p.parse_args()
setup_logging(args)
logger = logging.getLogger(__name__)
o = {}
if os.path.isfile(args.config):
o = read_config(args)
options = vars(args)
if not options.get('token'):
raise RuntimeError('missing --token')
verify_ssl = True
if o.get('no_verify_ssl') or options.get('no_verify_ssl'):
verify_ssl = False
options = vars(args)
from cifsdk.client.http import HTTP as HTTPClient
cli = HTTPClient(args.remote, args.token, verify_ssl=verify_ssl)
if options.get('name'):
options['username'] = options['name']
rv = False
if options.get('create'):
if not options.get('username'):
raise RuntimeError('missing --username')
if not (options.get('read') or options.get('write')):
logger.info('assuming --read token')
options['read'] = True
groups = set(options.get('groups').split(','))
if not options.get('no_everyone'):
if 'everyone' not in groups:
groups.add('everyone')
acl = options.get('acl').split(',')
try:
rv = cli.tokens_create({
'username': options.get('username'),
'admin': options.get('admin'),
'expires': options.get('expires'),
'read': options.get('read'),
'revoked': options.get('revoked'),
'write': options.get('write'),
'groups': list(groups),
'acl': acl
})
except AuthError as e:
logger.error(e)
except Exception as e:
logger.error('token create failed: {}'.format(e))
else:
if options.get('config_generate'):
data = {
'remote': options['remote'],
'token': str(rv['token']),
}
with open(options['config_generate'], 'w') as f:
f.write(yaml.dump(data, default_flow_style=False))
t = PrettyTable(args.columns.split(','))
l = []
for c in args.columns.split(','):
if c == 'last_activity_at' and rv.get(c):
rv[c] = arrow.get(rv[c]).format('YYYY-MM-DDTHH:MM:ss')
rv[c] = '{}Z'.format(rv[c])
if c == 'expires' and rv.get(c):
rv[c] = arrow.get(rv[c]).format('YYYY-MM-DDTHH:MM:ss')
rv[c] = '{}Z'.format(rv[c])
if rv.get(c):
if type(rv[c]) == list:
l.append(','.join(rv[c]))
else:
l.append(str(rv[c]))
else:
l.append(None)
t.add_row(l)
print(t)
elif options.get('delete_token'):
try:
rv = cli.tokens_delete({
'token': options.get('delete_token'),
'username': options.get('username')
})
if rv:
logger.info('deleted: {} tokens successfully'.format(rv))
else:
logger.error('no tokens deleted')
except Exception as e:
logger.error('token delete failed: %s' % e)
elif options.get('delete'):
if not (options.get('delete_token') or options.get('username')):
raise RuntimeError(
'--delete requires --delete-token or --username')
try:
rv = cli.tokens_delete({
'token': options.get('delete_token'),
'username': options.get('username')
})
if rv:
logger.info('deleted: {} tokens successfully'.format(rv))
else:
logger.error('no tokens deleted')
except Exception as e:
logger.error('token delete failed: %s' % e)
elif options.get('update'):
if not options.get('groups'):
raise RuntimeError('requires --groups')
groups = options['groups'].split(',')
rv = cli.tokens_edit({'token': options['update'], 'groups': groups})
if rv:
logger.info('token updated successfully')
rv = cli.tokens_search({'token': options['update']})
t = PrettyTable(args.columns.split(','))
for r in rv:
l = []
for c in args.columns.split(','):
if c == 'last_activity_at' and r[c] is not None:
r[c] = arrow.get(r[c]).format('YYYY-MM-DDTHH:MM:ss')
r[c] = '{}Z'.format(r[c])
if c == 'expires' and r[c] is not None:
r[c] = arrow.get(r[c]).format('YYYY-MM-DDTHH:MM:ss')
r[c] = '{}Z'.format(r[c])
l.append(r[c])
t.add_row(l)
print(t)
else:
logger.error(rv)
else:
filters = {}
if options.get('username'):
filters['username'] = options.get('username')
try:
rv = cli.tokens_search(filters)
except AuthError:
logger.error('unauthorized')
except Exception as e:
import traceback
traceback.print_exc()
print("\ntoken search failed: %s" % e)
else:
t = PrettyTable(args.columns.split(','))
for r in rv:
l = []
for c in args.columns.split(','):
if c == 'last_activity_at' and r.get(c) is not None:
r[c] = parse_timestamp(
r[c]).format('YYYY-MM-DDTHH:mm:ss.SS')
r[c] = '{}Z'.format(r[c])
if c == 'expires' and r.get(c) is not None:
r[c] = parse_timestamp(
r[c]).format('YYYY-MM-DDTHH:mm:ss.SS')
r[c] = '{}Z'.format(r[c])
if type(r.get(c)) == list:
r[c] = ','.join(r[c])
l.append(r.get(c))
t.add_row(l)
print(t)
def main():
p = get_argument_parser()
p = ArgumentParser(
description=textwrap.dedent('''\
example usage:
$ cif-tokens --name [email protected] --create --admin
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='cif',
parents=[p]
)
p.add_argument('--token', help='specify api token [default %(default)s]', default=TOKEN)
p.add_argument('--remote', help='specify API remote [default %(default)s]', default=REMOTE_ADDR)
p.add_argument('--create', help='create token (requires admin token', action='store_true')
p.add_argument('--delete', help='delete token (requires admin token)', action='store_true')
p.add_argument('--delete-token', help='specify the token to delete')
p.add_argument('--username', help='specify username')
p.add_argument('--name', help='specify username')
p.add_argument('--admin', action='store_true')
p.add_argument('--expires', help='set a token expiration timestamp')
p.add_argument('--read', help='set the token read flag', action='store_true')
p.add_argument('--write', help='set the token write flag', action='store_true')
p.add_argument('--revoked', help='set the token revoked flag', action='store_true')
p.add_argument('--groups', help='specify token groups (eg: everyone,group1,group2) [default %(default)s]',
default='everyone')
p.add_argument('--no-everyone', help="do not create key in the 'everyone' group", action='store_true')
p.add_argument('--acl', help='set the token itype acls (eg: ipv4,ipv6)', default='')
p.add_argument('--columns', help='specify columns to print when searching [default %(default)s]',
default=','.join(COLS))
p.add_argument('--config-generate', help='generate configuration file')
p.add_argument('--config', help='specify configuration file [default %(default)s]', default=CONFIG_PATH)
p.add_argument('--no-verify-ssl', help='Turn OFF TLS verification', action='store_true')
p.add_argument('--update', help='update a token')
args = p.parse_args()
setup_logging(args)
logger = logging.getLogger(__name__)
o = read_config(args)
options = vars(args)
for v in options:
if v == 'remote' and options[v] == REMOTE_ADDR and o.get('remote'):
options[v] = o['remote']
if v == 'token' and o.get('token'):
options[v] = o['token']
if options[v] is None:
options[v] = o.get(v)
if not options.get('token'):
raise RuntimeError('missing --token')
verify_ssl = True
if o.get('no_verify_ssl') or options.get('no_verify_ssl'):
verify_ssl = False
options = vars(args)
from cifsdk.client.http import HTTP as HTTPClient
cli = HTTPClient(args.remote, args.token, verify_ssl=verify_ssl)
if options.get('name'):
options['username'] = options['name']
rv = False
if options.get('create'):
if not options.get('username'):
raise RuntimeError('missing --username')
if not (options.get('read') or options.get('write')):
logger.info('assuming --read token')
options['read'] = True
groups = set(options.get('groups').split(','))
if not options.get('no_everyone'):
if 'everyone' not in groups:
groups.add('everyone')
acl = options.get('acl').split(',')
try:
rv = cli.tokens_create({
'username': options.get('username'),
'admin': options.get('admin'),
'expires': options.get('expires'),
'read': options.get('read'),
'revoked': options.get('revoked'),
'write': options.get('write'),
'groups': list(groups),
'acl': acl
})
except AuthError as e:
logger.error(e)
except Exception as e:
logger.error('token create failed: {}'.format(e))
else:
if options.get('config_generate'):
data = {
'remote': options['remote'],
'token': str(rv['token']),
}
with open(options['config_generate'], 'w') as f:
f.write(yaml.dump(data, default_flow_style=False))
t = PrettyTable(args.columns.split(','))
l = []
for c in args.columns.split(','):
if c == 'last_activity_at' and rv.get(c):
rv[c] = arrow.get(rv[c]).format('YYYY-MM-DDTHH:MM:ss')
rv[c] = '{}Z'.format(rv[c])
if c == 'expires' and rv.get(c):
rv[c] = arrow.get(rv[c]).format('YYYY-MM-DDTHH:MM:ss')
rv[c] = '{}Z'.format(rv[c])
if rv.get(c):
if type(rv[c]) == list:
l.append(','.join(rv[c]))
else:
l.append(str(rv[c]))
else:
l.append(None)
t.add_row(l)
print(t)
elif options.get('delete_token'):
try:
rv = cli.tokens_delete({
'token': options.get('delete_token'),
'username': options.get('username')
})
if rv:
logger.info('deleted: {} tokens successfully'.format(rv))
else:
logger.error('no tokens deleted')
except Exception as e:
logger.error('token delete failed: %s' % e)
elif options.get('delete'):
if not (options.get('delete_token') or options.get('username')):
raise RuntimeError('--delete requires --delete-token or --username')
try:
rv = cli.tokens_delete({
'token': options.get('delete_token'),
'username': options.get('username')
})
if rv:
logger.info('deleted: {} tokens successfully'.format(rv))
else:
logger.error('no tokens deleted')
except Exception as e:
logger.error('token delete failed: %s' % e)
elif options.get('update'):
if not options.get('groups'):
raise RuntimeError('requires --groups')
groups = options['groups'].split(',')
rv = cli.tokens_edit({
'token': options['update'],
'groups': groups
})
if rv:
print('token updated successfully')
print('refreshing tokens...')
sleep(2)
rv = cli.tokens_search({'token': options['update']})
t = PrettyTable(args.columns.split(','))
for r in rv:
l = []
for c in args.columns.split(','):
if c == 'last_activity_at' and r.get(c) is not None:
r[c] = parse_timestamp(r[c]).format('YYYY-MM-DDTHH:mm:ss.SS')
r[c] = '{}Z'.format(r[c])
if c == 'expires' and r.get(c) is not None:
r[c] = parse_timestamp(r[c]).format('YYYY-MM-DDTHH:mm:ss.SS')
r[c] = '{}Z'.format(r[c])
if type(r.get(c)) == list:
r[c] = ','.join(r[c])
l.append(r.get(c))
t.add_row(l)
print(t)
else:
logger.error(rv)
else:
filters = {}
if options.get('username'):
filters['username'] = options.get('username')
try:
rv = cli.tokens_search(filters)
except AuthError:
logger.error('unauthorized')
except Exception as e:
logger.error('token search failed: {}'.format(e))
else:
t = PrettyTable(args.columns.split(','))
for r in rv:
l = []
for c in args.columns.split(','):
if c == 'last_activity_at' and r.get(c) is not None:
r[c] = parse_timestamp(r[c]).format('YYYY-MM-DDTHH:mm:ss.SS')
r[c] = '{}Z'.format(r[c])
if c == 'expires' and r.get(c) is not None:
r[c] = parse_timestamp(r[c]).format('YYYY-MM-DDTHH:mm:ss.SS')
r[c] = '{}Z'.format(r[c])
if type(r.get(c)) == list:
r[c] = ','.join(r[c])
l.append(r.get(c))
t.add_row(l)
print(t)
def main():
p = get_argument_parser()
p = ArgumentParser(
description=textwrap.dedent('''\
Env Variables:
CIF_RUNTIME_PATH
CIF_STORE_ADDR
example usage:
$ cif-store -d
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='cif-store',
parents=[p]
)
p.add_argument("--store-address", help="specify the store address cif-router is listening on[default: %("
"default)s]", default=STORE_ADDR)
p.add_argument("--store", help="specify a store type {} [default: %(default)s]".format(', '.join(STORE_PLUGINS)),
default=STORE_DEFAULT)
p.add_argument('--config', help='specify config path [default %(default)s]', default=CONFIG_PATH)
p.add_argument('--token-create-admin', help='generate an admin token')
p.add_argument('--token-create-smrt')
p.add_argument('--token-create-smrt-remote', default=REMOTE_ADDR)
p.add_argument('--token-create-hunter')
args = p.parse_args()
pprint(args)
setup_logging(args)
logger = logging.getLogger(__name__)
logger.info('loglevel is: {}'.format(logging.getLevelName(logger.getEffectiveLevel())))
setup_signals(__name__)
# from cif.reloader import ModuleWatcher
# mw = ModuleWatcher()
# mw.watch_module('cif.store')
# mw.start_watching()
start = True
if args.token_create_smrt:
start = False
with Store(store=args.store) as s:
t = s.token_create_smrt()
if t:
data = {
'token': str(t),
}
with open(args.token_create_smrt, 'w') as f:
f.write(yaml.dump(data, default_flow_style=False))
logger.info('token config generated: {}'.format(args.token_create_smrt))
else:
logger.error('token not created')
if args.token_create_hunter:
start = False
with Store(store=args.store) as s:
t = s.token_create_hunter()
if t:
data = {
'token': str(t),
}
with open(args.token_create_hunter, 'w') as f:
f.write(yaml.dump(data, default_flow_style=False))
logger.info('token config generated: {}'.format(args.token_create_hunter))
else:
logger.error('token not created')
if args.token_create_admin:
start = False
with Store(store=args.store) as s:
t = s.token_create_admin()
if t:
data = {
'token': str(t),
}
with open(args.token_create_admin, 'w') as f:
f.write(yaml.dump(data, default_flow_style=False))
logger.info('token config generated: {}'.format(args.token_create_admin))
else:
logger.error('token not created')
if start:
with Store(store_address=args.store_address, store=args.store) as s:
try:
logger.info('starting up...')
s.start()
except KeyboardInterrupt:
logger.info('shutting down...')
def main():
p = get_argument_parser()
p = ArgumentParser(
description=textwrap.dedent('''\
Env Variables:
CIF_RUNTIME_PATH
CIF_STORE_ADDR
example usage:
$ cif-store -d
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='cif-store',
parents=[p]
)
p.add_argument("--store-address", help="specify the store address cif-router is listening on[default: %("
"default)s]", default=STORE_ADDR)
p.add_argument("--store", help="specify a store type {} [default: %(default)s]".format(', '.join(STORE_PLUGINS)),
default=STORE_DEFAULT)
p.add_argument('--nodes')
p.add_argument('--config', help='specify config path [default %(default)s]', default=CONFIG_PATH)
p.add_argument('--token-create-admin', help='generate an admin token', action="store_true")
p.add_argument('--token-create-smrt', action="store_true")
p.add_argument('--token-create-smrt-remote', default=REMOTE_ADDR)
p.add_argument('--token-create-hunter', action="store_true")
p.add_argument('--token-create-httpd', action="store_true")
p.add_argument('--config-path', help='store the token as a config')
p.add_argument('--token', help='specify the token to use', default=None)
p.add_argument('--token-groups', help="specify groups associated with token [default %(default)s]'", default='everyone')
p.add_argument('--remote', help='specify remote')
args = p.parse_args()
groups = args.token_groups.split(',')
setup_logging(args)
logger = logging.getLogger(__name__)
logger.info('loglevel is: {}'.format(logging.getLevelName(logger.getEffectiveLevel())))
setup_signals(__name__)
if not args.token_create_smrt and not args.token_create_admin and not args.token_create_hunter and not \
args.token_create_httpd:
logger.error('missing required arguments, see -h for more information')
raise SystemExit
if args.token_create_smrt:
with Store(store_type=args.store, nodes=args.nodes) as s:
s._load_plugin(store_type=args.store, nodes=args.nodes)
t = s.token_create_smrt(token=args.token, groups=groups)
if t:
if PYVERSION == 2:
t = t.encode('utf-8')
data = {
'token': t,
}
if args.remote:
data['remote'] = args.remote
if args.config_path:
with open(args.config_path, 'w') as f:
f.write(yaml.dump(data, default_flow_style=False))
logger.info('token config generated: {}'.format(args.token_create_smrt))
else:
logger.error('token not created')
if args.token_create_hunter:
with Store(store_type=args.store, nodes=args.nodes) as s:
s._load_plugin(store_type=args.store, nodes=args.nodes)
t = s.token_create_hunter(token=args.token, groups=groups)
if t:
if PYVERSION == 2:
t = t.encode('utf-8')
data = {
'hunter_token': t,
}
if args.config_path:
with open(args.config_path, 'w') as f:
f.write(yaml.dump(data, default_flow_style=False))
logger.info('token config generated: {}'.format(args.token_create_hunter))
else:
logger.error('token not created')
if args.token_create_admin:
with Store(store_type=args.store, nodes=args.nodes) as s:
s._load_plugin(store_type=args.store, nodes=args.nodes)
t = s.token_create_admin(token=args.token, groups=groups)
if t:
if PYVERSION == 2:
t = t.encode('utf-8')
data = {
'token': t,
}
if args.config_path:
with open(args.config_path, 'w') as f:
f.write(yaml.dump(data, default_flow_style=False))
logger.info('token config generated: {}'.format(args.token_create_admin))
else:
logger.error('token not created')
if args.token_create_httpd:
with Store(store_type=args.store, nodes=args.nodes) as s:
s._load_plugin(store_type=args.store, nodes=args.nodes)
t = s.token_create_httpd(token=args.token, groups=groups)
if t:
if PYVERSION == 2:
t = t.encode('utf-8')
data = {
'token': t,
}
if args.config_path:
with open(args.config_path, 'w') as f:
f.write(yaml.dump(data, default_flow_style=False))
logger.info('token config generated: {}'.format(args.token_create_httpd))
else:
logger.error('token not created')
