def main():
p = get_argument_parser()
p = ArgumentParser(
description=textwrap.dedent('''\
example usage:
$ cif-hunter -d
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='cif-hunter',
parents=[p],
)
p.add_argument('--remote', help="cif-router hunter address [default %(default)s]", default=HUNTER_ADDR)
p.add_argument('--router', help='cif-router front end address [default %(default)s]', default=ROUTER_ADDR)
p.add_argument('--token', help='specify cif-hunter token [default %(default)s]', default=TOKEN)
args = p.parse_args()
setup_logging(args)
logger = logging.getLogger(__name__)
logger.info('loglevel is: {}'.format(logging.getLevelName(logger.getEffectiveLevel())))
setup_signals(__name__)
with Hunter(remote=args.remote, router=args.router, token=args.token) as h:
try:
logger.info('starting up...')
h.start()
except KeyboardInterrupt:
logging.info("shutting down...")
def main():
p = get_argument_parser()
p = ArgumentParser(description=textwrap.dedent('''\
example usage:
$ cif-gatherer
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='cif-gatherer',
parents=[p])
p.add_argument('--remote',
help="cif-router gatherer address [default %(default)s]",
default=GATHER_ADDR)
p.add_argument('--router',
help='cif-router front end address [default %(default)s]',
default=ROUTER_ADDR)
p.add_argument('--token',
help='specify cif-gatherer token [default %(default)s]',
default=TOKEN)
args = p.parse_args()
setup_logging(args)
logger = logging.getLogger(__name__)
logger.info('loglevel is: {}'.format(
logging.getLevelName(logger.getEffectiveLevel())))
setup_signals(__name__)
with Gatherer(remote=args.remote, router=args.router,
token=args.token) as r:
logger.info('staring gatherer...')
try:
r.start()
except KeyboardInterrupt:
logger.info('shutting down...')
def main():
p = get_argument_parser()
p = ArgumentParser(description=textwrap.dedent('''\
Env Variables:
CIF_RUNTIME_PATH
CIF_ROUTER_ADDR
CIF_HUNTER_ADDR
CIF_STORAGE_ADDR
example usage:
$ cif-router --listen 0.0.0.0 -d
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='cif-router',
parents=[p])
p.add_argument('--listen',
help='address to listen on [default: %(default)s]',
default=ROUTER_ADDR)
p.add_argument('--hunter',
help='address hunters listen on on [default: %(default)s]',
default=HUNTER_ADDR)
p.add_argument("--storage",
help="specify a storage address [default: %(default)s]",
default=STORAGE_ADDR)
p.add_argument('--p2p',
action='store_true',
help='enable experimental p2p support')
args = p.parse_args()
setup_logging(args)
logger = logging.getLogger(__name__)
logger.info('loglevel is: {}'.format(
logging.getLevelName(logger.getEffectiveLevel())))
setup_signals(__name__)
with Router(listen=args.listen,
hunter=args.hunter,
storage=args.storage,
p2p=args.p2p) as r:
try:
logger.info('starting router..')
r.run()
except KeyboardInterrupt:
logger.info('shutting down...')
logger.info('Shutting down')
def main():
p = get_argument_parser()
p = ArgumentParser(
description=textwrap.dedent('''\
example usage:
$ cif-gatherer
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='cif-gatherer',
parents=[p]
)
p.add_argument("--remote", dest="remote", help="specify the cif-router publishing channel [default: %(default)s",
default=GATHER_ADDR)
args = p.parse_args()
logger = setup_logging(args)
logger.info('loglevel is: {}'.format(logging.getLevelName(logger.getEffectiveLevel())))
setup_signals(__name__)
with Gatherer() as r:
logger.info('staring gatherer...')
try:
r.run()
except KeyboardInterrupt:
logger.info('shutting down...')
def main():
p = get_argument_parser()
p = ArgumentParser(description=textwrap.dedent('''\
Env Variables:
CIF_RUNTIME_PATH
CIF_STORAGE_ADDR
example usage:
$ cif-storage -d
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='cif-storage',
parents=[p])
p.add_argument(
"--storage-address",
help="specify the storage address cif-router is listening on[default: %("
"default)s]",
default=STORAGE_ADDR)
p.add_argument(
"--store",
help="specify a store type {} [default: %(default)s]".format(
', '.join(STORE_PLUGINS)),
default=STORE_DEFAULT)
args = p.parse_args()
setup_logging(args)
logger = logging.getLogger(__name__)
logger.info('loglevel is: {}'.format(
logging.getLevelName(logger.getEffectiveLevel())))
setup_signals(__name__)
# from cif.reloader import ModuleWatcher
# mw = ModuleWatcher()
# mw.watch_module('cif.storage')
# mw.start_watching()
with Storage(storage_address=args.storage_address, store=args.store) as s:
try:
logger.info('starting up...')
s.start()
except KeyboardInterrupt:
logger.info('shutting down...')
def main():
p = get_argument_parser()
p = ArgumentParser(description=textwrap.dedent('''\
example usage:
$ cif-httpd -d
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='cif-httpd',
parents=[p])
p.add_argument("--router",
help="specify router frontend [default %(default)s]",
default=ROUTER_ADDR)
p.add_argument('--token',
help="specify cif-httpd token [default %(default)s]",
default=TOKEN)
p.add_argument(
'--listen',
help='specify the interface to listen on [default %(default)s]',
default=HTTP_LISTEN)
p.add_argument('--listen-port',
help='specify the port to listen on [default %(default)s]',
default=HTTP_LISTEN_PORT)
p.add_argument('--fdebug', action='store_true')
args = p.parse_args()
setup_logging(args)
logger = logging.getLogger(__name__)
logger.info('loglevel is: {}'.format(
logging.getLevelName(logger.getEffectiveLevel())))
setup_signals(__name__)
try:
logger.info('pinging router...')
if Client(args.router, args.token).ping():
app.config["SECRET_KEY"] = os.urandom(1024)
logger.info('starting up...')
app.run(host=args.listen, port=args.listen_port, debug=args.fdebug)
else:
logger.error('router unavailable...')
except KeyboardInterrupt:
logger.info('shutting down...')
raise SystemExit
def main():
p = get_argument_parser()
p = ArgumentParser(
description=textwrap.dedent('''\
Env Variables:
CIF_RUNTIME_PATH
CIF_STORAGE_ADDR
example usage:
$ cif-storage -d
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='cif-storage',
parents=[p]
)
p.add_argument("--storage-address", help="specify the storage address cif-router is listening on[default: %("
"default)s]", default=STORAGE_ADDR)
p.add_argument("--store", help="specify a store type {} [default: %(default)s]".format(', '.join(STORE_PLUGINS)),
default=STORE_DEFAULT)
args = p.parse_args()
setup_logging(args)
logger = logging.getLogger(__name__)
logger.info('loglevel is: {}'.format(logging.getLevelName(logger.getEffectiveLevel())))
setup_signals(__name__)
# from cif.reloader import ModuleWatcher
# mw = ModuleWatcher()
# mw.watch_module('cif.storage')
# mw.start_watching()
with Storage(storage_address=args.storage_address, store=args.store) as s:
try:
logger.info('starting up...')
s.start()
except KeyboardInterrupt:
logger.info('shutting down...')
def main():
p = get_argument_parser()
p = ArgumentParser(
description=textwrap.dedent('''\
Env Variables:
CIF_RUNTIME_PATH
CIF_ROUTER_ADDR
CIF_HUNTER_ADDR
CIF_STORAGE_ADDR
example usage:
$ cif-router --listen 0.0.0.0 -d
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='cif-router',
parents=[p]
)
p.add_argument('--listen', help='address to listen on [default: %(default)s]', default=ROUTER_ADDR)
p.add_argument('--hunter', help='address hunters listen on on [default: %(default)s]', default=HUNTER_ADDR)
p.add_argument("--storage", help="specify a storage address [default: %(default)s]",
default=STORAGE_ADDR)
p.add_argument('--p2p', action='store_true', help='enable experimental p2p support')
args = p.parse_args()
setup_logging(args)
logger = logging.getLogger(__name__)
logger.info('loglevel is: {}'.format(logging.getLevelName(logger.getEffectiveLevel())))
setup_signals(__name__)
with Router(listen=args.listen, hunter=args.hunter, storage=args.storage, p2p=args.p2p) as r:
try:
logger.info('starting router..')
r.run()
except KeyboardInterrupt:
logger.info('shutting down...')
logger.info('Shutting down')
def main():
p = get_argument_parser()
p = ArgumentParser(
description=textwrap.dedent(
"""\
Env Variables:
CIF_RUNTIME_PATH
CIF_ROUTER_ADDR
CIF_HUNTER_ADDR
CIF_STORAGE_ADDR
example usage:
$ cif-router --listen 0.0.0.0 -d
"""
),
formatter_class=RawDescriptionHelpFormatter,
prog="cif-router",
parents=[p],
)
p.add_argument("--listen", help="address to listen on [default: %(default)s]", default=ROUTER_ADDR)
p.add_argument("--hunter", help="address hunters listen on on [default: %(default)s]", default=HUNTER_ADDR)
p.add_argument("--storage", help="specify a storage address [default: %(default)s]", default=STORAGE_ADDR)
args = p.parse_args()
setup_logging(args)
logger = logging.getLogger(__name__)
logger.info("loglevel is: {}".format(logging.getLevelName(logger.getEffectiveLevel())))
setup_signals(__name__)
with Router(listen=args.listen, hunter=args.hunter, storage=args.storage) as r:
try:
logger.info("starting router..")
r.run()
except KeyboardInterrupt:
logger.info("shutting down...")
logger.info("Shutting down")
def main():
p = get_argument_parser()
p = ArgumentParser(
description=textwrap.dedent('''\
example usage:
$ cif-httpd -d
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='cif-httpd',
parents=[p]
)
p.add_argument("--router", help="specify router frontend [default %(default)s]", default=ROUTER_ADDR)
p.add_argument('--token', help="specify cif-httpd token [default %(default)s]", default=TOKEN)
p.add_argument('--listen', help='specify the interface to listen on [default %(default)s]', default=HTTP_LISTEN)
p.add_argument('--listen-port', help='specify the port to listen on [default %(default)s]',
default=HTTP_LISTEN_PORT)
p.add_argument('--fdebug', action='store_true')
args = p.parse_args()
setup_logging(args)
logger = logging.getLogger(__name__)
logger.info('loglevel is: {}'.format(logging.getLevelName(logger.getEffectiveLevel())))
setup_signals(__name__)
try:
logger.info('pinging router...')
if Client(args.router, args.token).ping():
app.config["SECRET_KEY"] = os.urandom(1024)
logger.info('starting up...')
app.run(host=args.listen, port=args.listen_port, debug=args.fdebug)
else:
logger.error('router unavailable...')
except KeyboardInterrupt:
logger.info('shutting down...')
raise SystemExit
def main():
p = get_argument_parser()
p = ArgumentParser(
description=textwrap.dedent('''\
example usage:
$ cif -q example.org -d
$ cif --search 1.2.3.0/24
$ cif --ping
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='cif',
parents=[p]
)
p.add_argument('--token', help='specify api token', default=str(1234))
p.add_argument('--remote', help='specify API remote [default %(default)s]', default=REMOTE_ADDR)
p.add_argument('-p', '--ping', action="store_true") # meg?
p.add_argument('-q', '--search', help="search")
p.add_argument('--itype', help='filter by indicator type') ## need to fix sqlite for non-ascii stuff first
p.add_argument("--submit", action="store_true", help="submit an indicator")
p.add_argument('--limit', help='limit results [default %(default)s]', default=SEARCH_LIMIT)
p.add_argument('--nolog', help='do not log search', default=False)
p.add_argument('--indicator')
p.add_argument('--tags', nargs='+')
p.add_argument("--zmq", dest="zmq", help="use zmq as a transport instead of http", action="store_true")
args = p.parse_args()
setup_logging(args)
logger = logging.getLogger(__name__)
options = vars(args)
if options.get("zmq"):
from cif.client.zeromq import ZMQ as ZMQClient
cli = ZMQClient(**options)
else:
from cif.client.http import HTTP as HTTPClient
cli = HTTPClient(args.remote, args.token)
if options.get('ping'):
logger.info('running ping')
for num in range(0, 4):
ret = cli.ping()
if ret != 0:
print("roundtrip: {} ms".format(ret))
select.select([], [], [], 1)
else:
logger.error('ping failed')
raise RuntimeError
elif options.get('itype'):
logger.info('searching for {}'.format(options['itype']))
try:
rv = cli.search({
'itype': options['itype'],
'limit': options['limit'],
})
except RuntimeError as e:
import traceback
traceback.print_exc()
logger.error(e)
else:
print(Table(data=rv))
elif options.get('search'):
logger.info("searching for {0}".format(options.get("search")))
try:
rv = cli.search({
'indicator': options['search'],
'limit': options['limit'],
'nolog': options['nolog']
}
)
except RuntimeError as e:
import traceback
traceback.print_exc()
logger.error(e)
else:
print(Table(data=rv))
elif options.get("submit"):
logger.info("submitting {0}".format(options.get("submit")))
rv = cli.submit(indicator=args.indicator, tags=args.tags)
def main():
p = get_argument_parser()
p = ArgumentParser(description=textwrap.dedent('''\
example usage:
$ cif -q example.org -d
$ cif --search 1.2.3.0/24
$ cif --ping
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='cif',
parents=[p])
p.add_argument('--token', help='specify api token', default=str(1234))
p.add_argument('--remote',
help='specify API remote [default %(default)s]',
default=REMOTE_ADDR)
p.add_argument('-p', '--ping', action="store_true") # meg?
p.add_argument('-q', '--search', help="search")
p.add_argument('--itype', help='filter by indicator type'
) ## need to fix sqlite for non-ascii stuff first
p.add_argument("--submit", action="store_true", help="submit an indicator")
p.add_argument('--limit',
help='limit results [default %(default)s]',
default=SEARCH_LIMIT)
p.add_argument('--nolog', help='do not log search', default=False)
p.add_argument('--indicator')
p.add_argument('--tags', nargs='+')
p.add_argument("--zmq",
dest="zmq",
help="use zmq as a transport instead of http",
action="store_true")
args = p.parse_args()
setup_logging(args)
logger = logging.getLogger(__name__)
options = vars(args)
if options.get("zmq"):
from cif.client.zeromq import ZMQ as ZMQClient
cli = ZMQClient(**options)
else:
from cif.client.http import HTTP as HTTPClient
cli = HTTPClient(args.remote, args.token)
if options.get('ping'):
logger.info('running ping')
for num in range(0, 4):
ret = cli.ping()
if ret != 0:
print("roundtrip: {} ms".format(ret))
select.select([], [], [], 1)
else:
logger.error('ping failed')
raise RuntimeError
elif options.get('itype'):
logger.info('searching for {}'.format(options['itype']))
try:
rv = cli.search({
'itype': options['itype'],
'limit': options['limit'],
})
except RuntimeError as e:
import traceback
traceback.print_exc()
logger.error(e)
else:
print(Table(data=rv))
elif options.get('search'):
logger.info("searching for {0}".format(options.get("search")))
try:
rv = cli.search({
'indicator': options['search'],
'limit': options['limit'],
'nolog': options['nolog']
})
except RuntimeError as e:
import traceback
traceback.print_exc()
logger.error(e)
else:
print(Table(data=rv))
elif options.get("submit"):
logger.info("submitting {0}".format(options.get("submit")))
rv = cli.submit(indicator=args.indicator, tags=args.tags)
def main():
p = get_argument_parser()
p = ArgumentParser(
description=textwrap.dedent('''\
Env Variables:
CIF_RUNTIME_PATH
CIF_HTTP_ADDR
CIF_TOKEN
example usage:
$ cif-smrt -v --rules rules/default
$ cif-smrt --rules rules/default/drg.yml --feed ssh
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='cif-smrt',
parents=[p],
)
p.add_argument(
"-r",
"--rule",
help=
"specify the rules directory or specific rules file [default: %(default)s",
default=SMRT_RULES_PATH)
p.add_argument("-f", "--feed", help="specify the feed to process")
p.add_argument("--remote",
dest="remote",
help="specify the remote api url [default: %(default)s",
default=REMOTE_ADDR)
p.add_argument('--cache',
help="specify feed cache [default %(default)s]",
default=SMRT_CACHE)
p.add_argument(
"--limit",
dest="limit",
help="limit the number of records processed [default: %(default)s]",
default=None)
p.add_argument("--token",
dest="token",
help="specify token [default: %(default)s]",
default=TOKEN)
p.add_argument('--test', action='store_true')
p.add_argument('--sleep', default=60)
p.add_argument('--ignore-unknown', action='store_true')
args = p.parse_args()
setup_logging(args)
logger = logging.getLogger(__name__)
logger.info('loglevel is: {}'.format(
logging.getLevelName(logger.getEffectiveLevel())))
setup_signals(__name__)
stop = False
r = False
if not args.test:
r = randint(5, 55)
logger.info(
"random delay is {}, then running every 60min after that".format(
r))
sleep((r * 60))
while not stop:
if args.test:
stop = True
logger.info('starting...')
try:
with Smrt(args.remote, args.token) as s:
logger.info('staring up...')
x = s.process(args.rule, feed=args.feed, limit=args.limit)
logger.info('complete')
if not args.test:
logger.info('sleeping for 1 hour')
sleep((60 * 60))
except RuntimeError as e:
logger.error(e)
if str(e).startswith('submission failed'):
stop = True
else:
logging.exception('Got exception on main handler')
except KeyboardInterrupt:
logger.info('shutting down')
stop = True
logger.info('completed')
def main():
p = get_argument_parser()
p = ArgumentParser(
description=textwrap.dedent('''\
Env Variables:
CIF_RUNTIME_PATH
CIF_HTTP_ADDR
CIF_TOKEN
example usage:
$ cif-smrt -v --rules rules/default
$ cif-smrt --rules rules/default/drg.yml --feed ssh
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='cif-smrt',
parents=[p],
)
p.add_argument("-r", "--rule", help="specify the rules directory or specific rules file [default: %(default)s",
default=SMRT_RULES_PATH)
p.add_argument("-f", "--feed", help="specify the feed to process")
p.add_argument("--remote", dest="remote", help="specify the remote api url [default: %(default)s",
default=REMOTE_ADDR)
p.add_argument('--cache', help="specify feed cache [default %(default)s]", default=SMRT_CACHE)
p.add_argument("--limit", dest="limit", help="limit the number of records processed [default: %(default)s]",
default=None)
p.add_argument("--token", dest="token", help="specify token [default: %(default)s]", default=TOKEN)
p.add_argument('--test', action='store_true')
p.add_argument('--sleep', default=60)
p.add_argument('--ignore-unknown', action='store_true')
args = p.parse_args()
setup_logging(args)
logger = logging.getLogger(__name__)
logger.info('loglevel is: {}'.format(logging.getLevelName(logger.getEffectiveLevel())))
setup_signals(__name__)
stop = False
r = False
if not args.test:
r = randint(5, 55)
logger.info("random delay is {}, then running every 60min after that".format(r))
sleep((r * 60))
while not stop:
if args.test:
stop = True
logger.info('starting...')
try:
with Smrt(args.remote, args.token) as s:
logger.info('staring up...')
x = s.process(args.rule, feed=args.feed, limit=args.limit)
logger.info('complete')
if not args.test:
logger.info('sleeping for 1 hour')
sleep((60 * 60))
except RuntimeError as e:
logger.error(e)
if str(e).startswith('submission failed'):
stop = True
else:
logging.exception('Got exception on main handler')
except KeyboardInterrupt:
logger.info('shutting down')
stop = True
logger.info('completed')
import logging
import os
import tempfile
from argparse import Namespace
import py.test
from cif.indicator import Indicator
from cif.storage import Storage
from cif.utils import setup_logging
from pprint import pprint
args = Namespace(debug=True, verbose=None)
setup_logging(args)
logger = logging.getLogger(__name__)
@py.test.yield_fixture
def storage():
dbfile = tempfile.mktemp()
with Storage(store='sqlite', dbfile=dbfile) as s:
yield s
os.unlink(dbfile)
@py.test.fixture
def obs():
return Indicator(indicator='example.com', tags=['botnet'], provider='csirtgadgets.org')
import logging
import os
import tempfile
from argparse import Namespace
import py.test
from cif.indicator import Indicator
from cif.storage import Storage
from cif.utils import setup_logging
args = Namespace(debug=True, verbose=None)
setup_logging(args)
logger = logging.getLogger(__name__)
@py.test.fixture
def obs():
return Indicator(indicator='example.com',
tags=['botnet'],
provider='csirtgadgets.org')
def test_storage(obs):
with Storage(store='dummy') as s:
x = s.handle_search('1234', obs.__dict__)
assert x[0]['indicator'] == 'example.com'
x = s.handle_submission('1234', obs.__dict__)
assert x[0]['indicator'] == 'example.com'
