def main(): p = get_argument_parser() p = ArgumentParser( description=textwrap.dedent('''\ example usage: $ cif-hunter -d '''), formatter_class=RawDescriptionHelpFormatter, prog='cif-hunter', parents=[p], ) p.add_argument('--remote', help="cif-router hunter address [default %(default)s]", default=HUNTER_ADDR) p.add_argument('--router', help='cif-router front end address [default %(default)s]', default=ROUTER_ADDR) p.add_argument('--token', help='specify cif-hunter token [default %(default)s]', default=TOKEN) args = p.parse_args() setup_logging(args) logger = logging.getLogger(__name__) logger.info('loglevel is: {}'.format(logging.getLevelName(logger.getEffectiveLevel()))) setup_signals(__name__) with Hunter(remote=args.remote, router=args.router, token=args.token) as h: try: logger.info('starting up...') h.start() except KeyboardInterrupt: logging.info("shutting down...")
def main(): p = get_argument_parser() p = ArgumentParser(description=textwrap.dedent('''\ example usage: $ cif-gatherer '''), formatter_class=RawDescriptionHelpFormatter, prog='cif-gatherer', parents=[p]) p.add_argument('--remote', help="cif-router gatherer address [default %(default)s]", default=GATHER_ADDR) p.add_argument('--router', help='cif-router front end address [default %(default)s]', default=ROUTER_ADDR) p.add_argument('--token', help='specify cif-gatherer token [default %(default)s]', default=TOKEN) args = p.parse_args() setup_logging(args) logger = logging.getLogger(__name__) logger.info('loglevel is: {}'.format( logging.getLevelName(logger.getEffectiveLevel()))) setup_signals(__name__) with Gatherer(remote=args.remote, router=args.router, token=args.token) as r: logger.info('staring gatherer...') try: r.start() except KeyboardInterrupt: logger.info('shutting down...')
def main(): p = get_argument_parser() p = ArgumentParser(description=textwrap.dedent('''\ Env Variables: CIF_RUNTIME_PATH CIF_ROUTER_ADDR CIF_HUNTER_ADDR CIF_STORAGE_ADDR example usage: $ cif-router --listen 0.0.0.0 -d '''), formatter_class=RawDescriptionHelpFormatter, prog='cif-router', parents=[p]) p.add_argument('--listen', help='address to listen on [default: %(default)s]', default=ROUTER_ADDR) p.add_argument('--hunter', help='address hunters listen on on [default: %(default)s]', default=HUNTER_ADDR) p.add_argument("--storage", help="specify a storage address [default: %(default)s]", default=STORAGE_ADDR) p.add_argument('--p2p', action='store_true', help='enable experimental p2p support') args = p.parse_args() setup_logging(args) logger = logging.getLogger(__name__) logger.info('loglevel is: {}'.format( logging.getLevelName(logger.getEffectiveLevel()))) setup_signals(__name__) with Router(listen=args.listen, hunter=args.hunter, storage=args.storage, p2p=args.p2p) as r: try: logger.info('starting router..') r.run() except KeyboardInterrupt: logger.info('shutting down...') logger.info('Shutting down')
def main(): p = get_argument_parser() p = ArgumentParser( description=textwrap.dedent('''\ example usage: $ cif-gatherer '''), formatter_class=RawDescriptionHelpFormatter, prog='cif-gatherer', parents=[p] ) p.add_argument("--remote", dest="remote", help="specify the cif-router publishing channel [default: %(default)s", default=GATHER_ADDR) args = p.parse_args() logger = setup_logging(args) logger.info('loglevel is: {}'.format(logging.getLevelName(logger.getEffectiveLevel()))) setup_signals(__name__) with Gatherer() as r: logger.info('staring gatherer...') try: r.run() except KeyboardInterrupt: logger.info('shutting down...')
def main(): p = get_argument_parser() p = ArgumentParser(description=textwrap.dedent('''\ Env Variables: CIF_RUNTIME_PATH CIF_STORAGE_ADDR example usage: $ cif-storage -d '''), formatter_class=RawDescriptionHelpFormatter, prog='cif-storage', parents=[p]) p.add_argument( "--storage-address", help="specify the storage address cif-router is listening on[default: %(" "default)s]", default=STORAGE_ADDR) p.add_argument( "--store", help="specify a store type {} [default: %(default)s]".format( ', '.join(STORE_PLUGINS)), default=STORE_DEFAULT) args = p.parse_args() setup_logging(args) logger = logging.getLogger(__name__) logger.info('loglevel is: {}'.format( logging.getLevelName(logger.getEffectiveLevel()))) setup_signals(__name__) # from cif.reloader import ModuleWatcher # mw = ModuleWatcher() # mw.watch_module('cif.storage') # mw.start_watching() with Storage(storage_address=args.storage_address, store=args.store) as s: try: logger.info('starting up...') s.start() except KeyboardInterrupt: logger.info('shutting down...')
def main(): p = get_argument_parser() p = ArgumentParser(description=textwrap.dedent('''\ example usage: $ cif-httpd -d '''), formatter_class=RawDescriptionHelpFormatter, prog='cif-httpd', parents=[p]) p.add_argument("--router", help="specify router frontend [default %(default)s]", default=ROUTER_ADDR) p.add_argument('--token', help="specify cif-httpd token [default %(default)s]", default=TOKEN) p.add_argument( '--listen', help='specify the interface to listen on [default %(default)s]', default=HTTP_LISTEN) p.add_argument('--listen-port', help='specify the port to listen on [default %(default)s]', default=HTTP_LISTEN_PORT) p.add_argument('--fdebug', action='store_true') args = p.parse_args() setup_logging(args) logger = logging.getLogger(__name__) logger.info('loglevel is: {}'.format( logging.getLevelName(logger.getEffectiveLevel()))) setup_signals(__name__) try: logger.info('pinging router...') if Client(args.router, args.token).ping(): app.config["SECRET_KEY"] = os.urandom(1024) logger.info('starting up...') app.run(host=args.listen, port=args.listen_port, debug=args.fdebug) else: logger.error('router unavailable...') except KeyboardInterrupt: logger.info('shutting down...') raise SystemExit
def main(): p = get_argument_parser() p = ArgumentParser( description=textwrap.dedent('''\ Env Variables: CIF_RUNTIME_PATH CIF_STORAGE_ADDR example usage: $ cif-storage -d '''), formatter_class=RawDescriptionHelpFormatter, prog='cif-storage', parents=[p] ) p.add_argument("--storage-address", help="specify the storage address cif-router is listening on[default: %(" "default)s]", default=STORAGE_ADDR) p.add_argument("--store", help="specify a store type {} [default: %(default)s]".format(', '.join(STORE_PLUGINS)), default=STORE_DEFAULT) args = p.parse_args() setup_logging(args) logger = logging.getLogger(__name__) logger.info('loglevel is: {}'.format(logging.getLevelName(logger.getEffectiveLevel()))) setup_signals(__name__) # from cif.reloader import ModuleWatcher # mw = ModuleWatcher() # mw.watch_module('cif.storage') # mw.start_watching() with Storage(storage_address=args.storage_address, store=args.store) as s: try: logger.info('starting up...') s.start() except KeyboardInterrupt: logger.info('shutting down...')
def main(): p = get_argument_parser() p = ArgumentParser( description=textwrap.dedent('''\ Env Variables: CIF_RUNTIME_PATH CIF_ROUTER_ADDR CIF_HUNTER_ADDR CIF_STORAGE_ADDR example usage: $ cif-router --listen 0.0.0.0 -d '''), formatter_class=RawDescriptionHelpFormatter, prog='cif-router', parents=[p] ) p.add_argument('--listen', help='address to listen on [default: %(default)s]', default=ROUTER_ADDR) p.add_argument('--hunter', help='address hunters listen on on [default: %(default)s]', default=HUNTER_ADDR) p.add_argument("--storage", help="specify a storage address [default: %(default)s]", default=STORAGE_ADDR) p.add_argument('--p2p', action='store_true', help='enable experimental p2p support') args = p.parse_args() setup_logging(args) logger = logging.getLogger(__name__) logger.info('loglevel is: {}'.format(logging.getLevelName(logger.getEffectiveLevel()))) setup_signals(__name__) with Router(listen=args.listen, hunter=args.hunter, storage=args.storage, p2p=args.p2p) as r: try: logger.info('starting router..') r.run() except KeyboardInterrupt: logger.info('shutting down...') logger.info('Shutting down')
def main(): p = get_argument_parser() p = ArgumentParser( description=textwrap.dedent( """\ Env Variables: CIF_RUNTIME_PATH CIF_ROUTER_ADDR CIF_HUNTER_ADDR CIF_STORAGE_ADDR example usage: $ cif-router --listen 0.0.0.0 -d """ ), formatter_class=RawDescriptionHelpFormatter, prog="cif-router", parents=[p], ) p.add_argument("--listen", help="address to listen on [default: %(default)s]", default=ROUTER_ADDR) p.add_argument("--hunter", help="address hunters listen on on [default: %(default)s]", default=HUNTER_ADDR) p.add_argument("--storage", help="specify a storage address [default: %(default)s]", default=STORAGE_ADDR) args = p.parse_args() setup_logging(args) logger = logging.getLogger(__name__) logger.info("loglevel is: {}".format(logging.getLevelName(logger.getEffectiveLevel()))) setup_signals(__name__) with Router(listen=args.listen, hunter=args.hunter, storage=args.storage) as r: try: logger.info("starting router..") r.run() except KeyboardInterrupt: logger.info("shutting down...") logger.info("Shutting down")
def main(): p = get_argument_parser() p = ArgumentParser( description=textwrap.dedent('''\ example usage: $ cif-httpd -d '''), formatter_class=RawDescriptionHelpFormatter, prog='cif-httpd', parents=[p] ) p.add_argument("--router", help="specify router frontend [default %(default)s]", default=ROUTER_ADDR) p.add_argument('--token', help="specify cif-httpd token [default %(default)s]", default=TOKEN) p.add_argument('--listen', help='specify the interface to listen on [default %(default)s]', default=HTTP_LISTEN) p.add_argument('--listen-port', help='specify the port to listen on [default %(default)s]', default=HTTP_LISTEN_PORT) p.add_argument('--fdebug', action='store_true') args = p.parse_args() setup_logging(args) logger = logging.getLogger(__name__) logger.info('loglevel is: {}'.format(logging.getLevelName(logger.getEffectiveLevel()))) setup_signals(__name__) try: logger.info('pinging router...') if Client(args.router, args.token).ping(): app.config["SECRET_KEY"] = os.urandom(1024) logger.info('starting up...') app.run(host=args.listen, port=args.listen_port, debug=args.fdebug) else: logger.error('router unavailable...') except KeyboardInterrupt: logger.info('shutting down...') raise SystemExit
def main(): p = get_argument_parser() p = ArgumentParser( description=textwrap.dedent('''\ example usage: $ cif -q example.org -d $ cif --search 1.2.3.0/24 $ cif --ping '''), formatter_class=RawDescriptionHelpFormatter, prog='cif', parents=[p] ) p.add_argument('--token', help='specify api token', default=str(1234)) p.add_argument('--remote', help='specify API remote [default %(default)s]', default=REMOTE_ADDR) p.add_argument('-p', '--ping', action="store_true") # meg? p.add_argument('-q', '--search', help="search") p.add_argument('--itype', help='filter by indicator type') ## need to fix sqlite for non-ascii stuff first p.add_argument("--submit", action="store_true", help="submit an indicator") p.add_argument('--limit', help='limit results [default %(default)s]', default=SEARCH_LIMIT) p.add_argument('--nolog', help='do not log search', default=False) p.add_argument('--indicator') p.add_argument('--tags', nargs='+') p.add_argument("--zmq", dest="zmq", help="use zmq as a transport instead of http", action="store_true") args = p.parse_args() setup_logging(args) logger = logging.getLogger(__name__) options = vars(args) if options.get("zmq"): from cif.client.zeromq import ZMQ as ZMQClient cli = ZMQClient(**options) else: from cif.client.http import HTTP as HTTPClient cli = HTTPClient(args.remote, args.token) if options.get('ping'): logger.info('running ping') for num in range(0, 4): ret = cli.ping() if ret != 0: print("roundtrip: {} ms".format(ret)) select.select([], [], [], 1) else: logger.error('ping failed') raise RuntimeError elif options.get('itype'): logger.info('searching for {}'.format(options['itype'])) try: rv = cli.search({ 'itype': options['itype'], 'limit': options['limit'], }) except RuntimeError as e: import traceback traceback.print_exc() logger.error(e) else: print(Table(data=rv)) elif options.get('search'): logger.info("searching for {0}".format(options.get("search"))) try: rv = cli.search({ 'indicator': options['search'], 'limit': options['limit'], 'nolog': options['nolog'] } ) except RuntimeError as e: import traceback traceback.print_exc() logger.error(e) else: print(Table(data=rv)) elif options.get("submit"): logger.info("submitting {0}".format(options.get("submit"))) rv = cli.submit(indicator=args.indicator, tags=args.tags)
def main(): p = get_argument_parser() p = ArgumentParser(description=textwrap.dedent('''\ example usage: $ cif -q example.org -d $ cif --search 1.2.3.0/24 $ cif --ping '''), formatter_class=RawDescriptionHelpFormatter, prog='cif', parents=[p]) p.add_argument('--token', help='specify api token', default=str(1234)) p.add_argument('--remote', help='specify API remote [default %(default)s]', default=REMOTE_ADDR) p.add_argument('-p', '--ping', action="store_true") # meg? p.add_argument('-q', '--search', help="search") p.add_argument('--itype', help='filter by indicator type' ) ## need to fix sqlite for non-ascii stuff first p.add_argument("--submit", action="store_true", help="submit an indicator") p.add_argument('--limit', help='limit results [default %(default)s]', default=SEARCH_LIMIT) p.add_argument('--nolog', help='do not log search', default=False) p.add_argument('--indicator') p.add_argument('--tags', nargs='+') p.add_argument("--zmq", dest="zmq", help="use zmq as a transport instead of http", action="store_true") args = p.parse_args() setup_logging(args) logger = logging.getLogger(__name__) options = vars(args) if options.get("zmq"): from cif.client.zeromq import ZMQ as ZMQClient cli = ZMQClient(**options) else: from cif.client.http import HTTP as HTTPClient cli = HTTPClient(args.remote, args.token) if options.get('ping'): logger.info('running ping') for num in range(0, 4): ret = cli.ping() if ret != 0: print("roundtrip: {} ms".format(ret)) select.select([], [], [], 1) else: logger.error('ping failed') raise RuntimeError elif options.get('itype'): logger.info('searching for {}'.format(options['itype'])) try: rv = cli.search({ 'itype': options['itype'], 'limit': options['limit'], }) except RuntimeError as e: import traceback traceback.print_exc() logger.error(e) else: print(Table(data=rv)) elif options.get('search'): logger.info("searching for {0}".format(options.get("search"))) try: rv = cli.search({ 'indicator': options['search'], 'limit': options['limit'], 'nolog': options['nolog'] }) except RuntimeError as e: import traceback traceback.print_exc() logger.error(e) else: print(Table(data=rv)) elif options.get("submit"): logger.info("submitting {0}".format(options.get("submit"))) rv = cli.submit(indicator=args.indicator, tags=args.tags)
def main(): p = get_argument_parser() p = ArgumentParser( description=textwrap.dedent('''\ Env Variables: CIF_RUNTIME_PATH CIF_HTTP_ADDR CIF_TOKEN example usage: $ cif-smrt -v --rules rules/default $ cif-smrt --rules rules/default/drg.yml --feed ssh '''), formatter_class=RawDescriptionHelpFormatter, prog='cif-smrt', parents=[p], ) p.add_argument( "-r", "--rule", help= "specify the rules directory or specific rules file [default: %(default)s", default=SMRT_RULES_PATH) p.add_argument("-f", "--feed", help="specify the feed to process") p.add_argument("--remote", dest="remote", help="specify the remote api url [default: %(default)s", default=REMOTE_ADDR) p.add_argument('--cache', help="specify feed cache [default %(default)s]", default=SMRT_CACHE) p.add_argument( "--limit", dest="limit", help="limit the number of records processed [default: %(default)s]", default=None) p.add_argument("--token", dest="token", help="specify token [default: %(default)s]", default=TOKEN) p.add_argument('--test', action='store_true') p.add_argument('--sleep', default=60) p.add_argument('--ignore-unknown', action='store_true') args = p.parse_args() setup_logging(args) logger = logging.getLogger(__name__) logger.info('loglevel is: {}'.format( logging.getLevelName(logger.getEffectiveLevel()))) setup_signals(__name__) stop = False r = False if not args.test: r = randint(5, 55) logger.info( "random delay is {}, then running every 60min after that".format( r)) sleep((r * 60)) while not stop: if args.test: stop = True logger.info('starting...') try: with Smrt(args.remote, args.token) as s: logger.info('staring up...') x = s.process(args.rule, feed=args.feed, limit=args.limit) logger.info('complete') if not args.test: logger.info('sleeping for 1 hour') sleep((60 * 60)) except RuntimeError as e: logger.error(e) if str(e).startswith('submission failed'): stop = True else: logging.exception('Got exception on main handler') except KeyboardInterrupt: logger.info('shutting down') stop = True logger.info('completed')
def main(): p = get_argument_parser() p = ArgumentParser( description=textwrap.dedent('''\ Env Variables: CIF_RUNTIME_PATH CIF_HTTP_ADDR CIF_TOKEN example usage: $ cif-smrt -v --rules rules/default $ cif-smrt --rules rules/default/drg.yml --feed ssh '''), formatter_class=RawDescriptionHelpFormatter, prog='cif-smrt', parents=[p], ) p.add_argument("-r", "--rule", help="specify the rules directory or specific rules file [default: %(default)s", default=SMRT_RULES_PATH) p.add_argument("-f", "--feed", help="specify the feed to process") p.add_argument("--remote", dest="remote", help="specify the remote api url [default: %(default)s", default=REMOTE_ADDR) p.add_argument('--cache', help="specify feed cache [default %(default)s]", default=SMRT_CACHE) p.add_argument("--limit", dest="limit", help="limit the number of records processed [default: %(default)s]", default=None) p.add_argument("--token", dest="token", help="specify token [default: %(default)s]", default=TOKEN) p.add_argument('--test', action='store_true') p.add_argument('--sleep', default=60) p.add_argument('--ignore-unknown', action='store_true') args = p.parse_args() setup_logging(args) logger = logging.getLogger(__name__) logger.info('loglevel is: {}'.format(logging.getLevelName(logger.getEffectiveLevel()))) setup_signals(__name__) stop = False r = False if not args.test: r = randint(5, 55) logger.info("random delay is {}, then running every 60min after that".format(r)) sleep((r * 60)) while not stop: if args.test: stop = True logger.info('starting...') try: with Smrt(args.remote, args.token) as s: logger.info('staring up...') x = s.process(args.rule, feed=args.feed, limit=args.limit) logger.info('complete') if not args.test: logger.info('sleeping for 1 hour') sleep((60 * 60)) except RuntimeError as e: logger.error(e) if str(e).startswith('submission failed'): stop = True else: logging.exception('Got exception on main handler') except KeyboardInterrupt: logger.info('shutting down') stop = True logger.info('completed')
import logging import os import tempfile from argparse import Namespace import py.test from cif.indicator import Indicator from cif.storage import Storage from cif.utils import setup_logging from pprint import pprint args = Namespace(debug=True, verbose=None) setup_logging(args) logger = logging.getLogger(__name__) @py.test.yield_fixture def storage(): dbfile = tempfile.mktemp() with Storage(store='sqlite', dbfile=dbfile) as s: yield s os.unlink(dbfile) @py.test.fixture def obs(): return Indicator(indicator='example.com', tags=['botnet'], provider='csirtgadgets.org')
import logging import os import tempfile from argparse import Namespace import py.test from cif.indicator import Indicator from cif.storage import Storage from cif.utils import setup_logging args = Namespace(debug=True, verbose=None) setup_logging(args) logger = logging.getLogger(__name__) @py.test.fixture def obs(): return Indicator(indicator='example.com', tags=['botnet'], provider='csirtgadgets.org') def test_storage(obs): with Storage(store='dummy') as s: x = s.handle_search('1234', obs.__dict__) assert x[0]['indicator'] == 'example.com' x = s.handle_submission('1234', obs.__dict__) assert x[0]['indicator'] == 'example.com'