Exemplo n.º 1
0
def main():
    p = get_argument_parser()
    p = ArgumentParser(
        description=textwrap.dedent('''\
            Env Variables:
                CSIRTG_RUNTIME_PATH

            example usage:
                $ csirtg-ufw -f /var/log/ufw.log
                $ ZYRE_GROUP=honeynet csirtg-ufw -d -f /var/log/ufw.log --client zyre
                $ csirtg-ufw -f /var/log/ufw.log --client csirtg --user wes --feed scanners -d
            '''),
        formatter_class=RawDescriptionHelpFormatter,
        prog='csirtg-ufw',
        parents=[p],
    )

    p.add_argument('--no-verify-ssl', help='turn TLS/SSL verification OFF', action='store_true')
    p.add_argument('-f', '--file', default=FILENAME)
    p.add_argument('--client', default='stdout')
    p.add_argument('--user')
    p.add_argument('--feed')
    p.add_argument('--format', default='csv')
    p.add_argument('--provider', help='specify provider [default %(default)s]', default=PROVIDER)
    p.add_argument('--ignore-client-errors', help='skip when client errors out (eg: HTTP 5XX, etc)', action='store_true')
    p.add_argument('--aggregate', help='specify how many seconds to aggregate batches before sending to client '
                                       '[default %(default)s]', default=60)

    args = p.parse_args()

    if not args.provider:
        raise RuntimeError('Missing --provider flag')

    # setup logging
    setup_logging(args)

    logger.debug('starting on: {}'.format(args.file))

    verify_ssl = True
    if args.no_verify_ssl:
        verify_ssl = False

    from csirtg_smrt import Smrt
    s = Smrt(client=args.client, username=args.user, feed=args.feed, verify_ssl=verify_ssl)

    bucket = set()
    last_t = round_time(round=int(args.aggregate))
    try:
        for line in tail(args.file):

            if 'csirtg-ufw' in line:
                continue

            if '[UFW BLOCK]' not in line:
                continue

            if ' SYN ' not in line:
                continue

            logger.debug(line)

            try:
                i = parse_line(line)

            except AttributeError:
                logger.debug("line not matched: \n{}".format(line))
                continue

            i = Indicator(**i)
            i.provider = args.provider
            u_indicator = ':'.join([i.indicator,'/'.join([i.portlist,i.protocol])])

            if args.aggregate:
                t = round_time(dt=datetime.now(), round=int(args.aggregate))
                if t != last_t:
                    bucket = set()

                last_t = t

                if u_indicator in bucket:
                    logger.info('skipping send {}'.format(u_indicator))
                    continue

                bucket.add(u_indicator)

            if args.client == 'stdout':
                print(FORMATS[args.format](data=[i]))
                continue

            try:
                s.client.indicators_create(i)
                logger.info('indicator created: {}'.format(u_indicator))

            except Exception as e:
                logger.error(e)
                if args.ignore_client_errors:
                    pass

    except KeyboardInterrupt:
        logger.info('SIGINT caught... stopping')
        if args.client != 'stdout':
            try:
                s.client.stop()
            except AttributeError:
                pass

    logger.info('exiting...')
Exemplo n.º 2
0
def main():
    p = get_argument_parser()
    p = ArgumentParser(
        description=textwrap.dedent('''\
        Env Variables:
            CSIRTG_RUNTIME_PATH
            CSIRTG_SMRT_TOKEN

        example usage:
            $ csirtg-smrt --rule rules/default
            $ csirtg-smrt --rule default/csirtg.yml --feed port-scanners --remote http://localhost:5000
        '''),
        formatter_class=RawDescriptionHelpFormatter,
        prog='csirtg-smrt',
        parents=[p],
    )

    p.add_argument(
        "-r",
        "--rule",
        help=
        "specify the rules directory or specific rules file [default: %(default)s",
        default=SMRT_RULES_PATH)

    p.add_argument("-f", "--feed", help="specify the feed to process")

    p.add_argument("--remote", help="specify the remote api url")
    p.add_argument(
        '--remote-type',
        help="specify remote type [cif, csirtg, elasticsearch, syslog, etc]")
    p.add_argument('--client', default='stdout')

    p.add_argument('--cache',
                   help="specify feed cache [default %(default)s]",
                   default=SMRT_CACHE)

    p.add_argument(
        "--limit",
        help="limit the number of records processed [default: %(default)s]",
        default=None)

    p.add_argument("--token",
                   help="specify token [default: %(default)s]",
                   default=TOKEN)

    p.add_argument('--service',
                   action='store_true',
                   help="start in service mode")
    p.add_argument('--service-interval',
                   help='set run interval [in minutes, default %(default)s]',
                   default=SERVICE_INTERVAL)
    p.add_argument('--ignore-unknown', action='store_true')

    p.add_argument('--config',
                   help='specify csirtg-smrt config path [default %(default)s',
                   default=CONFIG_PATH)

    p.add_argument('--user')

    p.add_argument('--delay',
                   help='specify initial delay',
                   default=randint(5, 55))

    p.add_argument('--remember-path',
                   help='specify remember db path [default: %(default)s',
                   default=ARCHIVE_PATH)
    p.add_argument('--remember',
                   help='remember what has been already processed',
                   action='store_true')

    p.add_argument('--format',
                   help='specify output format [default: %(default)s]"',
                   default=FORMAT,
                   choices=FORMATS.keys())

    p.add_argument('--filter-indicator',
                   help='filter for specific indicator, useful in testing')

    p.add_argument('--fireball',
                   help='run in fireball mode, bulk+async magic',
                   action='store_true')
    p.add_argument('--no-fetch',
                   help='do not re-fetch if the cache exists',
                   action='store_true')

    p.add_argument('--no-verify-ssl',
                   help='turn TLS/SSL verification OFF',
                   action='store_true')

    p.add_argument(
        '--goback',
        help=
        'specify default number of days to start out at [default %(default)s]',
        default=GOBACK_DAYS)

    p.add_argument('--fields',
                   help='specify fields for stdout [default %(default)s]"',
                   default=','.join(STDOUT_FIELDS))

    p.add_argument('--skip-invalid',
                   help="skip invalid indicators in DEBUG (-d) mode",
                   action="store_true")
    p.add_argument('--skip-broken',
                   help='skip seemingly broken feeds',
                   action='store_true')

    p.add_argument(
        '--send-retries',
        help=
        'specify how many times to re-try sending indicators after a failure '
        '[default: %(default)s',
        default=5)
    p.add_argument(
        '--send-retries-wait',
        help='how many seconds to wait between retries [default: %(default)s',
        default=30)

    args = p.parse_args()

    o = read_config(args)
    options = vars(args)
    for v in options:
        if options[v] is None:
            options[v] = o.get(v)

    setup_logging(args)
    logger.info('loglevel is: {}'.format(
        logging.getLevelName(logger.getEffectiveLevel())))

    setup_runtime_path(args.runtime_path)

    verify_ssl = True
    if options.get('no_verify_ssl') or o.get('no_verify_ssl'):
        verify_ssl = False

    goback = args.goback
    if goback:
        goback = arrow.utcnow().replace(days=-int(goback))

    if not args.service:
        data = None
        if select.select([
                sys.stdin,
        ], [], [], 0.0)[0]:
            data = sys.stdin.read()

        try:
            _run_smrt(
                options, **{
                    'args': args,
                    'data': data,
                    'verify_ssl': verify_ssl,
                    'goback': goback
                })
        except KeyboardInterrupt:
            logger.info('exiting..')

        raise SystemExit

    # we're running as a service
    setup_signals(__name__)
    service_interval = int(args.service_interval)
    r = int(args.delay)
    logger.info(
        "random delay is {}, then running every {} min after that".format(
            r, service_interval))

    if r != 0:
        try:
            sleep((r * 60))

        except KeyboardInterrupt:
            logger.info('shutting down')
            raise SystemExit

        except Exception as e:
            logger.error(e)
            raise SystemExit

    logger.info('starting...')

    def _run():
        logger.debug('forking process...')
        p = Process(target=_run_smrt,
                    args=(options, ),
                    kwargs={
                        'args': args,
                        'verify_ssl': verify_ssl,
                        'goback': goback,
                        'service_mode': True
                    })

        p.daemon = False
        p.start()
        p.join()
        logger.debug('child process re-joined')

    # first run, PeriodicCallback has builtin wait..
    _run()

    main_loop = ioloop.IOLoop()
    service_interval = (service_interval * 60000)
    loop = ioloop.PeriodicCallback(_run, service_interval)

    try:
        loop.start()
        main_loop.start()

    except KeyboardInterrupt:
        logger.info('exiting..')
        pass

    except Exception as e:
        logger.error(e)
        pass
Exemplo n.º 3
0
def main():
    p = get_argument_parser()
    p = ArgumentParser(
        description=textwrap.dedent('''\
            Env Variables:
                CSIRTG_RUNTIME_PATH

            example usage:
                $ csirtg-cef -f /var/log/foo.log
                $ ZYRE_GROUP=honeynet csirtg-cef -d -f /var/log/foo.log --client zyre
                $ csirtg-cef -f /var/log/foo.log --client csirtg --user wes --feed scanners -d
            '''),
        formatter_class=RawDescriptionHelpFormatter,
        prog='csirtg-cef',
        parents=[p],
    )

    p.add_argument('--no-verify-ssl',
                   help='turn TLS/SSL verification OFF',
                   action='store_true')
    p.add_argument('-f', '--file')
    p.add_argument('--client', default='stdout')
    p.add_argument('--user')
    p.add_argument('--feed')
    p.add_argument('--format', default='csv')
    p.add_argument('--tags',
                   help='specify indicator tags [default %(default)s',
                   default='scanner')
    p.add_argument('--provider',
                   help='specify provider [default %(default)s]',
                   default=PROVIDER)

    p.add_argument('--tail-docker')

    args = p.parse_args()

    if not args.provider:
        raise RuntimeError('Missing --provider flag')
    if not args.file:
        raise RuntimeError('Missing --file flag')

    # setup logging
    setup_logging(args)

    logger.debug('starting on: {}'.format(args.file))

    verify_ssl = True
    if args.no_verify_ssl:
        verify_ssl = False

    f = open(args.file)
    from csirtg_smrt import Smrt
    s = Smrt(client=args.client,
             username=args.user,
             feed=args.feed,
             verify_ssl=verify_ssl)

    try:
        for line in tailer.follow(f):
            i = parse_line(line)

            if not i:
                logger.debug('skipping line')
                continue

            i = Indicator(**i)

            logger.debug(i)

            i.provider = args.provider
            i.tags = args.tags

            if args.client == 'stdout':
                print(FORMATS[args.format](data=[i]))
            else:
                s.client.indicators_create(i)
                logger.info('indicator created: {}'.format(i.indicator))

    except KeyboardInterrupt:
        logger.info('SIGINT caught... stopping')
        if args.client != 'stdout':
            s.client.stop()

    logger.info('exiting...')
Exemplo n.º 4
0
def main():
    p = get_argument_parser()
    p = ArgumentParser(
        description=textwrap.dedent('''\
        Env Variables:
            CSIRTG_RUNTIME_PATH
            CSIRTG_SMRT_TOKEN

        example usage:
            $ csirtg-smrt --rule rules/default
            $ csirtg-smrt --rule default/csirtg.yml --feed port-scanners --remote http://localhost:5000
        '''),
        formatter_class=RawDescriptionHelpFormatter,
        prog='csirtg-smrt',
        parents=[p],
    )

    p.add_argument("-r", "--rule", help="specify the rules directory or specific rules file [default: %(default)s",
                   default=SMRT_RULES_PATH)

    p.add_argument("-f", "--feed", help="specify the feed to process")

    p.add_argument("--remote", help="specify the remote api url")
    p.add_argument('--remote-type', help="specify remote type [cif, csirtg, elasticsearch, syslog, etc]")
    p.add_argument('--client', default='stdout')

    p.add_argument('--cache', help="specify feed cache [default %(default)s]", default=SMRT_CACHE)

    p.add_argument("--limit", help="limit the number of records processed [default: %(default)s]",
                   default=None)

    p.add_argument("--token", help="specify token [default: %(default)s]", default=TOKEN)

    p.add_argument('--service', action='store_true', help="start in service mode")
    p.add_argument('--service-interval', help='set run interval [in minutes, default %(default)s]',
                   default=SERVICE_INTERVAL)
    p.add_argument('--ignore-unknown', action='store_true')

    p.add_argument('--config', help='specify csirtg-smrt config path [default %(default)s', default=CONFIG_PATH)

    p.add_argument('--user')

    p.add_argument('--delay', help='specify initial delay', default=randint(5, 55))

    p.add_argument('--remember-path', help='specify remember db path [default: %(default)s', default=ARCHIVE_PATH)
    p.add_argument('--remember', help='remember what has been already processed', action='store_true')

    p.add_argument('--format', help='specify output format [default: %(default)s]"', default=FORMAT,
                   choices=FORMATS.keys())

    p.add_argument('--filter-indicator', help='filter for specific indicator, useful in testing')

    p.add_argument('--fireball', help='run in fireball mode, bulk+async magic', action='store_true')
    p.add_argument('--no-fetch', help='do not re-fetch if the cache exists', action='store_true')

    p.add_argument('--no-verify-ssl', help='turn TLS/SSL verification OFF', action='store_true')

    p.add_argument('--goback', help='specify default number of days to start out at [default %(default)s]',
                   default=GOBACK_DAYS)

    p.add_argument('--fields', help='specify fields for stdout [default %(default)s]"', default=','.join(STDOUT_FIELDS))

    p.add_argument('--skip-invalid', help="skip invalid indicators in DEBUG (-d) mode", action="store_true")
    p.add_argument('--skip-broken', help='skip seemingly broken feeds', action='store_true')

    p.add_argument('--send-retries', help='specify how many times to re-try sending indicators after a failure '
                                          '[default: %(default)s', default=5)
    p.add_argument('--send-retries-wait', help='how many seconds to wait between retries [default: %(default)s',
                   default=30)

    args = p.parse_args()

    o = read_config(args)
    options = vars(args)
    for v in options:
        if options[v] is None:
            options[v] = o.get(v)

    setup_logging(args)
    logger.info('loglevel is: {}'.format(logging.getLevelName(logger.getEffectiveLevel())))

    setup_runtime_path(args.runtime_path)

    verify_ssl = True
    if options.get('no_verify_ssl') or o.get('no_verify_ssl'):
        verify_ssl = False

    goback = args.goback
    if goback:
        goback = arrow.utcnow().replace(days=-int(goback))

    if not args.service:
        data = None
        if select.select([sys.stdin, ], [], [], 0.0)[0]:
            data = sys.stdin.read()

        try:
            _run_smrt(options, **{
                'args': args,
                'data': data,
                'verify_ssl': verify_ssl,
                'goback': goback
            })
        except KeyboardInterrupt:
            logger.info('exiting..')

        raise SystemExit

    # we're running as a service
    setup_signals(__name__)
    service_interval = int(args.service_interval)
    r = int(args.delay)
    logger.info("random delay is {}, then running every {} min after that".format(r, service_interval))

    if r != 0:
        try:
            sleep((r * 60))

        except KeyboardInterrupt:
            logger.info('shutting down')
            raise SystemExit

        except Exception as e:
            logger.error(e)
            raise SystemExit

    logger.info('starting...')

    def _run():
        logger.debug('forking process...')
        p = Process(target=_run_smrt, args=(options,), kwargs={
            'args': args,
            'verify_ssl': verify_ssl,
            'goback': goback,
            'service_mode': True
        })

        p.daemon = False
        p.start()
        p.join()
        logger.debug('child process re-joined')

    # first run, PeriodicCallback has builtin wait..
    _run()

    main_loop = ioloop.IOLoop()
    service_interval = (service_interval * 60000)
    loop = ioloop.PeriodicCallback(_run, service_interval)

    try:
        loop.start()
        main_loop.start()

    except KeyboardInterrupt:
        logger.info('exiting..')
        pass

    except Exception as e:
        logger.error(e)
        pass
Exemplo n.º 5
0
def main():
    p = get_argument_parser()
    p = ArgumentParser(
        description=textwrap.dedent('''\
            Env Variables:
                CSIRTG_RUNTIME_PATH

            example usage:
                $ csirtg-cef -f /var/log/foo.log
                $ ZYRE_GROUP=honeynet csirtg-cef -d -f /var/log/foo.log --client zyre
                $ csirtg-cef -f /var/log/foo.log --client csirtg --user wes --feed scanners -d
            '''),
        formatter_class=RawDescriptionHelpFormatter,
        prog='csirtg-cef',
        parents=[p],
    )

    p.add_argument('--no-verify-ssl', help='turn TLS/SSL verification OFF', action='store_true')
    p.add_argument('-f', '--file')
    p.add_argument('--client', default='stdout')
    p.add_argument('--user')
    p.add_argument('--feed')
    p.add_argument('--format', default='csv')
    p.add_argument('--tags', help='specify indicator tags [default %(default)s', default='scanner')
    p.add_argument('--provider', help='specify provider [default %(default)s]', default=PROVIDER)
    p.add_argument('--aggregate', help='specify how many seconds to aggregate batches before sending to client '
                                       '[default %(default)s]', default=60)

    p.add_argument('--tail-docker')

    args = p.parse_args()

    # setup logging
    setup_logging(args)

    verify_ssl = True
    if args.no_verify_ssl:
        verify_ssl = False

    if args.file:
        logger.debug('starting on: {}'.format(args.file))
        data_source = tail(args.file)
    elif args.tail_docker:
        logger.debug('starting on container: {}'.format(args.tail_docker))
        #data_source = subprocess.Popen(["docker", "logs", "-f", "--tail", "0", args.tail_docker], bufsize=1, stdout=subprocess.PIPE).stdout
        client = docker.from_env(version='auto')
        container = client.containers.get(args.tail_docker)
        data_source = container.logs(stream=True, follow=True, tail=0)
    else:
        logger.error('Missing --file or --tail-docker flag')
        raise SystemExit

    logger.info('sending data as: %s' % args.provider)

    s = Smrt(client=args.client, username=args.user, feed=args.feed, verify_ssl=verify_ssl)

    bucket = set()
    last_t = round_time(round=int(args.aggregate))
    try:
        for line in data_source:
            i = parse_line(line)

            if not i:
                logger.debug('skipping line')
                continue

            i = Indicator(**i)

            logger.debug(i)

            i.provider = args.provider
            i.tags = args.tags

            if args.aggregate:
                t = round_time(dt=datetime.now(), round=int(args.aggregate))
                if t != last_t:
                    bucket = set()
                
                last_t = t

                if i.indicator in bucket:
                    logger.info('skipping send {}'.format(i.indicator))
                    continue

                bucket.add(i.indicator)

            if args.client == 'stdout':
                print(FORMATS[args.format](data=[i]))
            else:
                try:
                    s.client.indicators_create(i)
                    logger.info('indicator created: {}'.format(i.indicator))
                except Exception as e:
                    logger.error(e)

    except Exception as e:
        logger.error(e)

    except KeyboardInterrupt:
        logger.info('SIGINT caught... stopping')
        if args.client != 'stdout':
            s.client.stop()

    logger.info('exiting...')
Exemplo n.º 6
0
def main():
    p = get_argument_parser()
    p = ArgumentParser(
        description=textwrap.dedent('''\
        Env Variables:
            CSIRTG_RUNTIME_PATH
            CSIRTG_TOKEN

        example usage:
            $ csirtg-smrt --rule rules/default
            $ csirtg-smrt --rule default/csirtg.yml --feed port-scanners --remote http://localhost:5000
        '''),
        formatter_class=RawDescriptionHelpFormatter,
        prog='cif-smrt',
        parents=[p],
    )

    p.add_argument("-r", "--rule", help="specify the rules directory or specific rules file [default: %(default)s",
                   default=SMRT_RULES_PATH)

    p.add_argument("-f", "--feed", help="specify the feed to process")

    p.add_argument("--remote", dest="remote", help="specify the remote api url [default: %(default)s",
                   default=REMOTE_ADDR)

    p.add_argument('--cache', help="specify feed cache [default %(default)s]", default=SMRT_CACHE)

    p.add_argument("--limit", dest="limit", help="limit the number of records processed [default: %(default)s]",
                   default=None)

    p.add_argument("--token", dest="token", help="specify token [default: %(default)s]", default=TOKEN)

    p.add_argument('--test', action='store_true')
    p.add_argument('--sleep', default=60)
    p.add_argument('--ignore-unknown', action='store_true')

    p.add_argument('--config', help='specify csirtg-smrt config path [default %(default)s', default=CONFIG_PATH)

    p.add_argument('--client', default='cif')

    p.add_argument('--user')

    args = p.parse_args()

    o = read_config(args)
    options = vars(args)
    for v in options:
        if options[v] is None:
            options[v] = o.get(v)

    setup_logging(args)
    logger = logging.getLogger(__name__)
    logger.info('loglevel is: {}'.format(logging.getLevelName(logger.getEffectiveLevel())))

    setup_signals(__name__)

    setup_runtime_path(args.runtime_path)



    stop = False

    r = False
    if not args.test:
        r = randint(5, 55)
        logger.info("random delay is {}, then running every 60min after that".format(r))
        sleep((r * 60))

    while not stop:
        if args.test:
            stop = True

        logger.info('starting...')
        try:
            with Smrt(options.get('remote'), options.get('token'), client=args.client, user=args.user,
                      feed=args.feed) as s:
                logger.info('staring up...')
                logger.info('testing router connection...')
                s.ping_router()

                x = s.process(args.rule, feed=args.feed, limit=args.limit)
                logger.info('complete')

                if not args.test:
                    logger.info('sleeping for 1 hour')
                    sleep((60 * 60))
        except AuthError as e:
            logger.error(e)
            stop = True
        except RuntimeError as e:
            logger.error(e)
            if str(e).startswith('submission failed'):
                stop = True
            else:
                logging.exception('Got exception on main handler')
        except TimeoutError as e:
            logger.error(e)
            stop = True
        except KeyboardInterrupt:
            logger.info('shutting down')
            stop = True

        logger.info('completed')