def main(): from contextlib import closing import os, errno, logging, metrics, shaper import argparse parser = argparse.ArgumentParser( description='Compress or drop traffiic between two' ' zmq sockets if its volume is above defined thresholds.') parser.add_argument('src', help='Receiving ZMQ socket address to bind to.') parser.add_argument('dst', help='ZMQ socket address to relay data to.') parser.add_argument( '--zmq-buffer', type=int, metavar='msg_count', default=50, help='ZMQ_SNDHWM for the sending socket - number of (possibly' ' compressed) packets to buffer in RAM before blocking (default: %(default)s).' ) parser.add_argument('--debug', action='store_true', help='Verbose operation mode.') shaper.add_compress_optz(parser, always_enabled=True) metrics.add_statsd_optz(parser) optz = parser.parse_args() logging.basicConfig( level=logging.DEBUG if optz.debug else logging.WARNING, logfmt='%(asctime)s :: %(levelname)s :: %(name)s: %(message)s', datefmt='%Y-%m-%d %H:%M:%S') log = logging.getLogger('zmq_compress') statsd = metrics.statsd_from_optz(optz) compressor = shaper\ .compress_pipe_from_optz(optz, always_enabled=True) import zmq context = zmq.Context() try: with closing(context.socket(zmq.PULL)) as src,\ closing(context.socket(zmq.PUSH)) as dst: src.bind(optz.src) dst.setsockopt(zmq.SNDHWM, optz.zmq_buffer) dst.setsockopt(zmq.LINGER, 0) # it's lossy either way dst.connect(optz.dst) log.debug('Starting pipeline loop') while True: buff = src.recv() while src.getsockopt(zmq.RCVMORE): buff += src.recv() if statsd: statsd.send('compress_in.pkt') statsd.send(('compress_in.bytes', len(buff))) pkt = compressor.send(buff) if pkt is None: continue try: dst.send(pkt, zmq.NOBLOCK) except zmq.ZMQError as err: if err.errno != errno.EAGAIN: raise continue # so zmq-dropped packets won't be counted in statsd if statsd: statsd.send('compress_out.pkt') statsd.send(('compress_out.bytes', len(pkt))) finally: log.debug('Finishing') context.term()
def main(): from contextlib import closing import os, logging, nflog, pcap, metrics, shaper import argparse parser = argparse.ArgumentParser(description='Pipe nflog packet stream to zeromq.') parser.add_argument('src', help='Comma-separated list of nflog groups to receive.') parser.add_argument('dst', help='ZMQ socket address to send data to.') parser.add_argument('-u', '--user', help='User name to drop privileges to.') parser.add_argument('--libnflog-nlbufsiz', type=float, metavar='MiB', default=10.0, help='Netlink socket buffer size ("nlbufsiz", default: %(default)s).') parser.add_argument('--libnflog-qthresh', type=int, metavar='packets', help='NFLOG queue kernel-to-userspace' ' packet-count flush threshold ("qthresh", default: nlbufsiz * 100).') parser.add_argument('--libnflog-timeout', type=float, metavar='seconds', help='NFLOG queue kernel-to-userspace' ' flush timeout ("timeout", default: nlbufsiz / 5).') parser.add_argument('--zmq-buffer', type=int, metavar='msg_count', help='ZMQ_HWM for the socket - number of packets to' ' buffer in RAM before blocking (default: qthresh / 10).') parser.add_argument('--debug', action='store_true', help='Verbose operation mode.') shaper.add_compress_optz(parser) metrics.add_statsd_optz(parser) optz = parser.parse_args() logging.basicConfig( level=logging.DEBUG if optz.debug else logging.WARNING, logfmt='%(asctime)s :: %(levelname)s :: %(name)s: %(message)s', datefmt='%Y-%m-%d %H:%M:%S' ) log = logging.getLogger('pcap_send') if optz.libnflog_qthresh is None: optz.libnflog_qthresh = int(optz.libnflog_nlbufsiz * 100) if optz.libnflog_timeout is None: optz.libnflog_timeout = int(optz.libnflog_nlbufsiz / 5.0) if optz.zmq_buffer is None: optz.zmq_buffer = int(optz.libnflog_qthresh / 10.0) src = nflog.nflog_generator( map(int, optz.src.split(',')), qthresh=max(1, optz.libnflog_qthresh), timeout=optz.libnflog_timeout, nlbufsiz=int(optz.libnflog_nlbufsiz * 2**20), extra_attrs=['len', 'ts'] ) next(src) # no use for polling here if optz.user: import pwd optz.user = pwd.getpwnam(optz.user) os.setresgid(*[optz.user.pw_uid]*3) os.setresuid(*[optz.user.pw_gid]*3) statsd = metrics.statsd_from_optz(optz) shaper = shaper.compress_pipe_from_optz(optz) import zmq context = zmq.Context() try: with closing(context.socket(zmq.PUSH)) as dst: dst.setsockopt(zmq.HWM, optz.zmq_buffer) dst.setsockopt(zmq.LINGER, 0) # it's lossy either way dst.connect(optz.dst) log.debug('Entering NFLOG reader loop') for pkt, pkt_len, ts in src: if pkt is None: continue if statsd: statsd.send('raw_in.pkt') statsd.send(('raw_in.bytes', len(pkt))) pkt = pcap.construct(pkt, pkt_len=pkt_len, ts=ts) if shaper: pkt = shaper.send(pkt) if pkt is None: continue try: dst.send(pkt, zmq.NOBLOCK) except zmq.ZMQError as err: if err.errno != zmq.EAGAIN: raise finally: log.debug('Finishing') context.term()
def main(): from contextlib import closing import os, errno, logging, metrics, shaper import argparse parser = argparse.ArgumentParser( description='Compress or drop traffiic between two' ' zmq sockets if its volume is above defined thresholds.') parser.add_argument('src', help='Receiving ZMQ socket address to bind to.') parser.add_argument('dst', help='ZMQ socket address to relay data to.') parser.add_argument('--zmq-buffer', type=int, metavar='msg_count', default=50, help='ZMQ_SNDHWM for the sending socket - number of (possibly' ' compressed) packets to buffer in RAM before blocking (default: %(default)s).') parser.add_argument('--debug', action='store_true', help='Verbose operation mode.') shaper.add_compress_optz(parser, always_enabled=True) metrics.add_statsd_optz(parser) optz = parser.parse_args() logging.basicConfig( level=logging.DEBUG if optz.debug else logging.WARNING, logfmt='%(asctime)s :: %(levelname)s :: %(name)s: %(message)s', datefmt='%Y-%m-%d %H:%M:%S' ) log = logging.getLogger('zmq_compress') statsd = metrics.statsd_from_optz(optz) compressor = shaper\ .compress_pipe_from_optz(optz, always_enabled=True) import zmq context = zmq.Context() try: with closing(context.socket(zmq.PULL)) as src,\ closing(context.socket(zmq.PUSH)) as dst: src.bind(optz.src) dst.setsockopt(zmq.SNDHWM, optz.zmq_buffer) dst.setsockopt(zmq.LINGER, 0) # it's lossy either way dst.connect(optz.dst) log.debug('Starting pipeline loop') while True: buff = src.recv() while src.getsockopt(zmq.RCVMORE): buff += src.recv() if statsd: statsd.send('compress_in.pkt') statsd.send(('compress_in.bytes', len(buff))) pkt = compressor.send(buff) if pkt is None: continue try: dst.send(pkt, zmq.NOBLOCK) except zmq.ZMQError as err: if err.errno != errno.EAGAIN: raise continue # so zmq-dropped packets won't be counted in statsd if statsd: statsd.send('compress_out.pkt') statsd.send(('compress_out.bytes', len(pkt))) finally: log.debug('Finishing') context.term()
def main(): from contextlib import closing import os, logging, nflog, pcap, metrics, shaper import argparse parser = argparse.ArgumentParser( description='Pipe nflog packet stream to zeromq.') parser.add_argument( 'src', help='Comma-separated list of nflog groups to receive.') parser.add_argument('dst', help='ZMQ socket address to send data to.') parser.add_argument('-u', '--user', help='User name to drop privileges to.') parser.add_argument( '--libnflog-nlbufsiz', type=float, metavar='MiB', default=10.0, help='Netlink socket buffer size ("nlbufsiz", default: %(default)s).') parser.add_argument( '--libnflog-qthresh', type=int, metavar='packets', help='NFLOG queue kernel-to-userspace' ' packet-count flush threshold ("qthresh", default: nlbufsiz * 100).') parser.add_argument('--libnflog-timeout', type=float, metavar='seconds', help='NFLOG queue kernel-to-userspace' ' flush timeout ("timeout", default: nlbufsiz / 5).') parser.add_argument( '--zmq-buffer', type=int, metavar='msg_count', help='ZMQ_SNDHWM for the socket - number of packets to' ' buffer in RAM before blocking (default: qthresh / 10).') parser.add_argument('--debug', action='store_true', help='Verbose operation mode.') shaper.add_compress_optz(parser) metrics.add_statsd_optz(parser) optz = parser.parse_args() logging.basicConfig( level=logging.DEBUG if optz.debug else logging.WARNING, logfmt='%(asctime)s :: %(levelname)s :: %(name)s: %(message)s', datefmt='%Y-%m-%d %H:%M:%S') log = logging.getLogger('pcap_send') if optz.libnflog_qthresh is None: optz.libnflog_qthresh = int(optz.libnflog_nlbufsiz * 100) if optz.libnflog_timeout is None: optz.libnflog_timeout = int(optz.libnflog_nlbufsiz / 5.0) if optz.zmq_buffer is None: optz.zmq_buffer = int(optz.libnflog_qthresh / 10.0) src = nflog.nflog_generator(map(int, optz.src.split(',')), qthresh=max(1, optz.libnflog_qthresh), timeout=optz.libnflog_timeout, nlbufsiz=int(optz.libnflog_nlbufsiz * 2**20), extra_attrs=['len', 'ts']) next(src) # no use for polling here if optz.user: import pwd optz.user = pwd.getpwnam(optz.user) os.setresgid(*[optz.user.pw_uid] * 3) os.setresuid(*[optz.user.pw_gid] * 3) statsd = metrics.statsd_from_optz(optz) shaper = shaper.compress_pipe_from_optz(optz) import zmq context = zmq.Context() try: with closing(context.socket(zmq.PUSH)) as dst: dst.setsockopt(zmq.SNDHWM, optz.zmq_buffer) dst.setsockopt(zmq.LINGER, 0) # it's lossy either way dst.connect(optz.dst) log.debug('Entering NFLOG reader loop') for pkt, pkt_len, ts in src: if pkt is None: continue if statsd: statsd.send('raw_in.pkt') statsd.send(('raw_in.bytes', len(pkt))) pkt = pcap.construct(pkt, pkt_len=pkt_len, ts=ts) if pkt is None: continue if shaper: pkt = shaper.send(pkt) if pkt is None: continue try: dst.send(pkt, zmq.NOBLOCK) except zmq.ZMQError as err: if err.errno != zmq.EAGAIN: raise finally: log.debug('Finishing') context.term()