def main():
    from contextlib import closing
    import os, errno, logging, metrics, shaper

    import argparse
    parser = argparse.ArgumentParser(
        description='Compress or drop traffiic between two'
        ' zmq sockets if its volume is above defined thresholds.')
    parser.add_argument('src', help='Receiving ZMQ socket address to bind to.')
    parser.add_argument('dst', help='ZMQ socket address to relay data to.')
    parser.add_argument(
        '--zmq-buffer',
        type=int,
        metavar='msg_count',
        default=50,
        help='ZMQ_SNDHWM for the sending socket - number of (possibly'
        ' compressed) packets to buffer in RAM before blocking (default: %(default)s).'
    )
    parser.add_argument('--debug',
                        action='store_true',
                        help='Verbose operation mode.')
    shaper.add_compress_optz(parser, always_enabled=True)
    metrics.add_statsd_optz(parser)
    optz = parser.parse_args()

    logging.basicConfig(
        level=logging.DEBUG if optz.debug else logging.WARNING,
        logfmt='%(asctime)s :: %(levelname)s :: %(name)s: %(message)s',
        datefmt='%Y-%m-%d %H:%M:%S')
    log = logging.getLogger('zmq_compress')

    statsd = metrics.statsd_from_optz(optz)
    compressor = shaper\
     .compress_pipe_from_optz(optz, always_enabled=True)

    import zmq
    context = zmq.Context()

    try:
        with closing(context.socket(zmq.PULL)) as src,\
          closing(context.socket(zmq.PUSH)) as dst:
            src.bind(optz.src)
            dst.setsockopt(zmq.SNDHWM, optz.zmq_buffer)
            dst.setsockopt(zmq.LINGER, 0)  # it's lossy either way
            dst.connect(optz.dst)

            log.debug('Starting pipeline loop')
            while True:
                buff = src.recv()
                while src.getsockopt(zmq.RCVMORE):
                    buff += src.recv()

                if statsd:
                    statsd.send('compress_in.pkt')
                    statsd.send(('compress_in.bytes', len(buff)))

                pkt = compressor.send(buff)
                if pkt is None: continue

                try:
                    dst.send(pkt, zmq.NOBLOCK)
                except zmq.ZMQError as err:
                    if err.errno != errno.EAGAIN: raise
                    continue  # so zmq-dropped packets won't be counted in statsd

                if statsd:
                    statsd.send('compress_out.pkt')
                    statsd.send(('compress_out.bytes', len(pkt)))

    finally:
        log.debug('Finishing')
        context.term()
def main():
	from contextlib import closing
	import os, errno, logging, metrics, shaper

	import argparse
	parser = argparse.ArgumentParser(
		description='Compress or drop traffiic between two'
			' zmq sockets if its volume is above defined thresholds.')
	parser.add_argument('src', help='Receiving ZMQ socket address to bind to.')
	parser.add_argument('dst', help='ZMQ socket address to relay data to.')
	parser.add_argument('--zmq-buffer',
		type=int, metavar='msg_count', default=50,
		help='ZMQ_SNDHWM for the sending socket - number of (possibly'
			' compressed) packets to buffer in RAM before blocking (default: %(default)s).')
	parser.add_argument('--debug', action='store_true', help='Verbose operation mode.')
	shaper.add_compress_optz(parser, always_enabled=True)
	metrics.add_statsd_optz(parser)
	optz = parser.parse_args()

	logging.basicConfig(
		level=logging.DEBUG if optz.debug else logging.WARNING,
		logfmt='%(asctime)s :: %(levelname)s :: %(name)s: %(message)s',
		datefmt='%Y-%m-%d %H:%M:%S' )
	log = logging.getLogger('zmq_compress')

	statsd = metrics.statsd_from_optz(optz)
	compressor = shaper\
		.compress_pipe_from_optz(optz, always_enabled=True)

	import zmq
	context = zmq.Context()

	try:
		with closing(context.socket(zmq.PULL)) as src,\
				closing(context.socket(zmq.PUSH)) as dst:
			src.bind(optz.src)
			dst.setsockopt(zmq.SNDHWM, optz.zmq_buffer)
			dst.setsockopt(zmq.LINGER, 0) # it's lossy either way
			dst.connect(optz.dst)

			log.debug('Starting pipeline loop')
			while True:
				buff = src.recv()
				while src.getsockopt(zmq.RCVMORE): buff += src.recv()

				if statsd:
					statsd.send('compress_in.pkt')
					statsd.send(('compress_in.bytes', len(buff)))

				pkt = compressor.send(buff)
				if pkt is None: continue

				try: dst.send(pkt, zmq.NOBLOCK)
				except zmq.ZMQError as err:
					if err.errno != errno.EAGAIN: raise
					continue # so zmq-dropped packets won't be counted in statsd

				if statsd:
					statsd.send('compress_out.pkt')
					statsd.send(('compress_out.bytes', len(pkt)))

	finally:
		log.debug('Finishing')
		context.term()
예제 #3
0
def main():
	import itertools as it, operator as op, functools as ft
	from time import sleep
	from collections import deque
	import os, logging, pcap, metrics, shaper

	import argparse
	parser = argparse.ArgumentParser(
		description='Receive pcap stream from zeromq and push it to a fifo socket.')
	parser.add_argument('src', help='ZMQ socket address to bind to.')
	parser.add_argument('dst', help='Path to fifo to write stream to.')
	parser.add_argument('--rate-control',
		action='store_true', help='Handle possibly-compressed stream.')

	parser.add_argument('--buffer-interface',
		help='ZMQ socket to access traffic-buffer interface.'
			' Upon receiving request on it, pcap dump of'
				' a traffic will be generated and sent as a response.')
	parser.add_argument('--buffer-window',
		type=float, metavar='MiB', help='Amount of last traffic to keep buffered.')
	parser.add_argument('--buffer-timeout', default=1.0,
		type=float, metavar='s', help='Timeout for sending buffered stuff (default: %(default)s).')

	parser.add_argument('--debug', action='store_true', help='Verbose operation mode.')
	metrics.add_statsd_optz(parser)
	optz = parser.parse_args()

	logging.basicConfig(
		level=logging.DEBUG if optz.debug else logging.WARNING,
		logfmt='%(asctime)s :: %(levelname)s :: %(name)s: %(message)s',
		datefmt='%Y-%m-%d %H:%M:%S' )
	log = logging.getLogger('pcap_recv')

	optz.reopen = True # only fifo dst for now
	if optz.buffer_window:
		optz.buffer_window = optz.buffer_window * 2**20

	statsd = metrics.statsd_from_optz(optz)

	if optz.rate_control:
		shaper = shaper.decompress_pipe()
		next(shaper)
	else: shaper = None

	import zmq
	context = zmq.Context()

	src = bif = None
	try:
		src = context.socket(zmq.PULL)
		src.bind(optz.src)
		zmq_poll = zmq.Poller()
		zmq_poll.register(src, zmq.POLLIN)

		if optz.buffer_interface and optz.buffer_window:
			def bif_init():
				bif = context.socket(zmq.REP)
				bif.setsockopt(zmq.LINGER, 0)
				bif.bind(optz.buffer_interface)
				for k in zmq.RCVTIMEO, zmq.SNDTIMEO:
					bif.setsockopt(k, int(optz.buffer_timeout * 1e3))
				return bif
			bif_buff, bif_buff_len, bif = deque(), 0, bif_init()
			zmq_poll.register(bif, zmq.POLLIN)

		buff = None

		while True:
			with open(optz.dst, 'wb', 0) as dst:
				pcap_dst = pcap.writer(dst.write)
				next(pcap_dst)
				log.debug('(Re-)opened destination path')

				while True:
					if not buff:
						if bif in it.imap(op.itemgetter(0), zmq_poll.poll()):
							# Break from main activity to send traffic dump
							pcap_bif = pcap.writer(ft.partial(bif.send, flags=zmq.SNDMORE))
							try:
								bif.recv() # contents aren't used
								next(pcap_bif)
								for pkt_len, pkt in bif_buff: pcap_bif.send(pkt)
								bif.send('')
							except zmq.ZMQError as err:
								if err.errno != zmq.EAGAIN: raise
								zmq_poll.unregister(bif)
								bif.close()
								bif = bif_init()
								zmq_poll.register(bif, zmq.POLLIN)
							finally: del pcap_bif
							continue

						buff = ''.join(src.recv_multipart())

						if statsd:
							statsd.send('raw_in.pkt')
							statsd.send('raw_in.bytes', len(buff))
						buff = shaper.send(buff) if shaper else [buff]

					try:
						buff_len = 0
						if bif: bif_tmp = list()
						for pkt in buff:
							pkt_len = pcap_dst.send(pkt)
							buff_len += pkt_len
							if bif: bif_tmp.append((pkt_len, pkt))
					except IOError: break

					if bif:
						bif_buff.extend(bif_tmp)
						del bif_tmp
						bif_buff_len += buff_len
						while bif_buff and bif_buff_len >= optz.buffer_window:
							pkt_len, pkt = bif_buff.popleft()
							bif_buff_len -= pkt_len

					if statsd:
						statsd.send('raw_out.pkt', len(buff))
						statsd.send('raw_out.bytes', buff_len)
					buff = None

			if not optz.reopen: break
			sleep(1)

	finally:
		if src: src.close()
		if bif: bif.close()
		log.debug('Finishing')
		context.term()
예제 #4
0
def main():
	from contextlib import closing
	import os, logging, nflog, pcap, metrics, shaper

	import argparse
	parser = argparse.ArgumentParser(description='Pipe nflog packet stream to zeromq.')
	parser.add_argument('src', help='Comma-separated list of nflog groups to receive.')
	parser.add_argument('dst', help='ZMQ socket address to send data to.')
	parser.add_argument('-u', '--user', help='User name to drop privileges to.')

	parser.add_argument('--libnflog-nlbufsiz',
		type=float, metavar='MiB', default=10.0,
		help='Netlink socket buffer size ("nlbufsiz", default: %(default)s).')
	parser.add_argument('--libnflog-qthresh',
		type=int, metavar='packets',
		help='NFLOG queue kernel-to-userspace'
			' packet-count flush threshold ("qthresh", default: nlbufsiz * 100).')
	parser.add_argument('--libnflog-timeout',
		type=float, metavar='seconds',
		help='NFLOG queue kernel-to-userspace'
			' flush timeout ("timeout", default: nlbufsiz / 5).')
	parser.add_argument('--zmq-buffer',
		type=int, metavar='msg_count',
		help='ZMQ_HWM for the socket - number of packets to'
			' buffer in RAM before blocking (default: qthresh / 10).')

	parser.add_argument('--debug', action='store_true', help='Verbose operation mode.')

	shaper.add_compress_optz(parser)
	metrics.add_statsd_optz(parser)
	optz = parser.parse_args()

	logging.basicConfig(
		level=logging.DEBUG if optz.debug else logging.WARNING,
		logfmt='%(asctime)s :: %(levelname)s :: %(name)s: %(message)s',
		datefmt='%Y-%m-%d %H:%M:%S' )
	log = logging.getLogger('pcap_send')

	if optz.libnflog_qthresh is None:
		optz.libnflog_qthresh = int(optz.libnflog_nlbufsiz * 100)
	if optz.libnflog_timeout is None:
		optz.libnflog_timeout = int(optz.libnflog_nlbufsiz / 5.0)
	if optz.zmq_buffer is None:
		optz.zmq_buffer = int(optz.libnflog_qthresh / 10.0)

	src = nflog.nflog_generator(
		map(int, optz.src.split(',')),
		qthresh=max(1, optz.libnflog_qthresh),
		timeout=optz.libnflog_timeout,
		nlbufsiz=int(optz.libnflog_nlbufsiz * 2**20),
		extra_attrs=['len', 'ts'] )
	next(src) # no use for polling here

	if optz.user:
		import pwd
		optz.user = pwd.getpwnam(optz.user)
		os.setresgid(*[optz.user.pw_uid]*3)
		os.setresuid(*[optz.user.pw_gid]*3)

	statsd = metrics.statsd_from_optz(optz)
	shaper = shaper.compress_pipe_from_optz(optz)

	import zmq
	context = zmq.Context()

	try:
		with closing(context.socket(zmq.PUSH)) as dst:
			dst.setsockopt(zmq.HWM, optz.zmq_buffer)
			dst.setsockopt(zmq.LINGER, 0) # it's lossy either way
			dst.connect(optz.dst)

			log.debug('Entering NFLOG reader loop')
			for pkt, pkt_len, ts in src:
				if pkt is None: continue
				if statsd:
					statsd.send('raw_in.pkt')
					statsd.send(('raw_in.bytes', len(pkt)))

				pkt = pcap.construct(pkt, pkt_len=pkt_len, ts=ts)
				if shaper:
					pkt = shaper.send(pkt)
					if pkt is None: continue

				try: dst.send(pkt, zmq.NOBLOCK)
				except zmq.ZMQError as err:
					if err.errno != zmq.EAGAIN: raise

	finally:
		log.debug('Finishing')
		context.term()
def main():
    from contextlib import closing
    import os, logging, nflog, pcap, metrics, shaper

    import argparse
    parser = argparse.ArgumentParser(
        description='Pipe nflog packet stream to zeromq.')
    parser.add_argument(
        'src', help='Comma-separated list of nflog groups to receive.')
    parser.add_argument('dst', help='ZMQ socket address to send data to.')
    parser.add_argument('-u',
                        '--user',
                        help='User name to drop privileges to.')

    parser.add_argument(
        '--libnflog-nlbufsiz',
        type=float,
        metavar='MiB',
        default=10.0,
        help='Netlink socket buffer size ("nlbufsiz", default: %(default)s).')
    parser.add_argument(
        '--libnflog-qthresh',
        type=int,
        metavar='packets',
        help='NFLOG queue kernel-to-userspace'
        ' packet-count flush threshold ("qthresh", default: nlbufsiz * 100).')
    parser.add_argument('--libnflog-timeout',
                        type=float,
                        metavar='seconds',
                        help='NFLOG queue kernel-to-userspace'
                        ' flush timeout ("timeout", default: nlbufsiz / 5).')
    parser.add_argument(
        '--zmq-buffer',
        type=int,
        metavar='msg_count',
        help='ZMQ_SNDHWM for the socket - number of packets to'
        ' buffer in RAM before blocking (default: qthresh / 10).')

    parser.add_argument('--debug',
                        action='store_true',
                        help='Verbose operation mode.')

    shaper.add_compress_optz(parser)
    metrics.add_statsd_optz(parser)
    optz = parser.parse_args()

    logging.basicConfig(
        level=logging.DEBUG if optz.debug else logging.WARNING,
        logfmt='%(asctime)s :: %(levelname)s :: %(name)s: %(message)s',
        datefmt='%Y-%m-%d %H:%M:%S')
    log = logging.getLogger('pcap_send')

    if optz.libnflog_qthresh is None:
        optz.libnflog_qthresh = int(optz.libnflog_nlbufsiz * 100)
    if optz.libnflog_timeout is None:
        optz.libnflog_timeout = int(optz.libnflog_nlbufsiz / 5.0)
    if optz.zmq_buffer is None:
        optz.zmq_buffer = int(optz.libnflog_qthresh / 10.0)

    src = nflog.nflog_generator(map(int, optz.src.split(',')),
                                qthresh=max(1, optz.libnflog_qthresh),
                                timeout=optz.libnflog_timeout,
                                nlbufsiz=int(optz.libnflog_nlbufsiz * 2**20),
                                extra_attrs=['len', 'ts'])
    next(src)  # no use for polling here

    if optz.user:
        import pwd
        optz.user = pwd.getpwnam(optz.user)
        os.setresgid(*[optz.user.pw_uid] * 3)
        os.setresuid(*[optz.user.pw_gid] * 3)

    statsd = metrics.statsd_from_optz(optz)
    shaper = shaper.compress_pipe_from_optz(optz)

    import zmq
    context = zmq.Context()

    try:
        with closing(context.socket(zmq.PUSH)) as dst:
            dst.setsockopt(zmq.SNDHWM, optz.zmq_buffer)
            dst.setsockopt(zmq.LINGER, 0)  # it's lossy either way
            dst.connect(optz.dst)

            log.debug('Entering NFLOG reader loop')
            for pkt, pkt_len, ts in src:
                if pkt is None: continue
                if statsd:
                    statsd.send('raw_in.pkt')
                    statsd.send(('raw_in.bytes', len(pkt)))

                pkt = pcap.construct(pkt, pkt_len=pkt_len, ts=ts)
                if pkt is None: continue
                if shaper:
                    pkt = shaper.send(pkt)
                    if pkt is None: continue

                try:
                    dst.send(pkt, zmq.NOBLOCK)
                except zmq.ZMQError as err:
                    if err.errno != zmq.EAGAIN: raise

    finally:
        log.debug('Finishing')
        context.term()