def consumer(args): # connect to kafka brokers = args.kafka_brokers.split(",") kafka_consumer = kafka.KafkaConsumer(args.topic, bootstrap_servers=brokers) # if debug not set, write libpcap global header if args.debug == 0: sys.stdout.write(global_header()) # start packet capture packet_count = 0 for msg in kafka_consumer: # if debug not set, write the packet header and packet if args.debug == 0: sys.stdout.write(packet_header(msg.value, msg.key)) sys.stdout.write(msg.value) elif packet_count % args.debug == 0: print 'Packet: count=%s dt=%s topic=%s' % ( packet_count, to_date(unpack_ts(msg.key)), args.topic) print to_hex(msg.value) packet_count += 1 if args.packet_count > 0 and packet_count >= args.packet_count: break
def consumer(args): # connect to kafka brokers = args.kafka_brokers.split(",") kafka_consumer = kafka.KafkaConsumer(args.topic, bootstrap_servers=brokers) # if debug not set, write libpcap global header if args.debug == 0: sys.stdout.write(global_header()) # start packet capture packet_count = 0 for msg in kafka_consumer: # if debug not set, write the packet header and packet if args.debug == 0: sys.stdout.write(packet_header(msg.value, msg.key)) sys.stdout.write(msg.value) elif packet_count % args.debug == 0: print "Packet: count=%s dt=%s topic=%s" % (packet_count, to_date(unpack_ts(msg.key)), args.topic) print to_hex(msg.value) packet_count += 1 if args.packet_count > 0 and packet_count >= args.packet_count: break
def consumer(args, poll_timeout=3.0): """ Consumes packets from a Kafka topic. """ # setup the signal handler signal.signal(signal.SIGINT, signal_handler) # connect to kafka logging.debug("Connecting to Kafka; %s", args.kafka_configs) kafka_consumer = Consumer(args.kafka_configs) kafka_consumer.subscribe([args.kafka_topic]) # if 'pretty-print' not set, write libpcap global header if args.pretty_print == 0: sys.stdout.write(global_header(args)) sys.stdout.flush() try: pkts_in = 0 while not finished.is_set() and (args.max_packets <= 0 or pkts_in < args.max_packets): # consume a message from kafka msg = kafka_consumer.poll(timeout=poll_timeout) if msg is None: # no message received continue; elif msg.error(): if msg.error().code() == KafkaError._PARTITION_EOF: logging.debug("reached end of topar: topic=%s, partition=%d, offset=%s", msg.topic(), msg.partition(), msg.offset()) elif msg.error(): raise KafkaException(msg.error()) else: pkts_in += 1 logging.debug("Packet received: pkts_in=%d", pkts_in) if args.pretty_print == 0: # write the packet header and packet sys.stdout.write(packet_header(msg)) sys.stdout.write(msg.value()) sys.stdout.flush() elif pkts_in % args.pretty_print == 0: # pretty print print 'Packet: count=%s date=%s topic=%s' % ( pkts_in, to_date(unpack_ts(msg.key())), args.kafka_topic) print to_hex(msg.value()) finally: sys.stdout.close() kafka_consumer.close()
def delivery_callback(err, msg): """ Callback executed when message delivery either succeeds or fails. """ # initialize counter, if needed if not hasattr(delivery_callback, "pkts_out"): delivery_callback.pkts_out = 0 if err: logging.error("message delivery failed: error=%s", err) elif msg is not None: delivery_callback.pkts_out += 1 pretty_print = 0 pretty_print = producer_args.pretty_print if pretty_print > 0 and delivery_callback.pkts_out % pretty_print == 0: print 'Packet delivered[%s]: date=%s topic=%s partition=%s offset=%s len=%s' % ( delivery_callback.pkts_out, to_date(unpack_ts(msg.key())), msg.topic(), msg.partition(), msg.offset(), len(msg.value()))
def consumer(args, poll_timeout=3.0): """ Consumes packets from a Kafka topic. """ # setup the signal handler signal.signal(signal.SIGINT, signal_handler) # where to start consuming messages from kafka_offset_options = { "begin": seek_to_begin, "end": seek_to_end, "stored": seek_to_stored } on_assign_cb = kafka_offset_options[args.kafka_offset] # connect to kafka logging.debug("Connecting to Kafka; %s", args.kafka_configs) kafka_consumer = Consumer(args.kafka_configs) kafka_consumer.subscribe([args.kafka_topic], on_assign=on_assign_cb) # if 'pretty-print' not set, write libpcap global header if args.pretty_print == 0: sys.stdout.write(global_header(args)) sys.stdout.flush() try: pkts_in = 0 while not finished.is_set() and (args.max_packets <= 0 or pkts_in < args.max_packets): # consume a message from kafka msg = kafka_consumer.poll(timeout=poll_timeout) if msg is None: # no message received continue; elif msg.error(): if msg.error().code() == KafkaError._PARTITION_EOF: if args.pretty_print > 0: print "Reached end of topar: topic=%s, partition=%d, offset=%s" % ( msg.topic(), msg.partition(), msg.offset()) else: raise KafkaException(msg.error()) else: pkts_in += 1 logging.debug("Packet received: pkts_in=%d", pkts_in) if args.pretty_print == 0: # write the packet header and packet sys.stdout.write(packet_header(msg)) sys.stdout.write(msg.value()) sys.stdout.flush() elif pkts_in % args.pretty_print == 0: # pretty print print 'Packet[%s]: date=%s topic=%s partition=%s offset=%s len=%s' % ( pkts_in, to_date(unpack_ts(msg.key())), args.kafka_topic, msg.partition(), msg.offset(), len(msg.value())) finally: sys.stdout.close() kafka_consumer.close()
def consumer(args, poll_timeout=3.0): """ Consumes packets from a Kafka topic. """ # setup the signal handler signal.signal(signal.SIGINT, signal_handler) # where to start consuming messages from kafka_offset_options = { "begin": seek_to_begin, "end": seek_to_end, "stored": seek_to_stored } on_assign_cb = kafka_offset_options[args.kafka_offset] # connect to kafka logging.debug("Connecting to Kafka; %s", args.kafka_configs) kafka_consumer = Consumer(args.kafka_configs) kafka_consumer.subscribe([args.kafka_topic], on_assign=on_assign_cb) # if 'pretty-print' not set, write libpcap global header if args.pretty_print == 0: sys.stdout.write(global_header(args)) sys.stdout.flush() try: pkts_in = 0 while not finished.is_set() and (args.max_packets <= 0 or pkts_in < args.max_packets): # consume a message from kafka msg = kafka_consumer.poll(timeout=poll_timeout) if msg is None: # no message received continue; elif msg.error(): if msg.error().code() == KafkaError._PARTITION_EOF: if args.pretty_print > 0: print "Reached end of topar: topic=%s, partition=%d, offset=%s" % ( msg.topic(), msg.partition(), msg.offset()) else: raise KafkaException(msg.error()) else: pkts_in += 1 logging.debug("Packet received: pkts_in=%d", pkts_in) if args.pretty_print == 0: # write the packet header and packet # AT: We are just sending over the results of the scan -- a list of macs/rssi's -- where this code # was dealing with network packet sniffers -- sys.stdout.write(json.dumps(msg.value(), indent=2)) # sys.stdout.write(packet_header(msg)) # sys.stdout.write(msg.value()) sys.stdout.flush() elif pkts_in % args.pretty_print == 0: # pretty print print 'Packet[%s]: date=%s topic=%s partition=%s offset=%s len=%s' % ( pkts_in, to_date(unpack_ts(msg.key())), args.kafka_topic, msg.partition(), msg.offset(), len(msg.value())) finally: sys.stdout.close() kafka_consumer.close()