def test_version(): print('Using confluent_kafka module version %s (0x%x)' % confluent_kafka.version()) sver, iver = confluent_kafka.version() assert len(sver) > 0 assert iver > 0 print('Using librdkafka version %s (0x%x)' % confluent_kafka.libversion()) sver, iver = confluent_kafka.libversion() assert len(sver) > 0 assert iver > 0
def test_version(): print('Using confluent_kafka module version %s (0x%x)' % confluent_kafka.version()) sver, iver = confluent_kafka.version() assert len(sver) > 0 assert iver > 0 print('Using librdkafka version %s (0x%x)' % confluent_kafka.libversion()) sver, iver = confluent_kafka.libversion() assert len(sver) > 0 assert iver > 0
def write_to_kafka(bootstrap_servers, schema_registry_url, topic_name, data): print("Kafka Version : ", confluent_kafka.version(),confluent_kafka.libversion()) schema_registry_conf = {'url': schema_registry_url} schema_registry_client = SchemaRegistryClient(schema_registry_conf) value_avro_serializer = AvroSerializer(schemas.weather_source_schema, schema_registry_client) string_serializer = StringSerializer('utf-8') conf = {'bootstrap.servers': bootstrap_servers, 'client.id': socket.gethostname(), 'on_delivery': delivery_report, 'key.serializer': string_serializer, 'value.serializer': value_avro_serializer } avroProducer = SerializingProducer(conf) key=datetime.date.today() + '~' + str(data['lat']) + '~' + str(data['lon']) message = json.dumps(data, cls=DatetimeEncoder) print("Key Type : ", type(key)) print("Value Type : ", type(json.loads(message))) avroProducer.produce(topic=topic_name, key=key, value=json.loads(message)) avroProducer.flush()
def verify_producer(): """ Verify basic Producer functionality """ # Producer config conf = { 'bootstrap.servers': bootstrap_servers, 'error_cb': error_cb, 'api.version.request': api_version_request } # Create producer p = confluent_kafka.Producer(**conf) print('producer at %s' % p) headers = produce_headers # Produce some messages p.produce(topic, 'Hello Python!', headers=headers) p.produce(topic, key='Just a key and headers', headers=headers) p.produce(topic, key='Just a key') p.produce(topic, partition=1, value='Strictly for partition 1', key='mykey', headers=headers) # Produce more messages, now with delivery report callbacks in various forms. mydr = MyTestDr() p.produce(topic, value='This one has a dr callback', callback=mydr.delivery) p.produce(topic, value='This one has a lambda', callback=lambda err, msg: MyTestDr._delivery(err, msg)) p.produce(topic, value='This one has neither') # Try producing with a timestamp try: p.produce(topic, value='with a timestamp', timestamp=123456789000) except NotImplementedError: if confluent_kafka.libversion()[1] >= 0x00090400: raise # Produce even more messages for i in range(0, 10): p.produce(topic, value='Message #%d' % i, key=str(i), callback=mydr.delivery) p.poll(0) print('Waiting for %d messages to be delivered' % len(p)) # Block until all messages are delivered/failed p.flush() # # Additional isolated tests # test_producer_dr_only_error()
def print_packages_versions(): """Print versions for the passed packages. """ packages = get_packages() for pkg in packages: if pkg == "librdkafka": print("%s==%s" % (pkg, confluent_kafka.libversion()[0])) else: print("%s==%s" % (pkg, pkg_resources.get_distribution(pkg).version))
def main(): print("Confluent Kafka Version: %s - Libversion: %s" % (version(), libversion())) print("") print("Connecting to RTSP: %s" % rtsp_url) print("") print("Producing video frames to: %s" % topic_frames) print("") print( "Skipping every %s frames and then saving up to %s (-1 is unlimited) frames" % (skipframes, capmax)) print("") p = Producer({'bootstrap.servers': '', 'message.max.bytes': '2978246'}) output_loc = "./test" capcount = 0 cap = cv2.VideoCapture(rtsp_url) while capcount < capmax or capmax == -1: ret, frame = cap.read() if capcount % skipframes == 0: curtime = datetime.datetime.now() mystrtime = curtime.strftime("%Y-%m-%d %H:%M:%S") epochtime = int(time.time()) print("Saving to Stream at %s" % capcount) img_str = cv2.imencode('.jpg', frame)[1].tostring() encdata = base64.b64encode(img_str) encdatastr = encdata.decode('utf-8') myrec = OrderedDict() myrec['ts'] = mystrtime myrec['epoch_ts'] = epochtime myrec['cam_name'] = cam_name myrec['img'] = encdatastr produceMessage(p, topic_frames, json.dumps(myrec)) # This is more verbose method of writing so we can get raw bytes to save to a stream # w = open(output_loc + "/cap_test_%s.jpg" % capcount, 'wb') # w.write(img_str) # w.close() # This is the prefered method for writing to a file from OpenCV, I am using theimencode so I can get the byte to write directly to MapR Streams # cv2.imwrite(output_loc + "/cap_%s.jpg" % (capcount), frame) capcount += 1 if capcount >= 10000000: if capcount % skipframe == 0: capcount = 0
def verify_producer(): """ Verify basic Producer functionality """ # Producer config conf = {'bootstrap.servers': bootstrap_servers, 'error_cb': error_cb} # Create producer p = confluent_kafka.Producer(conf) print('producer at %s' % p) headers = produce_headers # Produce some messages p.produce(topic, 'Hello Python!', headers=headers) p.produce(topic, key='Just a key and headers', headers=headers) p.produce(topic, key='Just a key') p.produce(topic, partition=1, value='Strictly for partition 1', key='mykey', headers=headers) # Produce more messages, now with delivery report callbacks in various forms. mydr = MyTestDr() p.produce(topic, value='This one has a dr callback', callback=mydr.delivery) p.produce(topic, value='This one has a lambda', callback=lambda err, msg: MyTestDr._delivery(err, msg)) p.produce(topic, value='This one has neither') # Try producing with a timestamp try: p.produce(topic, value='with a timestamp', timestamp=123456789000) except NotImplementedError: if confluent_kafka.libversion()[1] >= 0x00090400: raise # Produce even more messages for i in range(0, 10): p.produce(topic, value='Message #%d' % i, key=str(i), callback=mydr.delivery) p.poll(0) print('Waiting for %d messages to be delivered' % len(p)) # Block until all messages are delivered/failed p.flush() # # Additional isolated tests # test_producer_dr_only_error()
def test_produce_timestamp(): """ Test produce() with timestamp arg """ p = Producer({'socket.timeout.ms': 10, 'error_cb': error_cb, 'message.timeout.ms': 10}) # Requires librdkafka >=v0.9.4 try: p.produce('mytopic', timestamp=1234567) except NotImplementedError: # Should only fail on non-supporting librdkafka if libversion()[1] >= 0x00090400: raise p.flush()
def test_produce_timestamp(): """ Test produce() with timestamp arg """ p = Producer({ 'socket.timeout.ms': 10, 'error_cb': error_cb, 'message.timeout.ms': 10 }) # Requires librdkafka >=v0.9.4 try: p.produce('mytopic', timestamp=1234567) except NotImplementedError: # Should only fail on non-supporting librdkafka if libversion()[1] >= 0x00090400: raise p.flush()
def main(): print("Confluent Kafka Version: %s - Libversion: %s" % (version(), libversion())) print("") conf = { 'group.id': 'img2file1', 'default.topic.config': { 'auto.offset.reset': 'earliest' } } c = Consumer(conf) c.subscribe([topic_frames]) running = True while running: msg = c.poll(timeout=1.0) if msg is None: continue if not msg.error(): mymsg = json.loads(msg.value().decode('utf-8'), object_pairs_hook=OrderedDict) outmsg = OrderedDict() outmsg['ts'] = mymsg['ts'] outmsg['epoch_ts'] = mymsg['epoch_ts'] outmsg['cam_name'] = mymsg['cam_name'] fileout = "%s/stream_%s_%s.jpg" % (output_loc, outmsg['cam_name'], outmsg['ts'].replace( " ", "_").replace(":", "_")) mybytes = base64.b64decode(mymsg['img']) print("Processing message: %s and saving to %s" % (outmsg, fileout)) w = open(fileout, 'wb') w.write(mybytes) w.close() elif msg.error().code() != KafkaError._PARTITION_EOF: print(msg.error()) running = False c.close()
def _to_confluent_kafka(self) -> Dict: config = { "bootstrap.servers": ",".join(self.broker_urls), "error_cb": self.error_callback, "group.id": self.group_id, "enable.auto.commit": True, "auto.commit.interval.ms": int(self.offset_commit_interval.total_seconds() * 1000), "enable.auto.offset.store": False, "queued.min.messages": 1000, "enable.partition.eof": not self.read_forever, } if self.start_at is ConsumerStartPosition.EARLIEST: default_topic_config = config.get("default.topic.config", {}) default_topic_config = { "auto.offset.reset": "EARLIEST", } config["default.topic.config"] = default_topic_config elif self.start_at is ConsumerStartPosition.LATEST: # FIXME: librdkafka has a bug in offset handling - it caches # "OFFSET_END", and will repeatedly move to the end of the # topic. See https://github.com/edenhill/librdkafka/pull/2876 - # it should get fixed in v1.5 of librdkafka. librdkafka_version = confluent_kafka.libversion()[0] if librdkafka_version < "1.5.0": self.logger.warn( "In librdkafka before v1.5, LATEST offsets have buggy behavior; you may " f"not receive data (your librdkafka version is {librdkafka_version}). See " "https://github.com/confluentinc/confluent-kafka-dotnet/issues/1254.") default_topic_config = config.get("default.topic.config", {}) default_topic_config = { "auto.offset.reset": "LATEST", } config["default.topic.config"] = default_topic_config if self.auth is not None: config.update(self.auth()) return config
def main(): print("Confluent Kafka Version: %s - Libversion: %s" % (version(), libversion())) topic = os.environ["MAPR_STREAMS_STREAM_LOCATION"].replace( '"', '') + ":" + os.environ["MAPR_STREAMS_TOPIC"].replace('"', '') print("Producing messages to: %s" % topic) p = Producer({'bootstrap.servers': ''}) # Listen for messages running = True lastflush = 0 while running: curtime = int(time.time()) curts = time.strftime("%m/%d/%Y %H:%M:%S") message = {} message['ts'] = curts message['field1'] = "This is fun" message['field2'] = "or is it?" message_json = json.dumps(message) print(message_json) try: p.produce(topic, value=message_json, callback=delivery_callback) p.poll(0) except BufferError as e: print("Buffer full, waiting for free space on the queue") p.poll(10) p.produce(topic, value=message_json, callback=delivery_callback) except KeyboardInterrupt: print("\n\nExiting per User Request") p.close() sys.exit(0) delay = random.randint(1, 9) print("Sleeping for %s" % delay) time.sleep(delay)
try: kc.list_topics(timeout=0.2) except KafkaException as e: assert e.args[0].code() in (KafkaError._TIMED_OUT, KafkaError._TRANSPORT) try: kc.list_topics(topic="hi", timeout=0.1) except KafkaException as e: assert e.args[0].code() in (KafkaError._TIMED_OUT, KafkaError._TRANSPORT) kc.close() @pytest.mark.skipif(libversion()[1] < 0x000b0000, reason="requires librdkafka >=0.11.0") def test_store_offsets(): """ Basic store_offsets() tests """ c = Consumer({ 'group.id': 'test', 'enable.auto.commit': True, 'enable.auto.offset.store': False, 'socket.timeout.ms': 50, 'session.timeout.ms': 100 }) c.subscribe(["test"]) try:
name = 'kafkacrypto' # version and license information in setup.py __all__ = [] import logging try: import confluent_kafka from kafkacrypto.confluent_kafka_wrapper import KafkaConsumer, KafkaProducer, TopicPartition, TopicPartitionOffset, OffsetAndMetadata logging.warning("Using confluent_kafka: %s, librdkafka: %s", confluent_kafka.version(), confluent_kafka.libversion()) except ImportError: # fallback to kafka-python logging.warning( "No confluent_kafka package found. Falling back to kafka-python. It is highly" ) logging.warning( "recommended that you install confluent_kafka and librdkafka for better performance," ) logging.warning("especially with large messages.") from kafkacrypto.kafka_python_wrapper import KafkaConsumer, KafkaProducer, TopicPartition, TopicPartitionOffset, OffsetAndMetadata del logging __all__.extend([ 'KafkaConsumer', 'KafkaProducer', 'TopicPartition', 'TopicPartitionOffset', 'OffsetAndMetadata' ]) from kafkacrypto.message import KafkaCryptoMessage from kafkacrypto.crypto import KafkaCrypto from kafkacrypto.controller import KafkaCryptoController from kafkacrypto.kafkacryptostore import KafkaCryptoStore
name='kafkacrypto' # version and license information in setup.py __all__ = [] # # Use warnings to not accidentally initialize the logging subsystem (these can and will # be logged to a logger if logger is configured with captureWarnings). The category # is RuntimeWarning because that is where information about library configurations # should be sent. # import warnings try: import confluent_kafka from kafkacrypto.confluent_kafka_wrapper import KafkaConsumer,KafkaProducer,TopicPartition,TopicPartitionOffset,OffsetAndMetadata warnings.warn("Using confluent_kafka: {}, librdkafka: {}".format(str(confluent_kafka.version()), str(confluent_kafka.libversion())),category=RuntimeWarning) except ImportError: # fallback to kafka-python warnings.warn("No confluent_kafka package found. Falling back to kafka-python. It is highly, recommended that you install confluent_kafka and librdkafka for better performance, especially with large messages.",category=RuntimeWarning) from kafkacrypto.kafka_python_wrapper import KafkaConsumer,KafkaProducer,TopicPartition,TopicPartitionOffset,OffsetAndMetadata del warnings __all__.extend(['KafkaConsumer', 'KafkaProducer', 'TopicPartition', 'TopicPartitionOffset', 'OffsetAndMetadata']) from kafkacrypto.message import KafkaCryptoMessage from kafkacrypto.crypto import KafkaCrypto from kafkacrypto.controller import KafkaCryptoController from kafkacrypto.kafkacryptostore import KafkaCryptoStore from kafkacrypto.chainserver import KafkaCryptoChainServer __all__.extend([ 'KafkaCryptoMessage', 'KafkaCrypto', 'KafkaCryptoStore', 'KafkaCryptoController', 'KafkaCryptoChainServer'])
def main(): print("Confluent Kafka Version: %s - Libversion: %s" % (version(), libversion())) topic_full = os.environ["MAPR_STREAMS_STREAM_LOCATION"].replace( '"', '') + ":" + os.environ["MAPR_STREAMS_TESLA_FULL_TOPIC"].replace( '"', '') topic_stream = os.environ["MAPR_STREAMS_STREAM_LOCATION"].replace( '"', '') + ":" + os.environ["MAPR_STREAMS_TESLA_STREAM_TOPIC"].replace( '"', '') print("Producing full messages to: %s" % topic_full) print("Producing stream messages to: %s" % topic_stream) p = Producer({'bootstrap.servers': ''}) # Listen for messages running = True lastflush = 0 app_auth = getAppAuth() token = loadToken(app_auth) vehicles = loadVehicleInfo(token) vehicle = None for v in vehicles['response']: if v['display_name'] == carname: vehicle = v break if vehicle is None: print("Could not find %s in vehicle list" % carname) sys.exit(1) all_data = loadData(token, vehicle['id'])['response'] stream_items = [ 'speed', 'odometer', 'soc', 'elevation', 'est_heading', 'est_lat', 'est_lng', 'power', 'shift_state', 'native_type', 'heading', 'native_latitude', 'native_longitude' ] stream_string = ",".join(stream_items) output_items = ['timestamp'] + stream_items stream_url = "https://streaming.vn.teslamotors.com/stream/%s/?values=%s" % ( vehicle['vehicle_id'], stream_string) tokenidx = 0 last_stream_line = "" last_all_data = False while running: curtime = int(time.time()) if curtime - last_data > data_secs: try: all_data = loadData(token, vehicle['id'])['response'] if all_data is None: last_all_data = False print( "%s - all_data is None, going to slowly try to refresh this to correct" % curtime) while all_data is None: all_data = loadData(token, vehicle['id'])['response'] if all_data is None: sleep(5) else: if last_all_data == False: print( "%s - all_data success on new start or after previous failure" % curtime) last_all_data = True produceMessage(p, topic_full, json.dumps(all_data)) if curtime % stdout_interval == 0: print("%s - logging at stdout interval - success" % curtime) except: print("%s - All Data load failure" % curtime) try: stream_resp = requests.get(stream_url, auth=(token['uname'], all_data['tokens'][tokenidx]), stream=True, timeout=http_timeout) for line in stream_resp.iter_lines(): curtime = int(time.time()) if line: decoded_line = line.decode('utf-8') if decoded_line.find("Can't validate password") >= 0: all_data = loadData(token, vehicle['id'])['response'] print("%s - Bad password - Refreshing Tokens" % curtime) time.sleep(2) elif last_stream_line != decoded_line: dline = decoded_line.split(",") myrec = OrderedDict(zip(output_items, dline)) produceMessage(p, topic_stream, json.dumps(myrec)) if curtime % stdout_interval == 0: print( "%s - Streaming well - success on stdout_interval" % curtime) myrec = None last_stream_line = decoded_line if int(time.time()) - last_data > data_secs: try: all_data = loadData(token, vehicle['id'])['response'] last_all_data = True produceMessage(p, topic_full, json.dumps(all_data)) except: last_all_data = False break except requests.exceptions.ConnectionError: pass except: pass
}) # Requires librdkafka >=v0.9.4 try: p.produce('mytopic', timestamp=1234567) except NotImplementedError: # Should only fail on non-supporting librdkafka if libversion()[1] >= 0x00090400: raise p.flush() # Should be updated to 0.11.4 when it is released @pytest.mark.skipif(libversion()[1] < 0x000b0400, reason="requires librdkafka >=0.11.4") def test_produce_headers(): """ Test produce() with timestamp arg """ p = Producer({ 'socket.timeout.ms': 10, 'error_cb': error_cb, 'message.timeout.ms': 10 }) binval = pack('hhl', 1, 2, 3) headers_to_test = [ [('headerkey', 'headervalue')], [('dupkey', 'dupvalue'), ('empty', ''), ('dupkey', 'dupvalue')], [('dupkey', 'dupvalue'), ('dupkey', 'diffvalue')],
assert e.args[0].code() == KafkaError._TIMED_OUT try: kc.list_topics(timeout=0.2) except KafkaException as e: assert e.args[0].code() in (KafkaError._TIMED_OUT, KafkaError._TRANSPORT) try: kc.list_topics(topic="hi", timeout=0.1) except KafkaException as e: assert e.args[0].code() in (KafkaError._TIMED_OUT, KafkaError._TRANSPORT) kc.close() @pytest.mark.skipif(libversion()[1] < 0x000b0000, reason="requires librdkafka >=0.11.0") def test_store_offsets(): """ Basic store_offsets() tests """ c = Consumer({'group.id': 'test', 'enable.auto.commit': True, 'enable.auto.offset.store': False, 'socket.timeout.ms': 50, 'session.timeout.ms': 100}) c.subscribe(["test"]) try: c.store_offsets(offsets=[TopicPartition("test", 0, 42)]) except KafkaException as e:
def main (): print("Confluent Kafka Version: %s - Libversion: %s" % (version(), libversion())) print("") print("Using weights file: %s" % weights_file) print("") print("Consuming raw video frames from %s" % topic_frames) print("") print("Producing classified video frames to: %s" % topic_class_frames) print("") print("Consumer group name: %s" % consumer_group) print("") print("Consumer group start: %s" % consumer_group_start) print("") print("Debug is set to %s" % debug) print("") print("nomsgcnt is set to %s - if this is greater than 0, then when we have that many attempts to read a msg from MapR Streams, we will exit" % nomsgcnt) con_conf = {'bootstrap.servers': '', 'group.id': consumer_group, 'default.topic.config': {'auto.offset.reset': consumer_group_start}} pro_conf = {'bootstrap.servers': '', 'message.max.bytes':'2978246'} c = Consumer(con_conf) p = Producer(pro_conf) c.subscribe([topic_frames]) lastmsgtime = time.time() nomsg = 0 running = True while running: msg = c.poll(timeout=1.0) if msg is None: nomsg += 1 if debug: print("No Message - Continuing") if nomsgcnt > 0 and nomsg >= nomsgcnt: print("%s itterations with no messages reached - Exiting Gracefully") sys.exit(0) continue if not msg.error(): mymsg = json.loads(msg.value().decode('utf-8'), object_pairs_hook=OrderedDict) mypart = msg.partition() myoffset = msg.offset() outmsg = OrderedDict() outmsg['ts'] = mymsg['ts'] outmsg['epoch_ts'] = mymsg['epoch_ts'] outmsg['cam_name'] = mymsg['cam_name'] outmsg['src_partition'] = mypart outmsg['src_offset'] = myoffset mybytes = base64.b64decode(mymsg['img']) o = open("/dev/shm/tmp.jpg", "wb") o.write(mybytes) o.close # myimage = np.array(Image.open(BytesIO(mybytes))) curmsgtime = time.time() msgdelta = curmsgtime - lastmsgtime if debug: print("Time between last processed messages: %s" % msgdelta) lastmsgtime = curmsgtime r = python.darknet.detect(net, meta, b'/dev/shm/tmp.jpg') if r != []: if debug: print("Got classification!") curtime = datetime.datetime.now() mystrtime = curtime.strftime("%Y-%m-%d %H:%M:%S") epochtime = int(time.time()) arclass = [] if save_images == 1: try: image = Image.open(BytesIO(mybytes)).convert("RGBA") except: continue draw = ImageDraw.Draw(image) for q in r: j = OrderedDict() name = q[0] j['name'] = name.decode() predict = q[1] j['predict'] = predict x = q[2][0] y = q[2][1] w = q[2][2] z = q[2][3] x_max = (2*x+w)/2 x_min = (2*x-w)/2 y_min = (2*y-z)/2 y_max = (2*y+z)/2 j['x_min'] = x_min j['x_max'] = x_max j['y_min'] = y_min j['y_max'] = y_max for x in range(border): draw.rectangle(((x_min - x, y_min - x), (x_max + x, y_max + x)), fill=None, outline="black") draw.text((x_min + border + 2, y_max - border - 5), name) arclass.append(j) imgSave = BytesIO() image.save(imgSave, format='JPEG') imgSave = imgSave.getvalue() encdata = base64.b64encode(imgSave) encdatastr = encdata.decode('utf-8') else: encdatastr = "" outmsg['class_json'] = arclass outmsg['class_ts'] = mystrtime outmsg['class_epoch_ts'] = epochtime outmsg['class_img'] = encdatastr produceMessage(p, topic_class_frames, json.dumps(outmsg)) else: pass
'message.timeout.ms': 10}) # Requires librdkafka >=v0.9.4 try: p.produce('mytopic', timestamp=1234567) except NotImplementedError: # Should only fail on non-supporting librdkafka if libversion()[1] >= 0x00090400: raise p.flush() # Should be updated to 0.11.4 when it is released @pytest.mark.skipif(libversion()[1] < 0x000b0400, reason="requires librdkafka >=0.11.4") def test_produce_headers(): """ Test produce() with timestamp arg """ p = Producer({'socket.timeout.ms': 10, 'error_cb': error_cb, 'message.timeout.ms': 10}) binval = pack('hhl', 1, 2, 3) headers_to_test = [ [('headerkey', 'headervalue')], [('dupkey', 'dupvalue'), ('empty', ''), ('dupkey', 'dupvalue')], [('dupkey', 'dupvalue'), ('dupkey', 'diffvalue')], [('key_with_null_value', None)], [('binaryval', binval)],
bootstrap_servers = testconf.get('bootstrap.servers') topic = testconf.get('topic') schema_registry_url = testconf.get('schema.registry.url') if len(modes) == 0: modes = test_modes if bootstrap_servers is None or topic is None: print_usage(1, "Missing required property bootstrap.servers") if topic is None: print_usage(1, "Missing required property topic") print('Using confluent_kafka module version %s (0x%x)' % confluent_kafka.version()) print('Using librdkafka version %s (0x%x)' % confluent_kafka.libversion()) print('Testing: %s' % modes) print('Brokers: %s' % bootstrap_servers) print('Topic prefix: %s' % topic) if 'producer' in modes: print('=' * 30, 'Verifying Producer', '=' * 30) verify_producer() if 'performance' in modes: print('=' * 30, 'Verifying Producer performance (with dr_cb)', '=' * 30) verify_producer_performance(with_dr_cb=True) if 'performance' in modes: print('=' * 30, 'Verifying Producer performance (without dr_cb)', '=' * 30) verify_producer_performance(with_dr_cb=False)
if len(sys.argv) > 1: bootstrap_servers = sys.argv[1] if len(sys.argv) > 2: topic = sys.argv[2] if len(sys.argv) > 3: schema_registry_url = sys.argv[3] else: print_usage(1) if len(modes) == 0: modes = ['consumer', 'producer', 'avro', 'performance'] print('Using confluent_kafka module version %s (0x%x)' % confluent_kafka.version()) print('Using librdkafka version %s (0x%x)' % confluent_kafka.libversion()) if 'producer' in modes: print('=' * 30, 'Verifying Producer', '=' * 30) verify_producer() if 'performance' in modes: print('=' * 30, 'Verifying Producer performance (with dr_cb)', '=' * 30) verify_producer_performance(with_dr_cb=True) if 'performance' in modes: print('=' * 30, 'Verifying Producer performance (without dr_cb)', '=' * 30) verify_producer_performance(with_dr_cb=False)
""" Default atexit registered functions will not be triggered on error signals. However - if we manually handle the error signals and exit, the functions will be triggered. """ import signal import sys import confluent_kafka import structlog logger = structlog.get_logger(__name__) logger.info("Using confluent kafka versions", version=confluent_kafka.version(), libversion=confluent_kafka.libversion()) def error_handler(signum, frame): logger.info("Received signal", signum=signum) exit_code = 128 + signum sys.exit(exit_code) def interrupt_handler(signum, frame): logger.info("Received signal", signum=signum) sys.exit(1) signal.signal(signal.SIGTERM, error_handler) # graceful shutdown signal.signal(signal.SIGINT, interrupt_handler) # keyboard interrupt