def test_version(): print('Using confluent_kafka module version %s (0x%x)' % confluent_kafka.version()) sver, iver = confluent_kafka.version() assert len(sver) > 0 assert iver > 0 print('Using librdkafka version %s (0x%x)' % confluent_kafka.libversion()) sver, iver = confluent_kafka.libversion() assert len(sver) > 0 assert iver > 0
def write_to_kafka(bootstrap_servers, schema_registry_url, topic_name, data): print("Kafka Version : ", confluent_kafka.version(),confluent_kafka.libversion()) schema_registry_conf = {'url': schema_registry_url} schema_registry_client = SchemaRegistryClient(schema_registry_conf) value_avro_serializer = AvroSerializer(schemas.weather_source_schema, schema_registry_client) string_serializer = StringSerializer('utf-8') conf = {'bootstrap.servers': bootstrap_servers, 'client.id': socket.gethostname(), 'on_delivery': delivery_report, 'key.serializer': string_serializer, 'value.serializer': value_avro_serializer } avroProducer = SerializingProducer(conf) key=datetime.date.today() + '~' + str(data['lat']) + '~' + str(data['lon']) message = json.dumps(data, cls=DatetimeEncoder) print("Key Type : ", type(key)) print("Value Type : ", type(json.loads(message))) avroProducer.produce(topic=topic_name, key=key, value=json.loads(message)) avroProducer.flush()
def main(): print("Confluent Kafka Version: %s - Libversion: %s" % (version(), libversion())) print("") print("Connecting to RTSP: %s" % rtsp_url) print("") print("Producing video frames to: %s" % topic_frames) print("") print( "Skipping every %s frames and then saving up to %s (-1 is unlimited) frames" % (skipframes, capmax)) print("") p = Producer({'bootstrap.servers': '', 'message.max.bytes': '2978246'}) output_loc = "./test" capcount = 0 cap = cv2.VideoCapture(rtsp_url) while capcount < capmax or capmax == -1: ret, frame = cap.read() if capcount % skipframes == 0: curtime = datetime.datetime.now() mystrtime = curtime.strftime("%Y-%m-%d %H:%M:%S") epochtime = int(time.time()) print("Saving to Stream at %s" % capcount) img_str = cv2.imencode('.jpg', frame)[1].tostring() encdata = base64.b64encode(img_str) encdatastr = encdata.decode('utf-8') myrec = OrderedDict() myrec['ts'] = mystrtime myrec['epoch_ts'] = epochtime myrec['cam_name'] = cam_name myrec['img'] = encdatastr produceMessage(p, topic_frames, json.dumps(myrec)) # This is more verbose method of writing so we can get raw bytes to save to a stream # w = open(output_loc + "/cap_test_%s.jpg" % capcount, 'wb') # w.write(img_str) # w.close() # This is the prefered method for writing to a file from OpenCV, I am using theimencode so I can get the byte to write directly to MapR Streams # cv2.imwrite(output_loc + "/cap_%s.jpg" % (capcount), frame) capcount += 1 if capcount >= 10000000: if capcount % skipframe == 0: capcount = 0
def main(): print("Confluent Kafka Version: %s - Libversion: %s" % (version(), libversion())) print("") conf = { 'group.id': 'img2file1', 'default.topic.config': { 'auto.offset.reset': 'earliest' } } c = Consumer(conf) c.subscribe([topic_frames]) running = True while running: msg = c.poll(timeout=1.0) if msg is None: continue if not msg.error(): mymsg = json.loads(msg.value().decode('utf-8'), object_pairs_hook=OrderedDict) outmsg = OrderedDict() outmsg['ts'] = mymsg['ts'] outmsg['epoch_ts'] = mymsg['epoch_ts'] outmsg['cam_name'] = mymsg['cam_name'] fileout = "%s/stream_%s_%s.jpg" % (output_loc, outmsg['cam_name'], outmsg['ts'].replace( " ", "_").replace(":", "_")) mybytes = base64.b64decode(mymsg['img']) print("Processing message: %s and saving to %s" % (outmsg, fileout)) w = open(fileout, 'wb') w.write(mybytes) w.close() elif msg.error().code() != KafkaError._PARTITION_EOF: print(msg.error()) running = False c.close()
def main(): print("Confluent Kafka Version: %s - Libversion: %s" % (version(), libversion())) topic = os.environ["MAPR_STREAMS_STREAM_LOCATION"].replace( '"', '') + ":" + os.environ["MAPR_STREAMS_TOPIC"].replace('"', '') print("Producing messages to: %s" % topic) p = Producer({'bootstrap.servers': ''}) # Listen for messages running = True lastflush = 0 while running: curtime = int(time.time()) curts = time.strftime("%m/%d/%Y %H:%M:%S") message = {} message['ts'] = curts message['field1'] = "This is fun" message['field2'] = "or is it?" message_json = json.dumps(message) print(message_json) try: p.produce(topic, value=message_json, callback=delivery_callback) p.poll(0) except BufferError as e: print("Buffer full, waiting for free space on the queue") p.poll(10) p.produce(topic, value=message_json, callback=delivery_callback) except KeyboardInterrupt: print("\n\nExiting per User Request") p.close() sys.exit(0) delay = random.randint(1, 9) print("Sleeping for %s" % delay) time.sleep(delay)
print_usage(1, 'unknown option ' + opt) if len(sys.argv) > 1: bootstrap_servers = sys.argv[1] if len(sys.argv) > 2: topic = sys.argv[2] if len(sys.argv) > 3: schema_registry_url = sys.argv[3] else: print_usage(1) if len(modes) == 0: modes = ['consumer', 'producer', 'avro', 'performance'] print('Using confluent_kafka module version %s (0x%x)' % confluent_kafka.version()) print('Using librdkafka version %s (0x%x)' % confluent_kafka.libversion()) if 'producer' in modes: print('=' * 30, 'Verifying Producer', '=' * 30) verify_producer() if 'performance' in modes: print('=' * 30, 'Verifying Producer performance (with dr_cb)', '=' * 30) verify_producer_performance(with_dr_cb=True) if 'performance' in modes: print('=' * 30, 'Verifying Producer performance (without dr_cb)', '=' * 30) verify_producer_performance(with_dr_cb=False)
name = 'kafkacrypto' # version and license information in setup.py __all__ = [] import logging try: import confluent_kafka from kafkacrypto.confluent_kafka_wrapper import KafkaConsumer, KafkaProducer, TopicPartition, TopicPartitionOffset, OffsetAndMetadata logging.warning("Using confluent_kafka: %s, librdkafka: %s", confluent_kafka.version(), confluent_kafka.libversion()) except ImportError: # fallback to kafka-python logging.warning( "No confluent_kafka package found. Falling back to kafka-python. It is highly" ) logging.warning( "recommended that you install confluent_kafka and librdkafka for better performance," ) logging.warning("especially with large messages.") from kafkacrypto.kafka_python_wrapper import KafkaConsumer, KafkaProducer, TopicPartition, TopicPartitionOffset, OffsetAndMetadata del logging __all__.extend([ 'KafkaConsumer', 'KafkaProducer', 'TopicPartition', 'TopicPartitionOffset', 'OffsetAndMetadata' ]) from kafkacrypto.message import KafkaCryptoMessage from kafkacrypto.crypto import KafkaCrypto from kafkacrypto.controller import KafkaCryptoController from kafkacrypto.kafkacryptostore import KafkaCryptoStore
with open(testconf) as f: testconf = json.load(f) resolve_envs(testconf) bootstrap_servers = testconf.get('bootstrap.servers', None) topic = testconf.get('topic', None) schema_registry_url = testconf.get('schema.registry.url', None) if len(modes) == 0: modes = default_modes if bootstrap_servers is None or topic is None: print_usage(1, "Properties bootstrap.servers and topic must be set. " "Use tests/testconf-example.json as a template when creating a new conf file.") print('Using confluent_kafka module version %s (0x%x)' % confluent_kafka.version()) print('Using librdkafka version %s (0x%x)' % confluent_kafka.libversion()) print('Testing: %s' % modes) print('Brokers: %s' % bootstrap_servers) print('Topic prefix: %s' % topic) if 'producer' in modes: print('=' * 30, 'Verifying Producer', '=' * 30) verify_producer() if 'performance' in modes: print('=' * 30, 'Verifying Producer performance (with dr_cb)', '=' * 30) verify_producer_performance(with_dr_cb=True) if 'performance' in modes: print('=' * 30, 'Verifying Producer performance (without dr_cb)', '=' * 30)
if i <= 0: raise ValueError("Configuration lines must be `name=value..`, not {}".format(line)) name = line[:i] value = line[i+1:] conf[name] = value if args.brokers is not None: # Overwrite any brokers specified in configuration file with # brokers from -b command line argument conf['bootstrap.servers'] = args.brokers if 'group.id' not in conf: # Generate a group.id bound to this client and python version conf['group.id'] = 'soakclient.py-{}-{}'.format(version()[0], sys.version.split(' ')[0]) # We don't care about partition EOFs conf['enable.partition.eof'] = False # Create SoakClient soak = SoakClient(args.topic, args.rate, conf) # Run until interrupted try: while soak.run: time.sleep(10) soak.logger.info("Soak client aborted") except (KeyboardInterrupt):
resolve_envs(testconf) bootstrap_servers = testconf.get('bootstrap.servers') topic = testconf.get('topic') schema_registry_url = testconf.get('schema.registry.url') if len(modes) == 0: modes = test_modes if bootstrap_servers is None or topic is None: print_usage(1, "Missing required property bootstrap.servers") if topic is None: print_usage(1, "Missing required property topic") print('Using confluent_kafka module version %s (0x%x)' % confluent_kafka.version()) print('Using librdkafka version %s (0x%x)' % confluent_kafka.libversion()) print('Testing: %s' % modes) print('Brokers: %s' % bootstrap_servers) print('Topic prefix: %s' % topic) if 'producer' in modes: print('=' * 30, 'Verifying Producer', '=' * 30) verify_producer() if 'performance' in modes: print('=' * 30, 'Verifying Producer performance (with dr_cb)', '=' * 30) verify_producer_performance(with_dr_cb=True) if 'performance' in modes: print('=' * 30, 'Verifying Producer performance (without dr_cb)', '=' * 30)
def __init__(self, topic, rate, conf): """ SoakClient constructor. conf is the client configuration """ self.topic = topic self.rate = rate self.disprate = int(rate * 10) self.run = True self.stats_cnt = {'producer': 0, 'consumer': 0} self.start_time = time.time() self.last_rusage = None self.last_rusage_time = None self.proc = psutil.Process(os.getpid()) self.logger = logging.getLogger('soakclient') self.logger.setLevel(logging.DEBUG) handler = logging.StreamHandler() handler.setFormatter(logging.Formatter('%(asctime)-15s %(levelname)-8s %(message)s')) self.logger.addHandler(handler) # Construct a unique id to use for metrics hostname so that # multiple instances of the SoakClient can run on the same machine. hostname = datadog.util.hostname.get_hostname() self.hostname = "py-{}-{}".format(hostname, self.topic) self.logger.info("SoakClient id {}".format(self.hostname)) if 'group.id' not in conf: # Generate a group.id bound to this client and python version conf['group.id'] = 'soakclient-{}-{}-{}'.format( self.hostname, version()[0], sys.version.split(' ')[0]) # Separate datadog config from client config datadog_conf = {k[len("datadog."):]: conf[k] for k in conf.keys() if k.startswith("datadog.")} conf = {k: v for k, v in conf.items() if not k.startswith("datadog.")} # Set up datadog agent self.init_datadog(datadog_conf) def filter_config(conf, filter_out, strip_prefix): len_sp = len(strip_prefix) out = {} for k, v in conf.items(): if len([x for x in filter_out if k.startswith(x)]) > 0: continue if k.startswith(strip_prefix): k = k[len_sp:] out[k] = v return out # Create topic (might already exist) aconf = filter_config(conf, ["consumer.", "producer."], "admin.") self.create_topic(self.topic, aconf) # # Create Producer and Consumer, each running in its own thread. # conf['stats_cb'] = self.stats_cb conf['statistics.interval.ms'] = 10000 # Producer pconf = filter_config(conf, ["consumer.", "admin."], "producer.") pconf['error_cb'] = self.producer_error_cb self.producer = Producer(pconf) # Consumer cconf = filter_config(conf, ["producer.", "admin."], "consumer.") cconf['error_cb'] = self.consumer_error_cb cconf['on_commit'] = self.consumer_commit_cb self.logger.info("consumer: using group.id {}".format(cconf['group.id'])) self.consumer = Consumer(cconf) # Create and start producer thread self.producer_thread = threading.Thread(target=self.producer_thread_main) self.producer_thread.start() # Create and start consumer thread self.consumer_thread = threading.Thread(target=self.consumer_thread_main) self.consumer_thread.start()
def main (): print("Confluent Kafka Version: %s - Libversion: %s" % (version(), libversion())) print("") print("Using weights file: %s" % weights_file) print("") print("Consuming raw video frames from %s" % topic_frames) print("") print("Producing classified video frames to: %s" % topic_class_frames) print("") print("Consumer group name: %s" % consumer_group) print("") print("Consumer group start: %s" % consumer_group_start) print("") print("Debug is set to %s" % debug) print("") print("nomsgcnt is set to %s - if this is greater than 0, then when we have that many attempts to read a msg from MapR Streams, we will exit" % nomsgcnt) con_conf = {'bootstrap.servers': '', 'group.id': consumer_group, 'default.topic.config': {'auto.offset.reset': consumer_group_start}} pro_conf = {'bootstrap.servers': '', 'message.max.bytes':'2978246'} c = Consumer(con_conf) p = Producer(pro_conf) c.subscribe([topic_frames]) lastmsgtime = time.time() nomsg = 0 running = True while running: msg = c.poll(timeout=1.0) if msg is None: nomsg += 1 if debug: print("No Message - Continuing") if nomsgcnt > 0 and nomsg >= nomsgcnt: print("%s itterations with no messages reached - Exiting Gracefully") sys.exit(0) continue if not msg.error(): mymsg = json.loads(msg.value().decode('utf-8'), object_pairs_hook=OrderedDict) mypart = msg.partition() myoffset = msg.offset() outmsg = OrderedDict() outmsg['ts'] = mymsg['ts'] outmsg['epoch_ts'] = mymsg['epoch_ts'] outmsg['cam_name'] = mymsg['cam_name'] outmsg['src_partition'] = mypart outmsg['src_offset'] = myoffset mybytes = base64.b64decode(mymsg['img']) o = open("/dev/shm/tmp.jpg", "wb") o.write(mybytes) o.close # myimage = np.array(Image.open(BytesIO(mybytes))) curmsgtime = time.time() msgdelta = curmsgtime - lastmsgtime if debug: print("Time between last processed messages: %s" % msgdelta) lastmsgtime = curmsgtime r = python.darknet.detect(net, meta, b'/dev/shm/tmp.jpg') if r != []: if debug: print("Got classification!") curtime = datetime.datetime.now() mystrtime = curtime.strftime("%Y-%m-%d %H:%M:%S") epochtime = int(time.time()) arclass = [] if save_images == 1: try: image = Image.open(BytesIO(mybytes)).convert("RGBA") except: continue draw = ImageDraw.Draw(image) for q in r: j = OrderedDict() name = q[0] j['name'] = name.decode() predict = q[1] j['predict'] = predict x = q[2][0] y = q[2][1] w = q[2][2] z = q[2][3] x_max = (2*x+w)/2 x_min = (2*x-w)/2 y_min = (2*y-z)/2 y_max = (2*y+z)/2 j['x_min'] = x_min j['x_max'] = x_max j['y_min'] = y_min j['y_max'] = y_max for x in range(border): draw.rectangle(((x_min - x, y_min - x), (x_max + x, y_max + x)), fill=None, outline="black") draw.text((x_min + border + 2, y_max - border - 5), name) arclass.append(j) imgSave = BytesIO() image.save(imgSave, format='JPEG') imgSave = imgSave.getvalue() encdata = base64.b64encode(imgSave) encdatastr = encdata.decode('utf-8') else: encdatastr = "" outmsg['class_json'] = arclass outmsg['class_ts'] = mystrtime outmsg['class_epoch_ts'] = epochtime outmsg['class_img'] = encdatastr produceMessage(p, topic_class_frames, json.dumps(outmsg)) else: pass
def main(): print("Confluent Kafka Version: %s - Libversion: %s" % (version(), libversion())) topic_full = os.environ["MAPR_STREAMS_STREAM_LOCATION"].replace( '"', '') + ":" + os.environ["MAPR_STREAMS_TESLA_FULL_TOPIC"].replace( '"', '') topic_stream = os.environ["MAPR_STREAMS_STREAM_LOCATION"].replace( '"', '') + ":" + os.environ["MAPR_STREAMS_TESLA_STREAM_TOPIC"].replace( '"', '') print("Producing full messages to: %s" % topic_full) print("Producing stream messages to: %s" % topic_stream) p = Producer({'bootstrap.servers': ''}) # Listen for messages running = True lastflush = 0 app_auth = getAppAuth() token = loadToken(app_auth) vehicles = loadVehicleInfo(token) vehicle = None for v in vehicles['response']: if v['display_name'] == carname: vehicle = v break if vehicle is None: print("Could not find %s in vehicle list" % carname) sys.exit(1) all_data = loadData(token, vehicle['id'])['response'] stream_items = [ 'speed', 'odometer', 'soc', 'elevation', 'est_heading', 'est_lat', 'est_lng', 'power', 'shift_state', 'native_type', 'heading', 'native_latitude', 'native_longitude' ] stream_string = ",".join(stream_items) output_items = ['timestamp'] + stream_items stream_url = "https://streaming.vn.teslamotors.com/stream/%s/?values=%s" % ( vehicle['vehicle_id'], stream_string) tokenidx = 0 last_stream_line = "" last_all_data = False while running: curtime = int(time.time()) if curtime - last_data > data_secs: try: all_data = loadData(token, vehicle['id'])['response'] if all_data is None: last_all_data = False print( "%s - all_data is None, going to slowly try to refresh this to correct" % curtime) while all_data is None: all_data = loadData(token, vehicle['id'])['response'] if all_data is None: sleep(5) else: if last_all_data == False: print( "%s - all_data success on new start or after previous failure" % curtime) last_all_data = True produceMessage(p, topic_full, json.dumps(all_data)) if curtime % stdout_interval == 0: print("%s - logging at stdout interval - success" % curtime) except: print("%s - All Data load failure" % curtime) try: stream_resp = requests.get(stream_url, auth=(token['uname'], all_data['tokens'][tokenidx]), stream=True, timeout=http_timeout) for line in stream_resp.iter_lines(): curtime = int(time.time()) if line: decoded_line = line.decode('utf-8') if decoded_line.find("Can't validate password") >= 0: all_data = loadData(token, vehicle['id'])['response'] print("%s - Bad password - Refreshing Tokens" % curtime) time.sleep(2) elif last_stream_line != decoded_line: dline = decoded_line.split(",") myrec = OrderedDict(zip(output_items, dline)) produceMessage(p, topic_stream, json.dumps(myrec)) if curtime % stdout_interval == 0: print( "%s - Streaming well - success on stdout_interval" % curtime) myrec = None last_stream_line = decoded_line if int(time.time()) - last_data > data_secs: try: all_data = loadData(token, vehicle['id'])['response'] last_all_data = True produceMessage(p, topic_full, json.dumps(all_data)) except: last_all_data = False break except requests.exceptions.ConnectionError: pass except: pass
name='kafkacrypto' # version and license information in setup.py __all__ = [] # # Use warnings to not accidentally initialize the logging subsystem (these can and will # be logged to a logger if logger is configured with captureWarnings). The category # is RuntimeWarning because that is where information about library configurations # should be sent. # import warnings try: import confluent_kafka from kafkacrypto.confluent_kafka_wrapper import KafkaConsumer,KafkaProducer,TopicPartition,TopicPartitionOffset,OffsetAndMetadata warnings.warn("Using confluent_kafka: {}, librdkafka: {}".format(str(confluent_kafka.version()), str(confluent_kafka.libversion())),category=RuntimeWarning) except ImportError: # fallback to kafka-python warnings.warn("No confluent_kafka package found. Falling back to kafka-python. It is highly, recommended that you install confluent_kafka and librdkafka for better performance, especially with large messages.",category=RuntimeWarning) from kafkacrypto.kafka_python_wrapper import KafkaConsumer,KafkaProducer,TopicPartition,TopicPartitionOffset,OffsetAndMetadata del warnings __all__.extend(['KafkaConsumer', 'KafkaProducer', 'TopicPartition', 'TopicPartitionOffset', 'OffsetAndMetadata']) from kafkacrypto.message import KafkaCryptoMessage from kafkacrypto.crypto import KafkaCrypto from kafkacrypto.controller import KafkaCryptoController from kafkacrypto.kafkacryptostore import KafkaCryptoStore from kafkacrypto.chainserver import KafkaCryptoChainServer __all__.extend([ 'KafkaCryptoMessage', 'KafkaCrypto', 'KafkaCryptoStore', 'KafkaCryptoController', 'KafkaCryptoChainServer'])