def main(): producer = KafkaProducer( bootstrap_servers=server, security_protocol='SSL', ssl_cafile='<CARoot>', ssl_certfile='<certificate>', ssl_keyfile='<key>', value_serializer=lambda v: dumps(v).encode("utf-8")) consumer = KafkaConsumer("my_topic", bootstrap_servers=server, security_protocol='SSL', ssl_cafile='<CARoot>', ssl_certfile='<certificate>', ssl_keyfile='<key>', value_deserializer=lambda v: loads(v)) producer_metrics, consumer_metrics = producer.metrics(), consumer.metrics() pprint(producer_metrics) pprint(consumer_metrics) for msg in consumer: print(msg) producer.send("new_topic", msg.value) producer.flush()
def main(kafka_broker, kafka_topic, file_path): """Parse the csv and send every line to the kafka topic""" producer = KafkaProducer( bootstrap_servers=kafka_broker, value_serializer=lambda x: json.dumps(x).encode("utf-8")) logger.info("Reading messages from csv") for item in read_data(file_path): producer.send(kafka_topic, item) logger.info("Sending all messages to Kafka") producer.close() pprint.pprint(producer.metrics())
def kafka_producer_test(): # https://github.com/dpkp/kafka-python#kafkaconsumer producer = KafkaProducer( bootstrap_servers=['172.26.210.149:9092', '172.26.210.150:9092', '172.26.210.151:9092'], value_serializer=lambda v: json.dumps(v).encode('utf-8') ) future = producer.send(topic='service_test', value={'foo': 'bar'}) result = future.get(timeout=60) print(result) metrics = producer.metrics() print(metrics)
def main(kafka_broker, kafka_topic, file_path): """Parse the csv and send every line to the kafka topic as a json dict of column headers and values""" producer = KafkaProducer( bootstrap_servers=kafka_broker, value_serializer=lambda x: json.dumps(x).encode("utf-8")) logger.info("Reading messages from csv") for item in read_data(file_path): # Important that both key and value are byte arrays producer.send(kafka_topic, value=item) logger.info("Sending all messages to Kafka") producer.flush() producer.close() pprint.pprint(producer.metrics())
def process_SEC_rss(item): index_rss = 'http://www.sec.gov/Archives/edgar/monthly/xbrlrss-{}.xml'.format( item) producer = KafkaProducer(bootstrap_servers=kafka_url) rss_feed = urllib2.urlopen(index_rss) index_data = rss_feed.read() rss_feed.close() index_doc = xmltodict.parse(index_data) item_list = index_doc['rss']['channel']['item'] msg_count = 0 for entry in item_list: formType = entry['edgar:xbrlFiling']['edgar:formType'] filingInfo = entry['edgar:xbrlFiling'] if (formType == '10-Q' or formType == '10-K'): newRow = { 'companyName': get_value(filingInfo, 'edgar:companyName'), 'guid': get_value(entry, 'guid'), 'xml_filing': index_rss, 'pubDate': get_value(entry, 'pubDate'), 'formType': formType, 'filingDate': get_value(filingInfo, 'edgar:filingDate'), 'cikNumbver': get_value(filingInfo, 'edgar:cikNumber'), 'accessionNumber': get_value(filingInfo, 'edgar:accessionNumber'), 'fileNumber': get_value(filingInfo, 'edgar:fileNumber'), 'filingInfo': get_value(filingInfo, 'edgar:period'), 'fiscalYearEnd': get_value(filingInfo, 'edgar:fiscalYearEnd'), } # cols = newRow.keys() # vals = [newRow[x] for x in cols] # vals_str_list = ["%s"] * len(vals) # vals_str = ", ".join(vals_str_list) try: jsec = json.dumps(newRow) producer.send(topic_name, jsec) producer.flush() msg_count = msg_count + 1 print("Added {} sec filings".format(msg_count)) except e: print "Exception encountered {e}" metrics = producer.metrics() print metrics producer.close()
class Kafka_Producer(Kafka_Connection): def __init__(self, config): super(Kafka_Producer, self).__init__(config) def connect(self): self.__log.info("Kafka Producer Connecting to Kafka Topic") if self.security_protocol == "PLAINTEXT": self.broker = KafkaProducer(bootstrap_servers=json.loads( self.bootstrap_servers), retries=5) self.__log.info("Status of the connection is {0}".format( self.broker.metrics())) elif self.security_protocol == "SASL_SSL": self.__log.info("Connecting over sasl_ssl") # SSL # create a new context using system defaults, disable all but TLS1.2 context = ssl.create_default_context() context.verify_mode = ssl.CERT_REQUIRED context.load_verify_locations(self.ssl_cafile) context.load_cert_chain(certfile=self.ssl_certfile, keyfile=self.ssl_keyfile) context.options &= ssl.OP_NO_TLSv1 context.options &= ssl.OP_NO_TLSv1_1 self.broker = KafkaProducer( bootstrap_servers=json.loads(self.bootstrap_servers), # consumer_timeout_ms=self.consumer_timeout_ms, security_protocol=self.security_protocol, sasl_mechanism=self.sasl_mechanism, ssl_context=context, api_version=(0, 10)) def send(self, msg): self.__log.info("Kafka Producer Send Function") self.broker.send(self.outbound_topic, b"{0}".format(msg)) return 1 def close(self): self.broker.close()
def run(self): producer = KafkaProducer(bootstrap_servers='localhost:9092', \ acks=1, \ linger_ms=100, \ compression_type="gzip", \ buffer_memory=64*1024*1024 \ ) totalNum = 0 # while not self.stop_event.is_set(): start = time.time() for i in range(self.event_range[0], self.event_range[1]): #producer.send('my-topic', b"event id="+str(i)) producer.send('my-topic', MY_MSG) totalNum+=1 if totalNum%NO_OF_EACHROUND==0: end = time.time() print("Producer,%d,%d" % (NO_OF_EACHROUND, (end-start)*1000)) start = time.time() #time.sleep(1) print "producer metrics######,", producer.metrics() producer.close()
if s >= 200 and s < 400: u2 = u urls_traversed = [ r.url for r in req.history if r.status_code >= 300 and r.status_code < 400 ] if len(urls_traversed) > 0: u2 = urls_traversed[-1] parts = urlparse(u2) # only successful contacts are placed into the canonical_url_queue, with the # recommended url to use based on whether http redirects are issued uuid_bytes = uuid.uuid1().bytes producer.send('canonical_sites', key=uuid_bytes, value={ 'original_url': u, 'canonical_url': u2, 'canonical_domain': parts.hostname, 'when': when.strftime("%d-%b-%Y %H:%M:%S%z"), 'http_status': s }) # ensure thug processes this url ie. the url is scanned with a view to sub-pages being scanned producer.send('4thug', {'url': u2}) else: producer.send('failed_url_queue', {'url': u, 'http_status': s}) producer.flush() print(json.dumps(producer.metrics(), sort_keys=True, indent=4))
class KafkaClient(object): def __init__(self, bootstrap_servers, topic, group_id=None): if group_id is not None: self.group_id = group_id self.allow_hotreload = True else: self.group_id = 'kafka_topic_dumper_{}'.format(uuid4()) self.allow_hotreload = False self.bootstrap_servers = bootstrap_servers.split(",") self.topic = topic self.consumer = None self.producer = None self.timeout_in_sec = 60 self.dump_state_topic = 'kafka-topic-dumper' self.s3_path = 'kafka-topic-dumper-data/' self.s3_client = None def _get_consumer(self): if self.consumer is not None: return try: logger.info('Starting consumer') self.consumer = KafkaConsumer( bootstrap_servers=self.bootstrap_servers, group_id=self.group_id, enable_auto_commit=True) except Exception as err: msg = 'Can not create KafkaConsumer instance. Reason=<{}>' logger.exception(msg.format(err)) raise err def _get_s3_client(self): if self.s3_client is None: self.s3_client = boto3.client('s3') return self.s3_client def _get_producer(self): if self.producer is not None: return try: logger.info('Starting producer') self.producer = KafkaProducer( bootstrap_servers=self.bootstrap_servers, key_serializer=bytes_serializer, value_serializer=bytes_serializer) except Exception as err: msg = 'Can not create KafkaProducer instance. Reason=<{}>' logger.exception(msg.format(err)) raise err def open(self): self._get_consumer() self._get_producer() def _close_consumer(self): logger.info("Closing consumer") self.consumer.close() self.consumer = None def _close_producer(self): logger.info("Closing producer") self.producer.flush() logger.debug('Statistics {}'.format(self.producer.metrics())) self.producer.close() self.producer = None def close(self): self._close_consumer() self._close_producer() def _get_partitions(self, topic): partitions = self.consumer.partitions_for_topic(topic) or [] count = 0 while not partitions and count < 500000: self.consumer.subscribe(topic) partitions = self.consumer.partitions_for_topic(topic) or [] sleep(0.1) msg = "Got the following partitions=<{}> for topic=<{}>" logger.info(msg.format(partitions, topic)) topic_partitions = list( map(lambda p: TopicPartition(topic, p), partitions)) msg = "Got the following topic partitions=<{}>" logger.info(msg.format(topic_partitions)) return topic_partitions def _get_offsets(self, topic=None): if topic is None: topic = self.topic topic_partitions = self._get_partitions(topic=topic) beginning_offsets = (self.consumer.beginning_offsets(topic_partitions) or {}) msg = "Got the following beginning offsets=<{}>" logger.info(msg.format(beginning_offsets)) commited_offsets = {} msg = "Partition=<{}> has the current offset=<{}> for <{}>" for tp in topic_partitions: offset = self.consumer.committed(tp) commited_offsets[tp] = offset logger.debug(msg.format(tp, offset, self.group_id)) end_offsets = self.consumer.end_offsets(topic_partitions) or {} msg = "Got the following end offsets=<{}>" logger.info(msg.format(end_offsets)) return beginning_offsets, commited_offsets, end_offsets def _calculate_offsets(self, beginning_offsets, end_offsets, num_messages_to_consume): perfect_displacement = ceil(num_messages_to_consume / max(len(beginning_offsets), 1)) offsets = {} num_messages_available = 0 for tp, offset in beginning_offsets.items(): offsets[tp] = max(beginning_offsets[tp], end_offsets[tp] - perfect_displacement) num_messages_available += end_offsets[tp] - offsets[tp] return offsets, num_messages_available def _set_offsets(self, offsets): offset_and_metadata = { tp: OffsetAndMetadata(offset, b'') for tp, offset in offsets.items() } msg = "Generated the following offsets=<{}>" logger.debug(msg.format(offset_and_metadata)) self.consumer.commit(offset_and_metadata) def _get_messages(self, num_messages_to_consume): messages = [] while len(messages) < num_messages_to_consume: record = next(self.consumer) line = (record.key, record.value) messages.append(line) self.consumer.commit() return messages def _write_messages_to_file(self, messages, local_path): df = pd.DataFrame(messages) table = pa.Table.from_pandas(df) pq.write_table(table, local_path, compression='gzip') def _send_dump_file(self, local_path, bucket_name, dump_id): file_name = path.basename(local_path) s3_path = path.join(self.s3_path, dump_id, file_name) logger.info('Sending file <{}> to s3'.format(file_name)) s3_client = self._get_s3_client() s3_client.upload_file(local_path, bucket_name, s3_path, ExtraArgs={'ACL': 'private'}, Callback=ProgressPercentage(local_path)) logger.debug('Deleting file <{}>'.format(file_name)) remove(local_path) def _get_transformer_class(self, transformer_id): [module_name, class_name] = transformer_id.split(":") module = __import__(module_name, globals(), locals(), [class_name], 0) cl = getattr(module, class_name) return cl() def get_messages(self, num_messages_to_consume, max_package_size_in_msgs, local_dir, bucket_name, dry_run, dump_id): # set offsets msg = ('Will ask kafka for <{}> messages ' + 'and save it in files with <{}> messages') logger.debug( msg.format(num_messages_to_consume, max_package_size_in_msgs)) beginning_offsets, commited_offsets, end_offsets = self._get_offsets() offsets, num_messages_available = self._calculate_offsets( beginning_offsets=beginning_offsets, end_offsets=end_offsets, num_messages_to_consume=num_messages_to_consume) self._set_offsets(offsets) # get messages self.consumer.subscribe(topics=[self.topic]) msg = 'Trying to dump <{}> messages' logger.info(msg.format(num_messages_available)) remaining_messages = num_messages_available num_dumped_messages = 0 dump_dir = path.join(local_dir, dump_id) makedirs(dump_dir, exist_ok=True) logger.debug('Dump directory <{}> created'.format(dump_dir)) while remaining_messages > 0: batch_size = min(remaining_messages, max_package_size_in_msgs) logger.debug('Fetching batch with size=<{}>'.format(batch_size)) file_name = '{}-{:015d}.parquet'.format(dump_id, num_dumped_messages) local_path = path.join(local_dir, dump_id, file_name) messages = self._get_messages(num_messages_to_consume=batch_size) self._write_messages_to_file(messages=messages, local_path=local_path) if not dry_run: self._send_dump_file(local_path=local_path, bucket_name=bucket_name, dump_id=dump_id) remaining_messages -= batch_size num_dumped_messages += batch_size logger.info('Dump done!') def find_latest_dump_id(self, bucket_name): paginator = self._get_s3_client().get_paginator('list_objects_v2') prefix = self.s3_path.rstrip('/') + '/' response_iterator = paginator.paginate(Bucket=bucket_name, Prefix=prefix, Delimiter='/') def strip(r): return r['Prefix'][len(prefix):].rstrip('/') prefixes = [] for response in response_iterator: prefixes.extend(map(strip, response['CommonPrefixes'])) dump_id = max(prefixes) logger.debug('Prefix chosen was <{}>'.format(dump_id)) return dump_id def _get_file_names(self, bucket_name, dump_id): paginator = self._get_s3_client().get_paginator('list_objects_v2') dump_path = path.join(self.s3_path, dump_id) + '/' response_iterator = paginator.paginate(Bucket=bucket_name, Prefix=dump_path) file_names = [] for response in response_iterator: if response['KeyCount'] > 0: file_names.extend( (f['Key'], f['Size']) for f in response['Contents']) file_names.sort() if not file_names: msg = 'Can not found files for this dump id <{}>' logger.error(msg.format(dump_id)) raise Exception('EmptyS3Response') return file_names def _gen_state(self, dump_id, transformer_id): _, _, end_offsets = self._get_offsets() if not end_offsets: msg = 'Can not find offsets for topic <{}>' raise Exception(msg.format(self.topic)) state_offsets = {} for partition, offset in end_offsets.items(): state_offsets[partition.partition] = offset state = { 'dump_id': dump_id, 'topic_name': self.topic, 'offsets': state_offsets, 'dump_date': int(time.time()), 'transformer_id': transformer_id } return state def _save_state(self, state): future = self.producer.send(topic=self.dump_state_topic, key=self.topic, value=json.dumps(state)) future.get(timeout=self.timeout_in_sec) logger.info('State saved') def _get_last_state_message(self): beginning_offsets, _, end_offsets = (self._get_offsets( topic=self.dump_state_topic)) if beginning_offsets: offsets, num_messages_available = self._calculate_offsets( beginning_offsets=beginning_offsets, end_offsets=end_offsets, num_messages_to_consume=1) self._set_offsets(offsets) self.consumer.subscribe(self.dump_state_topic) messages = [ json.loads(m.decode()) for k, m in self._get_messages(num_messages_available) ] if messages: last_state_message = max(messages, key=lambda m: m['dump_date']) return last_state_message return None def _get_state(self, dump_id, transformer_id): if self.allow_hotreload: state_message = self._get_last_state_message() if state_message and \ state_message['topic_name'] == self.topic and \ state_message['dump_id'] == dump_id and \ 'transformer_id' in state_message and \ state_message['transformer_id'] == transformer_id: return state_message['offsets'] return None def _reset_offsets(self, dump_offsets): logger.info('Messages already uploaded. Just resetting offsets') partitions = self._get_partitions(self.topic) offsets = {} for partition in partitions: offsets[partition] = dump_offsets[str(partition.partition)] logger.debug('Will reset offsets to <{}>'.format(offsets)) self._set_offsets(offsets) def _load_dump(self, bucket_name, dump_id, download_dir, files, transformer_instance): s3_client = self._get_s3_client() transformer_id = transformer_instance.get_id() state = self._gen_state(dump_id, transformer_id) current_file_number = 0 msg = "Loading messages from file {}/{} to kafka" for file_name, file_size in files: current_file_number += 1 tmp_name = '{}.tmp'.format(path.basename(file_name)) file_path = path.join(download_dir, tmp_name) s3_client.download_file(Bucket=bucket_name, Filename=file_path, Key=file_name, Callback=ProgressPercentage( tmp_name, file_size)) logger.info(msg.format(current_file_number, len(files))) try: table = pq.read_table(file_path) df = table.to_pandas() for raw_row in df.itertuples(): for row in transformer_instance.transform(raw_row): self.producer.send(self.topic, key=row[1], value=row[2]) logger.debug('File <{}> reloaded to kafka'.format(file_path)) self.producer.flush(self.timeout_in_sec) finally: remove(file_path) self._save_state(state) def reload_kafka_server(self, bucket_name, local_dir, dump_id, transformer_class): transformer_instance = self._get_transformer_class(transformer_class) msg = 'Using class=<{}> to transform events before production' logger.info(msg.format(type(transformer_instance))) transformer_id = transformer_instance.get_id() dump_offsets = self._get_state(dump_id, transformer_id) if dump_offsets: self._reset_offsets(dump_offsets=dump_offsets) else: files = self._get_file_names(bucket_name=bucket_name, dump_id=dump_id) self._load_dump(bucket_name=bucket_name, dump_id=dump_id, download_dir=local_dir, files=files, transformer_instance=transformer_instance) logger.info('Reload done!') def __enter__(self): self.open() return self def __exit__(self, exc_type, exc_value, traceback): self.close()
otherReader = csv.DictReader(unzip_file) out = json.dumps( [ row for row in otherReader ], sort_keys=True, indent=4).encode('utf-8') parsed = json.loads(out) for line in parsed: producer.send(topic, value=line) total_lines = total_lines + 1 except: print("failed to send data to kafka server.") end_folder = time.time() total_time = end_folder-start_folder lines_per_second = total_lines / total_time metrics = producer.metrics() metrics_file = open("benchmark.txt", "a") print('Record send rate per second: {metric}'.format(metric=metrics['producer-topic-metrics.{topic}'.format(topic=topic)]['record-send-rate'])) print('Incoming byte rate: {metric}'.format(metric=metrics['producer-node-metrics.node-0']['incoming-byte-rate'])) print('Total records sent per second: {metric}'.format(metric=metrics['producer-topic-metrics.{topic}'.format(topic=topic)]['record-send-rate'])) print('{total_lines} lines parsed and sent to kafka in {total_time}'.format(total_time=total_time, total_lines=total_lines)) print('{lines_per_second} lines per second'.format(lines_per_second=lines_per_second)) try: metrics_file = open("benchmark.txt", "a") metrics_file.write('Record send rate per second: {metric}'.format(metric=metrics['producer-topic-metrics.{topic}'.format(topic=topic)]['record-send-rate'])) metrics_file.write('Incoming byte rate: {metric}'.format(metric=metrics['producer-node-metrics.node-0']['incoming-byte-rate'])) metrics_file.write('Total records sent per second: {metric}'.format(metric=metrics['producer-topic-metrics.{topic}'.format(topic=topic)]['record-send-rate'])) metrics_file.write('{total_lines} lines parsed and sent to kafka in {total_time}'.format(total_time=total_time, total_lines=total_lines)) metrics_file.write('{lines_per_second} lines per second'.format(lines_per_second=lines_per_second))
class Producer: """ 封装kafka-python KafkaProducer """ def __init__(self): pass def __enter__(self): self.cfg = Config().cfg self.producer = KafkaProducer( bootstrap_servers=self.cfg["serList"], # api_version=self.cfg["apiVersion"], api_version_auto_timeout_ms=self.cfg["autoVersionTimeout"], security_protocol=self.cfg["protocol"], sasl_mechanism=self.cfg["mechanism"], sasl_kerberos_service_name=self.cfg["kerverosSerName"], ) return self def __exit__(self, exc_type, exc_val, exc_tb): self.producer.close() def flush(self): """ 调用此方法会使得所有缓存记录变成立即可发送状态.(一般用于send之后, 需要刷新) :return: """ try: self.producer.flush(timeout=TIME_OUT) except KafkaTimeoutError: log.tag_error(KafkaInfo.KafkaProducer, "Flush buffered record failed, TimeOut") raise ActionError(KafkaErr.FlushFailed) def metrics(self): """ 获取producer的性能记录(包含各个kafka broker) :return: """ performance = self.producer.metrics() return performance def partition_set_get(self, topic_name: str): """ 获取topic的所有分区 :param topic_name: :return: set """ return self.producer.partitions_for(topic_name) def send_message(self, topic_name: str, value: bytes, key: str): """ Producer产生数据 :param topic_name: topic where the message will be published :param value: message value :param key: key to associate with the message :return: """ try: result = self.producer.send(topic_name, value=value, key=key.encode("utf-8")).add_errback( self.send_err, topic=topic_name, value=value, key=key) except KafkaTimeoutError: log.tag_warn( KafkaInfo.KafkaProducer, "Kafka send data timeout, topic: %s, key: %s, msg: %s" % (topic_name, key, value.decode("utf-8"))) raise ActionError(KafkaErr.SendDataFailed) return result @staticmethod def send_err(topic: str, value: bytes, key: str): """ producer send data failed callback function :param topic: :param value: :param key: :return: :return: """ log.tag_error( KafkaInfo.KafkaProducer, "Kafka send data failed, topic: %s, " "key: %s msg: %s" % (topic, key, value.decode("utf-8"))) raise ActionError(KafkaErr.SendDataFailed)
from kafka import KafkaProducer producer = KafkaProducer(bootstrap_servers='localhost:9092') while True: print("\n\nType \"quit\" to exit") print("Enter message to be sent:") msg = input() if msg == "quit": print("Exiting") break producer.send('Hello-everyone', msg.encode('utf-8')) print("Sending msg \"{}\"".format(msg)) metric = producer.metrics(raw=False) print(metric) print('\nThe Producer metrics : ') print('Request size max : ') print(metric['producer-metrics']['request-size-max']) print('\nKafka-metrics-count : ') print(metric['kafka-metrics-count']) print(metric.keys()) print("Message sent!")
def dataPub(): cnt = 0 producer = KafkaProducer(bootstrap_servers='10.4.10.239:9092') # 连接kafka devNum = len(producer.partitions_for('testyg')) print(producer.metrics()) st = time.time() for i in range(30): print(i) # if i%2 == 0: img = Image.open(pwd + 'imgs/test.jpg') # else: # img = Image.open('../imgs/army.jpg') # img = clintInf.transformData(img) img = pickle.dumps(img) print(img.__len__()) # producer.send('byzantine', str(msg[i]).encode('utf-8')) # 发送的topic为test # producer.send('result', str(i).encode()) producer.send('testyg', img, str(i).encode()) producer.flush() # time.sleep(1) print('end') log.logSend("INFO " + localhost + " publish 30 msgs!") colDevs = [] for msg in consumer: print(cnt) # print(msg) # if cnt == 0: # st = time.time() cnt += 1 if colDevs.count(msg.key.decode()) == 0: colDevs.append(msg.key.decode()) if cnt == 30: # ed =time.time() # print(ed-st) ed = time.time() consumer.close() costTime = ed - st log.logSend("INFO" + localhost + "'s datas handle done, costs time " + costTime) taskInfo = { 'name': 'dataOffload', 'type': 'classification', 'startDevice': localhost, 'dataNum': 30, 'devNum': devNum, 'colDevs': colDevs, 'latency': ed - st } js = json.dumps(taskInfo) conn.lpush("kafkaTasks", js) producer.close()
import pickle import redis import json from MyConsumer import * consumer = KafkaConsumer('result', group_id="test_group_1", bootstrap_servers=['10.4.10.239:9092']) cnt = 0 producer = KafkaProducer(bootstrap_servers='10.4.10.239:9092') # 连接kafka devNum = len(producer.partitions_for('testyg')) print(devNum) print(producer.metrics()) clintInf = ClientInf(0, 18) # msg = [1,2,3,4] st = time.time() for i in range(30): print(i) # if i%2 == 0: img = Image.open('../imgs/test.jpg') # else: # img = Image.open('../imgs/army.jpg') # img = clintInf.transformData(img) img = pickle.dumps(img) print(img.__len__()) # producer.send('byzantine', str(msg[i]).encode('utf-8')) # 发送的topic为test
from kafka import KafkaProducer from kafka.errors import KafkaError import json import sys servid=sys.argv[1] print('arg',servid) producer = KafkaProducer(bootstrap_servers=['kafka:port'], security_protocol="SSL", ssl_cafile="ca.pem", ssl_certfile="service.cert", ssl_keyfile="service.key", value_serializer=lambda m: json.dumps(m).encode('ascii')) data = producer.send('shrtest', {'ServiceId': servid,'details':{'status': 'active','stdate': '12-01-2019','enddate': '01-01-9999'}}) producer.metrics() producer.flush() try: record_metadata = data.get(timeout=1000) print('response Partition',record_metadata.partition) print ('response offset',record_metadata.offset) except KafkaError: log.exception() #Exception
for f in foreFiles: with open(f, 'r') as fp: foreDat = json.load(fp) for city, dat in foreDat.items(): topic = f"wf-{city}" outdat = mapForeWeatherFlat(dat) if outdat is not None: if outdat['dt'] != 'NA': producer.send(topic, outdat) print(f"forecast weather city={city}", "time=", datetime.fromtimestamp(outdat['dt'])) else: print(f"no dat reported for city={city}...skipping") else: print(f"Could not report for city={city}...skipping") else: # implemenation for api currently missing pw_log('processing via api source is currently not implemented') pass adm.close() consumer.close() producer.flush() pi_log("kafkaProducerMetrics:") pi_log("performance:" + str(json.dumps(producer.metrics(), indent=4))) pi_log("successful completion") producer.close()