def __init_producer(self, topics):
'''
初始化生产者
:return:
'''
try:
for topic in topics:
producer = KafkaProducer(config.kafka_hosts, self._logger)
if producer.init(topic):
self._producer_queue[topic] = Queue()
th = Thread(target=producer.produce,
args=(self._producer_queue[topic], ))
th.start()
self._logger.debug('成功初始化生产者线程')
except Exception as e:
self._logger.error('[MAINSERVER]初始化生产者错误：{}'.format(e))
raise e
def processRecords(nb_records, topicname, keyname,docsToSend):
print("Producer to the topic " + topicname)
try:
producer = KafkaProducer(kafka_brokers = ebc.getBrokerEndPoints(),
kafka_apikey = ebc.getEndPointAPIKey(),
kafka_cacert = ebc.getKafkaCertificate(),
topic_name = topicname)
producer.prepare(groupID= GROUPID)
a = nb_records / len(docsToSend)
b = nb_records % len(docsToSend)
for i in range(0,int(a)):
for doc in docsToSend:
doc['timestamp'] = time.time()
print("sending -> " + str(doc))
producer.publishEvent(doc,keyname)
for i in range(0,int(b)):
docsToSend[i]['timestamp'] = time.time()
print("sending -> " + str(docsToSend[i]))
producer.publishEvent(docsToSend[i],keyname)
except KeyboardInterrupt:
input('Press enter to continue')
print("Thank you")
def processRecords(nb_records, topicname):
print("Produce " + str(nb_records) + " messages to the topic " + topicname)
try:
producer = KafkaProducer(kafka_brokers=ebc.getBrokerEndPoints(),
kafka_apikey=ebc.getEndPointAPIKey(),
kafka_cacert=ebc.getKafkaCertificate(),
topic_name=topicname)
producer.prepare(groupID=GROUPID)
for i in range(0, nb_records):
docToSend = {}
docToSend['storeName'] = STORES[random.randint(0, len(STORES) - 1)]
docToSend['sku'] = 'Item_' + str(random.randint(0, 9))
docToSend['quantity'] = random.randint(0, 20)
docToSend['price'] = random.randint(20, 200)
docToSend['id'] = i
dateTimeObj = datetime.now()
timestampStr = dateTimeObj.strftime("%d-%b-%Y %H:%M:%S")
docToSend['timestamp'] = timestampStr
print("sending -> " + str(docToSend))
producer.publishEvent(docToSend, KEYNAME)
except KeyboardInterrupt:
input('Press enter to continue')
print("Thank you")
import logging
from kafka.KafkaProducer import KafkaProducer
import kafka.eventStreamsConfig as config
'''
Product some item sold event
'''
if __name__ == '__main__':
print("Start Item Sold Event Producer")
logging.basicConfig(level=logging.INFO)
producer = KafkaProducer(kafka_brokers = config.KAFKA_BROKERS,
kafka_user = config.KAFKA_USER,
kafka_pwd = config.KAFKA_PASSWORD,
kafka_cacert = config.KAFKA_CERT_PATH,
kafka_sasl_mechanism=config.KAFKA_SASL_MECHANISM,
topic_name = "items")
producer.prepare("ItemSoldProducer-1")
item = {'id': 0,'storeName': "Store-3",'sku': 'Item-1', 'type': 'RESTOCK', 'quantity': 5}
producer.publishEvent(item,"storeName")
item = {'id': 1,'storeName': "Store-3",'sku': 'Item-1', 'type': 'SALE', 'quantity': 2, 'price': 10.0}
producer.publishEvent(item,"storeName")
item = {'id': 2,'storeName': "Store-4",'sku': 'Item-2', 'type': 'RESTOCK', 'quantity': 20 }
producer.publishEvent(item,"storeName")
item = {'id': 3,'storeName': "Store-4",'sku': 'Item-3', 'type': 'RESTOCK', 'quantity': 30 }
producer.publishEvent(item,"storeName")
item = {'id': 4,'storeName': "Store-4",'sku': 'Item-3', 'type': 'SALE', 'quantity': 5, 'price': 6.0 }
producer.publishEvent(item,"storeName")
def processRecords(nb_records, topicname):
print("Produce " + str(nb_records) + " messages to the topic " + topicname)
try:
producer = KafkaProducer(
kafka_brokers=config.KAFKA_BROKERS,
kafka_user=config.KAFKA_USER,
kafka_pwd=config.KAFKA_PASSWORD,
kafka_cacert=config.KAFKA_CERT_PATH,
kafka_sasl_mechanism=config.KAFKA_SASL_MECHANISM,
topic_name=topicname)
producer.prepare(groupID=GROUPID)
for i in range(0, nb_records):
docToSend = {
"schema": {
"type":
"struct",
"fields": [{
"type": "string",
"optional": False,
"field": "storeName"
}, {
"type": "string",
"optional": False,
"field": "sku"
}, {
"type": "decimal",
"optional": False,
"field": "id"
}, {
"type": "decimal",
"optional": True,
"field": "quantity"
}, {
"type": "decimal",
"optional": True,
"field": "price"
}, {
"type": "string",
"optional": True,
"field": "timestamp"
}],
"optional":
False,
"name":
"Inventory"
},
}
payload = {}
payload["storeName"] = STORES[random.randint(0, len(STORES) - 1)]
payload["sku"] = "Item_" + str(random.randint(0, 9))
payload["quantity"] = random.randint(0, 20)
payload["price"] = random.randint(20, 200)
payload["id"] = i
dateTimeObj = datetime.now()
timestampStr = dateTimeObj.strftime("%d-%b-%Y %H:%M:%S")
payload["timestamp"] = timestampStr
docToSend["payload"] = payload
print("sending -> " + str(docToSend))
producer.publishEvent(docToSend, payload["storeName"])
except KeyboardInterrupt:
input('Press enter to continue')
print("Thank you")
import logging
from kafka.KafkaProducer import KafkaProducer
import kafka.KafkaConfig as config
'''
Product some shipment plan
'''
if __name__ == '__main__':
print("Start Shipment plan Producer")
logging.basicConfig(level=logging.INFO)
producer = KafkaProducer(kafka_brokers=config.KAFKA_BROKERS,
kafka_user=config.KAFKA_USER,
kafka_pwd=config.KAFKA_PASSWORD,
kafka_cacert=config.KAFKA_CERT_PATH,
kafka_sasl_mechanism=config.KAFKA_SASL_MECHANISM,
topic_name="vaccine_shipment_plans")
producer.prepare("ShipmentPlanProducer")
shipmentPlan = {
'planID': 'plan01',
'orderID': '1',
'from': "Beerse, Belgium",
'departureDate': '2021-01-21',
'type': 'Delivery',
'to': 'Paris, France',
'arrivalDate': '2021-01-21',
'quantity': 150,
'reefers': 3,
'cost': 50
}
producer.publishEvent(shipmentPlan, "planID")