def __init__(self,
producer_name,
value_schema,
groupID='KafkaAvroProducer'):
# Consumer name for logging purposes
self.logging_prefix = '[' + producer_name + '][KafkaAvroProducer]'
# Schema Registry configuration
self.schema_registry_conf = EventBackboneConfig.getSchemaRegistryConf()
# Schema Registry Client
self.schema_registry_client = SchemaRegistryClient(
self.schema_registry_conf)
# String Serializer for the key
self.key_serializer = StringSerializer('utf_8')
# Avro Serializer for the value
self.value_serializer = AvroSerializer(value_schema,
self.schema_registry_client)
# Get the producer configuration
self.producer_conf = EventBackboneConfig.getProducerConfiguration(
groupID, self.key_serializer, self.value_serializer)
EventBackboneConfig.printProducerConfiguration(
self.logging_prefix, self.producer_conf,
self.schema_registry_conf['url'])
# Create the producer
self.producer = SerializingProducer(self.producer_conf)
def produceData(self):
# self.produceInventoryDataFromFile(EventBackboneConfig.getInventoryTopicName())
# self.produceReeferDataFromFile(EventBackboneConfig.getReeferTopicName())
# self.produceTransportationDataFromFile(EventBackboneConfig.getTransportationTopicName())
self.produceInventoryData(EventBackboneConfig.getInventoryTopicName())
self.produceReeferData(EventBackboneConfig.getReeferTopicName())
self.produceTransportationData(
EventBackboneConfig.getTransportationTopicName())
def __init__(self):
# logging.info("[InventoryConsumer] - Initializing the consumer")
print("[InventoryConsumer] - Initializing the consumer")
self.cloudEvent_schema = avroUtils.getCloudEventSchema()
self.store = InventoryDataStore()
self.kafkaconsumer = KafkaAvroConsumer(
'InventoryConsumer', json.dumps(self.cloudEvent_schema.to_json()),
EventBackboneConfig.getInventoryTopicName(),
EventBackboneConfig.getConsumerGroup(), AUTO_COMMIT)
def produceShipmentPlan(self, sol_json):
print('\n[OrderConsumer] - producing shipment plan')
shipments = json.loads(sol_json['Shipments'])
shipments_clean = []
if len(shipments) > 0:
for shipment in shipments:
shipments_clean.append(
{x.replace(' ', ''): v
for x, v in shipment.items()})
event_json = {}
event_json[
'type'] = "ibm.gse.eda.vaccine.orderoptimizer.VaccineOrderCloudEvent"
event_json['specversion'] = "1.0"
event_json['source'] = "Vaccine Order Optimizer engine"
event_json['id'] = str(uuid.uuid4())
event_json['time'] = datetime.datetime.now(
datetime.timezone.utc).isoformat()
event_json[
'dataschema'] = "https://raw.githubusercontent.com/ibm-cloud-architecture/vaccine-order-optimizer/master/data/avro/schemas/shipment_plan.avsc"
event_json['datacontenttype'] = "application/json"
event_json['data'] = {"Shipments": shipments_clean}
if self.debugOptimization:
print('[OrderConsumer] - Shipment Plan event to be produced:')
print(event_json)
self.kafkaproducer.publishEvent(
event_json['id'], event_json,
EventBackboneConfig.getShipmentPlanTopicName())
else:
print('[ERROR] - There is no shipment plan.')
def __init__(self, inventoryStore, reeferStore, transportationStore):
print("[OrderConsumer] - Initializing the consumer")
self.debugOptimization = self.debugOptimization()
self.orderStore = OrderDataStore.getInstance()
self.reeferStore = reeferStore
self.inventoryStore = inventoryStore
self.transporationStore = transportationStore
self.kafkaconsumer = KafkaAvroCDCConsumer(
'OrderConsumer', EventBackboneConfig.getOrderTopicName(),
EventBackboneConfig.getConsumerGroup(), AUTO_COMMIT)
# Avro data schemas location
self.schemas_location = "/app/data/avro/schemas/"
# CloudEvent Schema
self.cloudEvent_schema = avroUtils.getCloudEventSchema()
# Build the Kafka Avro Producers
self.kafkaproducer = KafkaAvroProducer(
"OrderConsumer", json.dumps(self.cloudEvent_schema.to_json()),
"VOO-ShipmentPlan")
def __init__(self, consumer_name, value_schema, topic_name = "kafka-avro-producer", groupID = 'KafkaAvroConsumer', autocommit = True):
# Consumer name for logging purposes
self.logging_prefix = '['+ consumer_name + '][KafkaAvroConsumer]'
# Schema Registry configuration
self.schema_registry_conf = EventBackboneConfig.getSchemaRegistryConf()
# Schema Registry Client
self.schema_registry_client = SchemaRegistryClient(self.schema_registry_conf)
# Key Deserializer
self.key_deserializer = StringDeserializer('utf_8')
# Get Schema for the value
self.schema_id_value = self.schema_registry_client.get_latest_version(topic_name + "-value").schema_id
# print('The Schema ID for the value is: {}'.format(self.schema_id_value))
self.value_schema = self.schema_registry_client.get_schema(self.schema_id_value).schema_str
print(self.logging_prefix + ' - Value Subject: {}'.format(topic_name))
print(self.logging_prefix + ' - Value Schema:')
print(self.logging_prefix + ' - -------------\n')
print(self.logging_prefix + ' - ' + self.value_schema + '\n')
# Value Deserializer
# Presenting the schema to the Avro Deserializer is needed at the moment. In the future it might change
# https://github.com/confluentinc/confluent-kafka-python/issues/834
self.value_deserializer = AvroDeserializer(self.value_schema,self.schema_registry_client)
# Get the consumer configuration
self.consumer_conf = EventBackboneConfig.getConsumerConfiguration(groupID, autocommit,
self.key_deserializer,
self.value_deserializer)
# Create the consumer
self.consumer = DeserializingConsumer(self.consumer_conf)
# Print consumer configuration
EventBackboneConfig.printConsumerConfiguration(self.logging_prefix,self.consumer_conf,self.schema_registry_conf['url'])
# Subscribe to the topic
self.consumer.subscribe([topic_name])