def instantiate_producer(**kwargs):
conf = utils.retrieve(kwargs['--path_to_config'], kwargs['--section'])
producer = Producer(config=conf,
key_serializer=AvroStringKeySerializer(schema_registry_url="http://localhost:8081"),
value_serializer=AvroSerializer(schema_registry_url="http://localhost:8081",
subject_name_strategy=SubjectNameStrategy.TopicRecordNameStrategy)
)
def produce(producer: Producer, record: AbstractRecord, **kwargs):
for key, value in
producer.produce(
config = utils.retrieve(kwargs['--path_to_config'], kwargs['--section']),
topic_name = kwargs['--topic'],
num_partitions = kwargs['--partitions'],
key=kwargs['--key'],
value={**record}
)
def __init__(self, **kwargs) -> None:
self.config = utils.retrieve(kwargs['--path_to_config'], kwargs['--section'])
self.topic_name = kwargs['--topic']
self.num_partitions = kwargs['--partitions']
self.key = kwargs['--key']
self.producer = avro.AvroProducer(self.config)
self.source_file = kwargs['--source_file']
self.key_schema = kwargs['--key_schema']
self.value_schema = kwargs['--value_schema']
self.file_extension = kwargs['--file_extension']
def consume(topic):
print("Consuming records from topic {} with group {}.".format(topic, conf['group_id']))
conf = retrieve("confs/finwire.yaml", "sec")["consumer"]
key_schema=avro.loads("""{"namespace":"ingestd.schema_registry.keys","type":"record","name":"finwireSECkey",
"fields":[{"name":PTS","type":"string"},{"name":"rec_type", "type":"string"},{"name":"
""")
consumer = AvroLazyConsumer(**conf, schema_registry="http://localhost:8081",
reader_key_schema=
reader_value_schema=
)
def io_stream(file_path):
def produce(producer: Producer, record: AbstractRecord, **kwargs):
for key, value in
producer.produce(
config = utils.retrieve(kwargs['--path_to_config'], kwargs['--section']),
topic_name = kwargs['--topic'],
num_partitions = kwargs['--partitions'],
key=kwargs['--key'],
value={**record}
)
class Producer()
def __init__(self, **kwargs) -> None:
self.config = utils.retrieve(kwargs['--path_to_config'], kwargs['--section'])
self.topic_name = kwargs['--topic']
self.num_partitions = kwargs['--partitions']
self.key = kwargs['--key']
self.producer = avro.AvroProducer(self.config)
self.source_file = kwargs['--source_file']
self.key_schema = kwargs['--key_schema']
self.value_schema = kwargs['--value_schema']
self.file_extension = kwargs['--file_extension']
def produce(self, **kwargs):
for rec_type, record_value in utils.generate_payload(self.source_file):
path_to_key_schema = Path('ingestd/kafka/schemas') / self.key_schema
path_to_value_schema = Path('ingestd/kafka/schemas') / self.value_schema
record_key = avro.load( path_to_key_schema.as_posix() )
record_schema = avro.load( path_to_value_schema.as_posix())
self.producer.produce(key=record_key, value=record_value,
key_schema=record_key,
value_schema=record_value)
p = avro.AvroProducer(config=utils.retrieve(file_path='confs/finwire.yaml', section='producer'))
# Route message production based on doc type
for file_path in Path('/data/').glob('FINWIRE*[1234]'):
for rec_type, record_value in utils.generate_payload(file_path.__str__()):
topic_subject = "finwire{0}".format(rec_type)
record_schema = avro.load((schema_dir / ('finwire{0}.avsc'.format(rec_type.lower()))).as_posix())
try:
p.produce(topic=topic_subject,
value=record_value,
value_schema=record_schema,
callback=utils.ackback)
p.poll(.25)
except RuntimeError as e:
print("Runtime Error: {}".format(e))
print("Completed file {}".format(file_path.as_posix()))
p.flush()