def consume(self):
topics = [anu_dp_wf_aligner_in_topic]
consumer = self.instantiate(topics)
service = AlignmentService()
util = AlignmentUtils()
rand_str = ''.join(
random.choice(string.ascii_letters) for i in range(4))
prefix = "Align-WFM-Consumer(" + rand_str + ")"
log_info(prefix + " running.......", None)
while True:
#thread_count = 0
for msg in consumer:
data = {}
try:
data = msg.value
if data:
log_info(
prefix + " | Received on Topic: " + msg.topic +
" | Partition: " + str(msg.partition), data)
service.wf_process(data)
break
except Exception as e:
log_exception("Exception while consuming: " + str(e), data,
e)
util.error_handler("ALIGNER_CONSUMER_ERROR",
"Exception while consuming", data, True)
break
def consume(self):
topics = [align_job_topic]
consumer = self.instantiate(topics)
service = AlignmentService()
util = AlignmentUtils()
rand_str = ''.join(random.choice(string.ascii_letters) for i in range(4))
prefix = "Align-Consumer(" + rand_str + ")"
log_info(prefix + " running.......", None)
while True:
thread_count = 0
for msg in consumer:
data = {}
try:
data = msg.value
if data:
log_info(prefix + " | Received on Topic: " + msg.topic + " | Partition: " + str(msg.partition), data)
thread_name = prefix + "--" + "thread--" + str(thread_count)
align_cons_thread = threading.Thread(target=service.process, args=(data, False), name=thread_name)
align_cons_thread.start()
log_info(prefix + " | Forked thread: " + thread_name, data)
thread_count += 1
except Exception as e:
log_exception("Exception while consuming: " + str(e), data, e)
util.error_handler("ALIGNER_CONSUMER_ERROR", "Exception while consuming: " + str(e), None, False)
break
def push_to_queue(self, object_in, topic):
producer = self.instantiate()
util = AlignmentUtils()
try:
if object_in:
producer.send(topic, value=object_in)
log_info("Pushed to topic: " + topic, object_in)
producer.flush()
return None
except Exception as e:
log_exception("Exception while producing: " + str(e), None, e)
return util.error_handler("ALIGNER_PRODUCER_ERROR",
"Exception while producing: " + str(e),
None, False)