def consumer_validator():
try:
consumer_class = Consumer(config.input_topic, config.bootstrap_server)
consumer = consumer_class.consumer_instantiate()
log_info("consumer_validator --- consumer running -----", None)
return consumer
except:
log_exception("consumer_validator : error in kafka opertation while listening to consumer on topic %s"%(config.input_topic), None, None)
raise KafkaConsumerError(400, "Can not connect to consumer.")
def process_merger_kf():
file_ops = FileOperation()
DOWNLOAD_FOLDER = file_ops.create_file_download_dir(config.download_folder)
task_id = str("BM-" + str(time.time()).replace('.', ''))
task_starttime = str(time.time()).replace('.', '')
# instatiation of consumer for respective topic
try:
consumer_class = Consumer(config.input_topic, config.bootstrap_server)
consumer = consumer_class.consumer_instantiate()
log_info("process_merger_kf", "trying to receive value from consumer",
None)
thread_instance = 0
for msg in consumer:
try:
data = msg.value
task_id = str("BM-" + str(time.time()).replace('.', ''))
task_starttime = str(time.time()).replace('.', '')
input_files, workflow_id, jobid, tool_name, step_order = file_ops.json_input_format(
data)
log_info("process_merger_kf", "kafka request arrived ", jobid)
response_gen = Response(data, DOWNLOAD_FOLDER)
t1 = threading.Thread(
target=response_gen.multi_thred_block_merger,
args=(task_id, task_starttime, jobid),
name='BM-thread-' + str(thread_instance))
t1.start()
thread_instance += 1
log_info("multithread", "block-merger running on multithread",
None)
'''
file_value_response = response_gen.workflow_response(task_id, task_starttime)
if "errorID" not in file_value_response.keys():
producer = Producer()
producer.push_data_to_queue(config.output_topic, file_value_response, jobid, task_id)
else:
log_info("process_merger_kf", "error send to error handler", jobid)'''
except Exception as e:
log_exception("process_pdf_kf",
"exception while consuming the records", jobid,
e)
except KafkaConsumerError as e:
response_custom = CustomResponse(Status.ERR_STATUS.value, None, None)
response_custom.status_code['message'] = str(e)
file_ops.error_handler(response_custom.status_code,
"KAFKA_CONSUMER_ERROR", True)
log_exception("process_pdf_kf", "Consumer didn't instantiate", None, e)
except KafkaProducerError as e:
response_custom = e.code
response_custom['message'] = e.message
file_ops.error_handler(response_custom, "KAFKA_PRODUCER_ERROR", True)
log_exception("process_pdf_kf",
"response send to topic %s" % (config.output_topic),
response_custom['jobID'], e)