def consume(self):
topics = [anu_dp_wf_aligner_in_topic]
consumer = self.instantiate(topics)
service = AlignmentService()
util = AlignmentUtils()
rand_str = ''.join(
random.choice(string.ascii_letters) for i in range(4))
prefix = "Align-WFM-Consumer(" + rand_str + ")"
log_info(prefix + " running.......", None)
while True:
#thread_count = 0
for msg in consumer:
data = {}
try:
data = msg.value
if data:
log_info(
prefix + " | Received on Topic: " + msg.topic +
" | Partition: " + str(msg.partition), data)
service.wf_process(data)
break
except Exception as e:
log_exception("Exception while consuming: " + str(e), data,
e)
util.error_handler("ALIGNER_CONSUMER_ERROR",
"Exception while consuming", data, True)
break
def consume(self):
topics = [align_job_topic]
consumer = self.instantiate(topics)
service = AlignmentService()
util = AlignmentUtils()
rand_str = ''.join(random.choice(string.ascii_letters) for i in range(4))
prefix = "Align-Consumer(" + rand_str + ")"
log_info(prefix + " running.......", None)
while True:
thread_count = 0
for msg in consumer:
data = {}
try:
data = msg.value
if data:
log_info(prefix + " | Received on Topic: " + msg.topic + " | Partition: " + str(msg.partition), data)
thread_name = prefix + "--" + "thread--" + str(thread_count)
align_cons_thread = threading.Thread(target=service.process, args=(data, False), name=thread_name)
align_cons_thread.start()
log_info(prefix + " | Forked thread: " + thread_name, data)
thread_count += 1
except Exception as e:
log_exception("Exception while consuming: " + str(e), data, e)
util.error_handler("ALIGNER_CONSUMER_ERROR", "Exception while consuming: " + str(e), None, False)
break
def push_to_queue(self, object_in, topic):
producer = self.instantiate()
util = AlignmentUtils()
try:
if object_in:
producer.send(topic, value=object_in)
log_info("Pushed to topic: " + topic, object_in)
producer.flush()
return None
except Exception as e:
log_exception("Exception while producing: " + str(e), None, e)
return util.error_handler("ALIGNER_PRODUCER_ERROR",
"Exception while producing: " + str(e),
None, False)
#!/bin/python
import logging
from configs.alignerconfig import anu_dp_wf_aligner_out_topic
from utilities.alignmentutils import AlignmentUtils
from repository.alignmentrepository import AlignmentRepository
from validator.alignmentvalidator import AlignmentValidator
from kafkawrapper.alignmentproducer import Producer
log = logging.getLogger('file')
alignmentutils = AlignmentUtils()
repo = AlignmentRepository()
producer = Producer()
util = AlignmentUtils()
validator = AlignmentValidator()
class AlignWflowService:
def __init__(self):
pass
# Wrapper to build response compatibile with the anuvaad etl wf manager.
def getwfresponse(self, result, object_in):
wfresponse = {
"taskID": object_in["taskID"],
"jobID": object_in["jobID"],
"input": result["input"],
"output": result["output"],
"workflowCode": object_in["workflowCode"],
"stepOrder": object_in["stepOrder"],
#!/bin/python
from utilities.alignmentutils import AlignmentUtils
util = AlignmentUtils()
class AlignmentValidator:
def __init__(self):
pass
# Validator that validates the input request for initiating the alignment job
def validate_input(self, data):
if 'source' not in data.keys():
return util.error_handler("SOURCE_NOT_FOUND",
"Details of the source not available",
None, False)
else:
source = data["source"]
if 'filepath' not in source.keys():
return util.error_handler(
"SOURCE_FILE_NOT_FOUND",
"Details of the source file not available", None, False)
elif 'locale' not in source.keys():
return util.error_handler(
"SOURCE_LOCALE_NOT_FOUND",
"Details of the source locale not available", None, False)
if 'target' not in data.keys():
return util.error_handler("TARGET_NOT_FOUND",
"Details of the target not available",
None, False)
else: