def consume(self): topics = [anu_dp_wf_aligner_in_topic] consumer = self.instantiate(topics) service = AlignmentService() util = AlignmentUtils() rand_str = ''.join( random.choice(string.ascii_letters) for i in range(4)) prefix = "Align-WFM-Consumer(" + rand_str + ")" log_info(prefix + " running.......", None) while True: #thread_count = 0 for msg in consumer: data = {} try: data = msg.value if data: log_info( prefix + " | Received on Topic: " + msg.topic + " | Partition: " + str(msg.partition), data) service.wf_process(data) break except Exception as e: log_exception("Exception while consuming: " + str(e), data, e) util.error_handler("ALIGNER_CONSUMER_ERROR", "Exception while consuming", data, True) break
def consume(self): topics = [align_job_topic] consumer = self.instantiate(topics) service = AlignmentService() util = AlignmentUtils() rand_str = ''.join(random.choice(string.ascii_letters) for i in range(4)) prefix = "Align-Consumer(" + rand_str + ")" log_info(prefix + " running.......", None) while True: thread_count = 0 for msg in consumer: data = {} try: data = msg.value if data: log_info(prefix + " | Received on Topic: " + msg.topic + " | Partition: " + str(msg.partition), data) thread_name = prefix + "--" + "thread--" + str(thread_count) align_cons_thread = threading.Thread(target=service.process, args=(data, False), name=thread_name) align_cons_thread.start() log_info(prefix + " | Forked thread: " + thread_name, data) thread_count += 1 except Exception as e: log_exception("Exception while consuming: " + str(e), data, e) util.error_handler("ALIGNER_CONSUMER_ERROR", "Exception while consuming: " + str(e), None, False) break
def push_to_queue(self, object_in, topic): producer = self.instantiate() util = AlignmentUtils() try: if object_in: producer.send(topic, value=object_in) log_info("Pushed to topic: " + topic, object_in) producer.flush() return None except Exception as e: log_exception("Exception while producing: " + str(e), None, e) return util.error_handler("ALIGNER_PRODUCER_ERROR", "Exception while producing: " + str(e), None, False)
#!/bin/python import logging from configs.alignerconfig import anu_dp_wf_aligner_out_topic from utilities.alignmentutils import AlignmentUtils from repository.alignmentrepository import AlignmentRepository from validator.alignmentvalidator import AlignmentValidator from kafkawrapper.alignmentproducer import Producer log = logging.getLogger('file') alignmentutils = AlignmentUtils() repo = AlignmentRepository() producer = Producer() util = AlignmentUtils() validator = AlignmentValidator() class AlignWflowService: def __init__(self): pass # Wrapper to build response compatibile with the anuvaad etl wf manager. def getwfresponse(self, result, object_in): wfresponse = { "taskID": object_in["taskID"], "jobID": object_in["jobID"], "input": result["input"], "output": result["output"], "workflowCode": object_in["workflowCode"], "stepOrder": object_in["stepOrder"],
#!/bin/python from utilities.alignmentutils import AlignmentUtils util = AlignmentUtils() class AlignmentValidator: def __init__(self): pass # Validator that validates the input request for initiating the alignment job def validate_input(self, data): if 'source' not in data.keys(): return util.error_handler("SOURCE_NOT_FOUND", "Details of the source not available", None, False) else: source = data["source"] if 'filepath' not in source.keys(): return util.error_handler( "SOURCE_FILE_NOT_FOUND", "Details of the source file not available", None, False) elif 'locale' not in source.keys(): return util.error_handler( "SOURCE_LOCALE_NOT_FOUND", "Details of the source locale not available", None, False) if 'target' not in data.keys(): return util.error_handler("TARGET_NOT_FOUND", "Details of the target not available", None, False) else: