Beispiel #1
0
 def consume(self):
     topics = [anu_dp_wf_aligner_in_topic]
     consumer = self.instantiate(topics)
     service = AlignmentService()
     util = AlignmentUtils()
     rand_str = ''.join(
         random.choice(string.ascii_letters) for i in range(4))
     prefix = "Align-WFM-Consumer(" + rand_str + ")"
     log_info(prefix + " running.......", None)
     while True:
         #thread_count = 0
         for msg in consumer:
             data = {}
             try:
                 data = msg.value
                 if data:
                     log_info(
                         prefix + " | Received on Topic: " + msg.topic +
                         " | Partition: " + str(msg.partition), data)
                     service.wf_process(data)
                 break
             except Exception as e:
                 log_exception("Exception while consuming: " + str(e), data,
                               e)
                 util.error_handler("ALIGNER_CONSUMER_ERROR",
                                    "Exception while consuming", data, True)
                 break
Beispiel #2
0
 def consume(self):
     topics = [align_job_topic]
     consumer = self.instantiate(topics)
     service = AlignmentService()
     util = AlignmentUtils()
     rand_str = ''.join(random.choice(string.ascii_letters) for i in range(4))
     prefix = "Align-Consumer(" + rand_str + ")"
     log_info(prefix + " running.......", None)
     while True:
         thread_count = 0
         for msg in consumer:
             data = {}
             try:
                 data = msg.value
                 if data:
                     log_info(prefix + " | Received on Topic: " + msg.topic + " | Partition: " + str(msg.partition), data)
                     thread_name = prefix + "--" + "thread--" + str(thread_count)
                     align_cons_thread = threading.Thread(target=service.process, args=(data, False), name=thread_name)
                     align_cons_thread.start()
                     log_info(prefix + " | Forked thread: " + thread_name, data)
                     thread_count += 1
             except Exception as e:
                 log_exception("Exception while consuming: " + str(e), data, e)
                 util.error_handler("ALIGNER_CONSUMER_ERROR", "Exception while consuming: " + str(e), None, False)
                 break
Beispiel #3
0
 def push_to_queue(self, object_in, topic):
     producer = self.instantiate()
     util = AlignmentUtils()
     try:
         if object_in:
             producer.send(topic, value=object_in)
             log_info("Pushed to topic: " + topic, object_in)
         producer.flush()
         return None
     except Exception as e:
         log_exception("Exception while producing: " + str(e), None, e)
         return util.error_handler("ALIGNER_PRODUCER_ERROR",
                                   "Exception while producing: " + str(e),
                                   None, False)
Beispiel #4
0
#!/bin/python
import logging

from configs.alignerconfig import anu_dp_wf_aligner_out_topic
from utilities.alignmentutils import AlignmentUtils
from repository.alignmentrepository import AlignmentRepository
from validator.alignmentvalidator import AlignmentValidator
from kafkawrapper.alignmentproducer import Producer

log = logging.getLogger('file')

alignmentutils = AlignmentUtils()
repo = AlignmentRepository()
producer = Producer()
util = AlignmentUtils()
validator = AlignmentValidator()


class AlignWflowService:
    def __init__(self):
        pass

    # Wrapper to build response compatibile with the anuvaad etl wf manager.
    def getwfresponse(self, result, object_in):
        wfresponse = {
            "taskID": object_in["taskID"],
            "jobID": object_in["jobID"],
            "input": result["input"],
            "output": result["output"],
            "workflowCode": object_in["workflowCode"],
            "stepOrder": object_in["stepOrder"],
Beispiel #5
0
#!/bin/python
from utilities.alignmentutils import AlignmentUtils

util = AlignmentUtils()


class AlignmentValidator:
    def __init__(self):
        pass

    # Validator that validates the input request for initiating the alignment job
    def validate_input(self, data):
        if 'source' not in data.keys():
            return util.error_handler("SOURCE_NOT_FOUND",
                                      "Details of the source not available",
                                      None, False)
        else:
            source = data["source"]
            if 'filepath' not in source.keys():
                return util.error_handler(
                    "SOURCE_FILE_NOT_FOUND",
                    "Details of the source file not available", None, False)
            elif 'locale' not in source.keys():
                return util.error_handler(
                    "SOURCE_LOCALE_NOT_FOUND",
                    "Details of the source locale not available", None, False)
        if 'target' not in data.keys():
            return util.error_handler("TARGET_NOT_FOUND",
                                      "Details of the target not available",
                                      None, False)
        else: