示例#1
0
 def consume(self):
     topics = [anu_dp_wf_aligner_in_topic]
     consumer = self.instantiate(topics)
     service = AlignmentService()
     util = AlignmentUtils()
     rand_str = ''.join(
         random.choice(string.ascii_letters) for i in range(4))
     prefix = "Align-WFM-Consumer(" + rand_str + ")"
     log_info(prefix + " running.......", None)
     while True:
         #thread_count = 0
         for msg in consumer:
             data = {}
             try:
                 data = msg.value
                 if data:
                     log_info(
                         prefix + " | Received on Topic: " + msg.topic +
                         " | Partition: " + str(msg.partition), data)
                     service.wf_process(data)
                 break
             except Exception as e:
                 log_exception("Exception while consuming: " + str(e), data,
                               e)
                 util.error_handler("ALIGNER_CONSUMER_ERROR",
                                    "Exception while consuming", data, True)
                 break
示例#2
0
 def consume(self):
     topics = [align_job_topic]
     consumer = self.instantiate(topics)
     service = AlignmentService()
     util = AlignmentUtils()
     rand_str = ''.join(random.choice(string.ascii_letters) for i in range(4))
     prefix = "Align-Consumer(" + rand_str + ")"
     log_info(prefix + " running.......", None)
     while True:
         thread_count = 0
         for msg in consumer:
             data = {}
             try:
                 data = msg.value
                 if data:
                     log_info(prefix + " | Received on Topic: " + msg.topic + " | Partition: " + str(msg.partition), data)
                     thread_name = prefix + "--" + "thread--" + str(thread_count)
                     align_cons_thread = threading.Thread(target=service.process, args=(data, False), name=thread_name)
                     align_cons_thread.start()
                     log_info(prefix + " | Forked thread: " + thread_name, data)
                     thread_count += 1
             except Exception as e:
                 log_exception("Exception while consuming: " + str(e), data, e)
                 util.error_handler("ALIGNER_CONSUMER_ERROR", "Exception while consuming: " + str(e), None, False)
                 break
示例#3
0
 def push_to_queue(self, object_in, topic):
     producer = self.instantiate()
     util = AlignmentUtils()
     try:
         if object_in:
             producer.send(topic, value=object_in)
             log_info("Pushed to topic: " + topic, object_in)
         producer.flush()
         return None
     except Exception as e:
         log_exception("Exception while producing: " + str(e), None, e)
         return util.error_handler("ALIGNER_PRODUCER_ERROR",
                                   "Exception while producing: " + str(e),
                                   None, False)