コード例 #1
0
ファイル: meta.py プロジェクト: ddinsight/dd-streamworks
    def __processEventQueue(self):
        while True:
            try:
                msg = json.loads(self.evtQueue.get())
                if not msg:
                    break
            except TypeError, e:
                break
            print "__processEventQueue", msg

            evt = msg.pop('event')
            if evt in ['evtRebalance', 'evtTaskConfUpdate']:
                self.callback(events.evtTaskRebalance, msg)
            else:
                log.warn("UNKNOWN EVENT - %s (%s)" % (evt, msg))
コード例 #2
0
ファイル: task.py プロジェクト: ddinsight/dd-streamworks
    def rebalance(self, tasks=None):
        """

        :param task: task list to re-balancing ('all' for re-balance all tasks)
        :return: None
        """
        if not tasks or len(tasks) == 0:
            log.warn("invalid task list for rebalance - %s" % tasks)
            return

        self.__initTaskMapTable(excludeTasks=tasks)
        rebalanceTasks = []

        try:
            taskInfos = self.metaClient.getTasks()
            workers = self.metaClient.getNodes()
        except Exception, e:
            print e
コード例 #3
0
ファイル: slot.py プロジェクト: ddinsight/dd-streamworks
            runTime = time.time() - start
            self.statSumProcessCount += 1
            self.statSumProcessTime = runTime
            if self.statNextPeriod <= time.time():
                log.debug("%s - %d messages. Avg. %f sec" % (self.taskName, self.statSumProcessCount, 1.0 * self.statSumProcessTime / self.statSumProcessCount))
                self.statSumProcessCount = 0
                self.statSumProcessTime = 0
                self.statNextPeriod = time.time() + 30
        except Exception, e:
            log.error("Task Exception - %s (%s) - %s" % (self.taskName, e, params))
            worker.sentry.client.captureException(data=params)
            exc_type, exc_value, exc_traceback = sys.exc_info()
            log.error(traceback.format_exception(exc_type, exc_value, exc_traceback))
            return False
        except KeyboardInterrupt:
            log.warn("CATCH KEYBOARD INTERRUPT in SLOT!!!!!")
            worker.stop()
            return False

        return True

    def stop(self):
        log.debug("Get SLOT STOP! - %s" % self.taskName)
        self.running = False
        if self.amqpConsumer:
            self.amqpConsumer.stop()
        return True

    def pause(self):
        pass
コード例 #4
0
ファイル: msgfeeder.py プロジェクト: ddinsight/dd-streamworks
    def _run(self):
        self.bLoop = True
        self.amqp.start()

        while self.bLoop:
            try:
                print self.taskStatus
                # no worker, no process
                if len(self.taskStatus) == 0 or len(filter(lambda (t, w): w == 0, self.taskStatus.items())) > 0:
                    log.warn("Found not assigned task. - %s" % filter(lambda (t, w): w == 0, self.taskStatus.items()))
                    sleep(1)
                    continue

                """
                Process MongoDB Queue
                """
                if self.pause:
                    sleep(1)
                    continue

                if len(self.exchangeInfo) == 0:
                    sleep(1)
                    log.warn("No Message exchange info.")
                    continue

                sentCount = Counter()
                startFeed = time.time()

                fetcher = msgqueue.burstFetchMessages(self.collection, limit=2000)
                if fetcher.count() == 0:
                    sleep(1)
                    log.info("NO DATA.")
                    continue

                for msg in fetcher:
                    if not self.bLoop:
                        msgqueue.flushAcks(self.collection)
                        break

                    try:
                        msg["_id"] = str(msg["_id"])

                        if "time" in msg and isinstance(msg["time"], datetime.datetime):
                            msg["time"] = str(msg["time"])

                        if "log_time" in msg and isinstance(msg["log_time"], datetime.datetime):
                            msg["log_time"] = str(msg["log_time"])

                    except Exception, e:
                        self.ackMessage(self.collection, msg["_id"])
                        log.error(e)
                        continue

                    if msg.get("log_type", None) == "debugreport":
                        self.ackMessage(self.collection, msg["_id"])
                        continue

                    eventId = LOG_TYPE_MAP.get(msg["log_type"], None)
                    if not eventId:
                        log.error("Undefined Log type - %s" % msg)
                        self.ackMessage(self.collection, msg["_id"])
                        continue

                    sentCount[eventId] += 1
                    if eventId not in self.exchangeInfo:
                        # Wait only ANS/AAT Log.

                        if eventId in ["evtPlayerLog", "evtNetworkLog"]:
                            log.warn("WARNING!!!! no worker wait %s event!!!!!" % eventId)
                            sleep(1)
                            continue
                        else:
                            # TODO: temporary, remember!!!!
                            self.ackMessage(self.collection, msg["_id"])
                            continue
                        pass

                    events = self.exchangeInfo[eventId]
                    try:
                        if (
                            self.amqp.publish(
                                events["exchange"],
                                events["routing"],
                                msg,
                                {"hashID": msg[events["hashKey"]]} if events["hashKey"] else None,
                            )
                            == True
                        ):
                            self.ackMessage(self.collection, msg["_id"])
                            sentCount["sent"] += 1
                        else:
                            print "!"
                            continue
                    except KeyError:
                        # delete message
                        self.ackMessage(self.collection, msg["_id"])
                    except Exception, e:
                        log.error("Publish error - %s" % e)