def __processEventQueue(self):
while True:
try:
msg = json.loads(self.evtQueue.get())
if not msg:
break
except TypeError, e:
break
print "__processEventQueue", msg
evt = msg.pop('event')
if evt in ['evtRebalance', 'evtTaskConfUpdate']:
self.callback(events.evtTaskRebalance, msg)
else:
log.warn("UNKNOWN EVENT - %s (%s)" % (evt, msg))
def rebalance(self, tasks=None):
"""
:param task: task list to re-balancing ('all' for re-balance all tasks)
:return: None
"""
if not tasks or len(tasks) == 0:
log.warn("invalid task list for rebalance - %s" % tasks)
return
self.__initTaskMapTable(excludeTasks=tasks)
rebalanceTasks = []
try:
taskInfos = self.metaClient.getTasks()
workers = self.metaClient.getNodes()
except Exception, e:
print e
runTime = time.time() - start
self.statSumProcessCount += 1
self.statSumProcessTime = runTime
if self.statNextPeriod <= time.time():
log.debug("%s - %d messages. Avg. %f sec" % (self.taskName, self.statSumProcessCount, 1.0 * self.statSumProcessTime / self.statSumProcessCount))
self.statSumProcessCount = 0
self.statSumProcessTime = 0
self.statNextPeriod = time.time() + 30
except Exception, e:
log.error("Task Exception - %s (%s) - %s" % (self.taskName, e, params))
worker.sentry.client.captureException(data=params)
exc_type, exc_value, exc_traceback = sys.exc_info()
log.error(traceback.format_exception(exc_type, exc_value, exc_traceback))
return False
except KeyboardInterrupt:
log.warn("CATCH KEYBOARD INTERRUPT in SLOT!!!!!")
worker.stop()
return False
return True
def stop(self):
log.debug("Get SLOT STOP! - %s" % self.taskName)
self.running = False
if self.amqpConsumer:
self.amqpConsumer.stop()
return True
def pause(self):
pass
def _run(self):
self.bLoop = True
self.amqp.start()
while self.bLoop:
try:
print self.taskStatus
# no worker, no process
if len(self.taskStatus) == 0 or len(filter(lambda (t, w): w == 0, self.taskStatus.items())) > 0:
log.warn("Found not assigned task. - %s" % filter(lambda (t, w): w == 0, self.taskStatus.items()))
sleep(1)
continue
"""
Process MongoDB Queue
"""
if self.pause:
sleep(1)
continue
if len(self.exchangeInfo) == 0:
sleep(1)
log.warn("No Message exchange info.")
continue
sentCount = Counter()
startFeed = time.time()
fetcher = msgqueue.burstFetchMessages(self.collection, limit=2000)
if fetcher.count() == 0:
sleep(1)
log.info("NO DATA.")
continue
for msg in fetcher:
if not self.bLoop:
msgqueue.flushAcks(self.collection)
break
try:
msg["_id"] = str(msg["_id"])
if "time" in msg and isinstance(msg["time"], datetime.datetime):
msg["time"] = str(msg["time"])
if "log_time" in msg and isinstance(msg["log_time"], datetime.datetime):
msg["log_time"] = str(msg["log_time"])
except Exception, e:
self.ackMessage(self.collection, msg["_id"])
log.error(e)
continue
if msg.get("log_type", None) == "debugreport":
self.ackMessage(self.collection, msg["_id"])
continue
eventId = LOG_TYPE_MAP.get(msg["log_type"], None)
if not eventId:
log.error("Undefined Log type - %s" % msg)
self.ackMessage(self.collection, msg["_id"])
continue
sentCount[eventId] += 1
if eventId not in self.exchangeInfo:
# Wait only ANS/AAT Log.
if eventId in ["evtPlayerLog", "evtNetworkLog"]:
log.warn("WARNING!!!! no worker wait %s event!!!!!" % eventId)
sleep(1)
continue
else:
# TODO: temporary, remember!!!!
self.ackMessage(self.collection, msg["_id"])
continue
pass
events = self.exchangeInfo[eventId]
try:
if (
self.amqp.publish(
events["exchange"],
events["routing"],
msg,
{"hashID": msg[events["hashKey"]]} if events["hashKey"] else None,
)
== True
):
self.ackMessage(self.collection, msg["_id"])
sentCount["sent"] += 1
else:
print "!"
continue
except KeyError:
# delete message
self.ackMessage(self.collection, msg["_id"])
except Exception, e:
log.error("Publish error - %s" % e)