def __init__(self, collection, ordering=False):
Greenlet.__init__(self)
self.pause = False
self.taskStatus = {}
self.exchangeInfo = {}
self.name = collection.name
self.collection = collection
self.doOrdering = ordering
log.info("Fetch Message from MongoQueue - Collection : %s" % self.collection.name)
# Parsing job server list
self.amqp = AMQPProducer(settings.conf.get("server", "broker"))
# Local event queue
self.bLoop = False
self.lastQueueCheckTime = 0
# Statsd
self.ackMessage = partial(msgqueue.ackMessage, flushInterval=1)
class MessageFeeder(Greenlet):
QUEUE_WAIT_TIME = 5
def __init__(self, collection, ordering=False):
Greenlet.__init__(self)
self.pause = False
self.taskStatus = {}
self.exchangeInfo = {}
self.name = collection.name
self.collection = collection
self.doOrdering = ordering
log.info("Fetch Message from MongoQueue - Collection : %s" % self.collection.name)
# Parsing job server list
self.amqp = AMQPProducer(settings.conf.get("server", "broker"))
# Local event queue
self.bLoop = False
self.lastQueueCheckTime = 0
# Statsd
self.ackMessage = partial(msgqueue.ackMessage, flushInterval=1)
def _run(self):
self.bLoop = True
self.amqp.start()
while self.bLoop:
try:
print self.taskStatus
# no worker, no process
if len(self.taskStatus) == 0 or len(filter(lambda (t, w): w == 0, self.taskStatus.items())) > 0:
log.warn("Found not assigned task. - %s" % filter(lambda (t, w): w == 0, self.taskStatus.items()))
sleep(1)
continue
"""
Process MongoDB Queue
"""
if self.pause:
sleep(1)
continue
if len(self.exchangeInfo) == 0:
sleep(1)
log.warn("No Message exchange info.")
continue
sentCount = Counter()
startFeed = time.time()
fetcher = msgqueue.burstFetchMessages(self.collection, limit=2000)
if fetcher.count() == 0:
sleep(1)
log.info("NO DATA.")
continue
for msg in fetcher:
if not self.bLoop:
msgqueue.flushAcks(self.collection)
break
try:
msg["_id"] = str(msg["_id"])
if "time" in msg and isinstance(msg["time"], datetime.datetime):
msg["time"] = str(msg["time"])
if "log_time" in msg and isinstance(msg["log_time"], datetime.datetime):
msg["log_time"] = str(msg["log_time"])
except Exception, e:
self.ackMessage(self.collection, msg["_id"])
log.error(e)
continue
if msg.get("log_type", None) == "debugreport":
self.ackMessage(self.collection, msg["_id"])
continue
eventId = LOG_TYPE_MAP.get(msg["log_type"], None)
if not eventId:
log.error("Undefined Log type - %s" % msg)
self.ackMessage(self.collection, msg["_id"])
continue
sentCount[eventId] += 1
if eventId not in self.exchangeInfo:
# Wait only ANS/AAT Log.
if eventId in ["evtPlayerLog", "evtNetworkLog"]:
log.warn("WARNING!!!! no worker wait %s event!!!!!" % eventId)
sleep(1)
continue
else:
# TODO: temporary, remember!!!!
self.ackMessage(self.collection, msg["_id"])
continue
pass
events = self.exchangeInfo[eventId]
try:
if (
self.amqp.publish(
events["exchange"],
events["routing"],
msg,
{"hashID": msg[events["hashKey"]]} if events["hashKey"] else None,
)
== True
):
self.ackMessage(self.collection, msg["_id"])
sentCount["sent"] += 1
else:
print "!"
continue
except KeyError:
# delete message
self.ackMessage(self.collection, msg["_id"])
except Exception, e:
log.error("Publish error - %s" % e)
if sentCount["sent"] == 0:
log.info("%s No Data? - sleep" % self.name)
sleep(1)
else:
msgqueue.flushAcks(self.collection)
# queue wait time
if sentCount["sent"] < 100:
sleep(1)
log.info(
"MONGO %s PROCESSED [%0.2f] - %d/%d (%s)"
% (
self.name,
time.time() - startFeed,
sentCount.pop("sent", 0),
sentCount.pop("wait", 0),
", ".join(map(lambda (k, v): "%s: %s" % (k, v), sentCount.iteritems())),
)
)
