def flushq(self, flush_hotkeys = False):
mq = RabbitHelper()
if self.ccq is not None:
logging.info("[Thread %s] flushing %s items to %s" %
(self.name, self.memq.qsize(), self.ccq))
# declare queue
mq.declare(self.ccq)
# empty the in memory queue
while self.memq.empty() == False:
try:
msg = self.memq.get_nowait()
msg = json.dumps(msg)
mq.putMsg(self.ccq, msg)
except queue.Empty:
pass
# hot keys
if flush_hotkeys and (len(self.hotkey_batches) > 0):
# try to put onto remote queue
queue = self.consume_queue or self.ccq
if queue is not None:
key_map = {'start' : self.hotkey_batches[0][0],
'end' : self.hotkey_batches[-1][-1]}
msg = json.dumps(key_map)
mq.putMsg(queue, msg)
self.hotkey_batches = []
def flushq(self):
if self.ccq is not None:
# declare queue
mq = RabbitHelper()
mq.declare(self.ccq)
while self.memq.empty() == False:
try:
msg = self.memq.get_nowait()
msg = json.dumps(msg)
mq.putMsg(self.ccq, msg)
except queue.Empty:
pass
# hot keys
if len(self.hotkeys) > 0:
key_map = {'start' : self.hotkeys[0],
'end' : self.hotkeys[-1]}
msg = json.dumps(key_map)
mq.putMsg(self.ccq, msg)
def flushq(self, flush_hotkeys=False):
if self.standalone:
return
mq = RabbitHelper()
if self.ccq is not None:
logging.info("[Thread %s] flushing %s items to %s" %
(self.name, self.memq.qsize(), self.ccq))
# declare queue
mq.declare(self.ccq)
# empty the in memory queue
while self.memq.empty() == False:
try:
msg = self.memq.get_nowait()
msg = json.dumps(msg)
mq.putMsg(self.ccq, msg)
except queue.Empty:
pass
# hot keys
if flush_hotkeys and (len(self.hotkey_batches) > 0):
# try to put onto remote queue
queue = self.consume_queue or self.ccq
if queue is not None:
key_map = {
'start': self.hotkey_batches[0][0],
'end': self.hotkey_batches[-1][-1]
}
msg = json.dumps(key_map)
mq.putMsg(queue, msg)
self.hotkey_batches = []
def main():
args = parser.parse_args()
CB_CLUSTER_TAG = args.cluster
exchange = CB_CLUSTER_TAG + "consumers"
# setup to consume messages from worker
mq = RabbitHelper()
mq.exchange_declare(exchange, "fanout")
queue = mq.declare()
queue_name = queue[0]
# bind to exchange
mq.bind(exchange, queue_name)
mq.putMsg('', 'init', exchange)
# consume messages
channel, conn = mq.channel()
channel.basic_consume(callback=init, queue=queue_name, no_ack=True)
while True:
conn.drain_events()
def main():
args = parser.parse_args()
CB_CLUSTER_TAG = args.cluster
exchange = CB_CLUSTER_TAG+"consumers"
# setup to consume messages from worker
mq = RabbitHelper()
mq.exchange_declare(exchange, "fanout")
queue = mq.declare()
queue_name = queue[0]
# bind to exchange
mq.bind(exchange, queue_name)
mq.putMsg('', 'init', exchange)
# consume messages
channel, conn = mq.channel()
channel.basic_consume(callback = init, queue = queue_name, no_ack = True)
while True:
conn.drain_events()