def record_messages(messages):
hbase_conn = HBaseConnection('localhost', 9090)
for message in messages:
hbase_conn.put('message_log', str(99999999999 - int(message["timestamp"])),
'messages:id', str(message["id"]),
'messages:from', str(message["from"]),
'messages:body', str(message["body"]),
'messages:html', str(message["html"]))
class MessageMixin(object):
waiters = []
cache = []
cache_size = 200
# Read initial cache from HBase
if hbase_enabled:
hbase_conn = HBaseConnection('localhost', 9090)
old_messages = hbase_conn.scan('message_log', cache_size)
for message in old_messages:
cache_entry = dict([(entry[u'qualifier'], entry[u'value'])
for entry in message[u'entries']])
cache.insert(0, cache_entry)
# TODO(hammer): Don't create a new connection every time
# TODO(hammer): Would be handy to have MultiPut here
@staticmethod
def record_messages(messages):
hbase_conn = HBaseConnection('localhost', 9090)
for message in messages:
hbase_conn.put('message_log', str(99999999999 - int(message["timestamp"])),
'messages:id', str(message["id"]),
'messages:from', str(message["from"]),
'messages:body', str(message["body"]),
'messages:html', str(message["html"]))
def wait_for_messages(self, callback, cursor=None):
cls = MessageMixin
if cursor:
index = 0
for i in xrange(len(cls.cache)):
index = len(cls.cache) - i - 1
if cls.cache[index]["id"] == cursor: break
recent = cls.cache[index + 1:]
if recent:
callback(recent)
return
cls.waiters.append(callback)
def new_messages(self, messages):
cls = MessageMixin
logging.info("Sending new message to %r listeners", len(cls.waiters))
for callback in cls.waiters:
try:
callback(messages)
except:
logging.error("Error in waiter callback", exc_info=True)
cls.waiters = []
cls.cache.extend(messages)
# add to HBase
if hbase_enabled: cls.record_messages(messages)
# Expire messages off the front of the cache
if len(cls.cache) > self.cache_size:
cls.cache = cls.cache[-self.cache_size:]
def __init__(self, host='localhost', port=9090):
self._port = port
self._host = host
self.connection = HBaseConnection(self._host, self._port)
try:
print "Connected to HBase %s" % self.connection.get_hbase_version()
except:
print "Error connecting to HBase"
self.__init__tables()
#! /usr/bin/env python
from pyhbase.connection import HBaseConnection
if __name__ == "__main__":
hbase_conn = HBaseConnection('localhost', 9090)
if not hbase_conn.table_exists('message_log'):
hbase_conn.create_table('message_log', 'messages')
