def record_messages(messages): hbase_conn = HBaseConnection('localhost', 9090) for message in messages: hbase_conn.put('message_log', str(99999999999 - int(message["timestamp"])), 'messages:id', str(message["id"]), 'messages:from', str(message["from"]), 'messages:body', str(message["body"]), 'messages:html', str(message["html"]))
def __init__(self, host='localhost', port=9090): self._port = port self._host = host self.connection = HBaseConnection(self._host, self._port) try: print "Connected to HBase %s" % self.connection.get_hbase_version() except: print "Error connecting to HBase" self.__init__tables()
class MessageMixin(object): waiters = [] cache = [] cache_size = 200 # Read initial cache from HBase if hbase_enabled: hbase_conn = HBaseConnection('localhost', 9090) old_messages = hbase_conn.scan('message_log', cache_size) for message in old_messages: cache_entry = dict([(entry[u'qualifier'], entry[u'value']) for entry in message[u'entries']]) cache.insert(0, cache_entry) # TODO(hammer): Don't create a new connection every time # TODO(hammer): Would be handy to have MultiPut here @staticmethod def record_messages(messages): hbase_conn = HBaseConnection('localhost', 9090) for message in messages: hbase_conn.put('message_log', str(99999999999 - int(message["timestamp"])), 'messages:id', str(message["id"]), 'messages:from', str(message["from"]), 'messages:body', str(message["body"]), 'messages:html', str(message["html"])) def wait_for_messages(self, callback, cursor=None): cls = MessageMixin if cursor: index = 0 for i in xrange(len(cls.cache)): index = len(cls.cache) - i - 1 if cls.cache[index]["id"] == cursor: break recent = cls.cache[index + 1:] if recent: callback(recent) return cls.waiters.append(callback) def new_messages(self, messages): cls = MessageMixin logging.info("Sending new message to %r listeners", len(cls.waiters)) for callback in cls.waiters: try: callback(messages) except: logging.error("Error in waiter callback", exc_info=True) cls.waiters = [] cls.cache.extend(messages) # add to HBase if hbase_enabled: cls.record_messages(messages) # Expire messages off the front of the cache if len(cls.cache) > self.cache_size: cls.cache = cls.cache[-self.cache_size:]
def __init__(self, host = 'localhost', port = 9090): self._port = port self._host = host self.connection = HBaseConnection(self._host, self._port) try: print "Connected to HBase %s" % self.connection.get_hbase_version() except: print "Error connecting to HBase" self.__init__tables()
class RaindropHBaseClient: def __init__(self, host='localhost', port=9090): self._port = port self._host = host self.connection = HBaseConnection(self._host, self._port) try: print "Connected to HBase %s" % self.connection.get_hbase_version() except: print "Error connecting to HBase" self.__init__tables() def __init__tables(self): if not self.connection.table_exists(MESSAGES_TABLE): self.__create_tables() def __create_tables(self): try: self.connection.create_table(MESSAGES_TABLE, IDS_COLUMN, SUMMARY_COLUMN, DATA_COLUMN) except AvroException, e: print e.message.get("message") raise e
class RaindropHBaseClient: def __init__(self, host = 'localhost', port = 9090): self._port = port self._host = host self.connection = HBaseConnection(self._host, self._port) try: print "Connected to HBase %s" % self.connection.get_hbase_version() except: print "Error connecting to HBase" self.__init__tables() def __init__tables(self): if not self.connection.table_exists(MESSAGES_TABLE): self.__create_tables() def __create_tables(self): try: self.connection.create_table(MESSAGES_TABLE, IDS_COLUMN, SUMMARY_COLUMN, DATA_COLUMN) except AvroException, e: print e.message.get("message") raise e
#! /usr/bin/env python from pyhbase.connection import HBaseConnection if __name__ == "__main__": hbase_conn = HBaseConnection('localhost', 9090) if not hbase_conn.table_exists('message_log'): hbase_conn.create_table('message_log', 'messages')