Пример #1
0
class HbaseSaver(StateSaver):
    ''' hbase saver '''
    def __init__(self, ip="localhost", port=9090):
        ''' constructor '''
        super(HbaseSaver, self).__init__()
        self.__hbase = HBaseLib(ip, port)
        self.__writeCache = {}

    def resetCols(self, cols):
        ''' create cols '''
        if self.tableName in self.__hbase.getTableNames():
            self.__hbase.deleteTable(self.tableName)

        LOG.debug("create table %s with cols %s" % (self.tableName, cols))
        self.__hbase.createTable(
            self.tableName,
            [ColumnDescriptor(name=str(col), maxVersions=5) for col in cols])

    def read(self, row, col):
        ''' read value with row and col '''
        oneRow = self.__hbase.getRow(self.tableName, row)
        keyValues = oneRow.columns
        key = "%s:" % col

        if key in keyValues:
            return keyValues[key].value
        else:
            return None

    def write(self, row, col, value):
        ''' write value with row and col '''
        self.__writeCache[(row, col)] = value

    def commit(self):
        ''' complete write operation '''
        if not self.tableName:
            raise UfException(Errors.TABLENAME_NOT_SET, "Table name not set")

        # reset table with all cols first
        cols = set()
        for (row, col) in self.__writeCache.iterkeys():
            cols.add(col)

        self.resetCols(cols)

        # write values
        for (row, col), value in self.__writeCache.iteritems():
            self.__hbase.updateRow(
                self.tableName, row,
                [Mutation(column="%s:" % col, value=str(value))])

    def setup(self, setting):
        ''' setup '''
        pass
Пример #2
0
class HbaseSaver(StateSaver):
    ''' hbase saver '''
    def __init__(self, ip = "localhost", port = 9090):
        ''' constructor '''
        super(HbaseSaver, self).__init__()
        self.__hbase = HBaseLib(ip, port)
        self.__writeCache = {}

    def resetCols(self, cols):
        ''' create cols '''
        if self.tableName in self.__hbase.getTableNames():
            self.__hbase.deleteTable(self.tableName)

        LOG.debug("create table %s with cols %s" % (self.tableName, cols))
        self.__hbase.createTable(self.tableName, [ColumnDescriptor(name = str(col), maxVersions = 5) for col in cols])

    def read(self, row, col):
        ''' read value with row and col '''
        oneRow = self.__hbase.getRow(self.tableName, row)
        keyValues = oneRow.columns
        key = "%s:" % col

        if key in keyValues:
            return keyValues[key].value
        else:
            return None

    def write(self, row, col, value):
        ''' write value with row and col '''
        self.__writeCache[(row, col)] = value

    def commit(self):
        ''' complete write operation '''
        if not self.tableName:
            raise UfException(Errors.TABLENAME_NOT_SET,
                              "Table name not set")

        # reset table with all cols first
        cols = set()
        for (row, col) in self.__writeCache.iterkeys():
            cols.add(col)

        self.resetCols(cols)

        # write values
        for (row, col), value in self.__writeCache.iteritems():
            self.__hbase.updateRow(self.tableName,
                                   row,
                                   [Mutation(column = "%s:" % col, value = str(value))])

    def setup(self, setting):
        ''' setup '''
        pass
class HBaseDAM(BaseDAM):
    ''' HBase DAO '''
    QUOTE = 'quote'
    TICK = 'tick'

    def __init__(self, ip="localhost", port=9090):
        ''' constructor '''
        super(HBaseDAM, self).__init__()
        self.__hbase = HBaseLib(ip, port)

    def tableName(self, kind):
        return "%s-%s" % (self.symbol, kind)

    def __rowResultToQuote(self, row):
        ''' convert rowResult from Hbase to Quote'''
        keyValues = row.columns
        for field in QUOTE_FIELDS:
            key = "%s:%s" % (HBaseDAM.QUOTE, field)
            if 'time' != field and keyValues[key].value:
                keyValues[key].value = float(keyValues[key].value)

        return Quote(*[keyValues["%s:%s" % (HBaseDAM.QUOTE, field)].value for field in QUOTE_FIELDS])

    def __rowResultToTick(self, row):
        ''' convert rowResult from Hbase to Tick'''
        keyValues = row.columns
        for field in TICK_FIELDS:
            key = "%s:%s" % (HBaseDAM.TICK, field)
            if 'time' != field and keyValues[key].value:
                keyValues[key].value = float(keyValues[key].value)

        return Tick(*[keyValues["%s:%s" % (HBaseDAM.TICK, field)].value for field in TICK_FIELDS])

    def readQuotes(self, start, end):
        ''' read quotes '''
        rows = self.__hbase.scanTable(self.tableName(HBaseDAM.QUOTE), [HBaseDAM.QUOTE], start, end)

        return [self.__rowResultToQuote(row) for row in rows]

    def writeQuotes(self, quotes):
        ''' write quotes '''
        tName = self.tableName(HBaseDAM.QUOTE)
        if tName not in self.__hbase.getTableNames():
            self.__hbase.createTable(tName, [ColumnDescriptor(name=HBaseDAM.QUOTE, maxVersions=5)])

        for quote in quotes:
            self.__hbase.updateRow(self.tableName(HBaseDAM.QUOTE),
                                   quote.time,
                                   [Mutation(column = "%s:%s" % (HBaseDAM.QUOTE, field),
                                             value = getattr(quote, field) ) for field in QUOTE_FIELDS])

    def readTicks(self, start, end):
        ''' read ticks '''
        rows = self.__hbase.scanTable(self.tableName(HBaseDAM.TICK), [HBaseDAM.TICK], start, end)
        return [self.__rowResultToTick(row) for row in rows]

    def writeTicks(self, ticks):
        ''' read quotes '''
        tName = self.tableName(HBaseDAM.TICK)
        if tName not in self.__hbase.getTableNames():
            self.__hbase.createTable(tName, [ColumnDescriptor(name=HBaseDAM.TICK, maxVersions=5)])

        for tick in ticks:
            self.__hbase.updateRow(self.tableName(HBaseDAM.TICK),
                                   tick.time,
                                   [Mutation(column = "%s:%s" % (HBaseDAM.TICK, field),
                                             value = getattr(tick, field) ) for field in TICK_FIELDS])