def _write_to_row(self, row1=None, row2=None, row3=None, row4=None): timestamp1 = datetime.datetime.utcnow().replace(tzinfo=pytz.utc) # Must be millisecond granularity. timestamp1 = _microseconds_to_timestamp( _timestamp_to_microseconds(timestamp1)) # 1000 microseconds is a millisecond timestamp2 = timestamp1 + datetime.timedelta(microseconds=1000) timestamp3 = timestamp1 + datetime.timedelta(microseconds=2000) timestamp4 = timestamp1 + datetime.timedelta(microseconds=3000) if row1 is not None: row1.set_cell(COLUMN_FAMILY_ID1, COL_NAME1, CELL_VAL1, timestamp=timestamp1) if row2 is not None: row2.set_cell(COLUMN_FAMILY_ID1, COL_NAME1, CELL_VAL2, timestamp=timestamp2) if row3 is not None: row3.set_cell(COLUMN_FAMILY_ID1, COL_NAME2, CELL_VAL3, timestamp=timestamp3) if row4 is not None: row4.set_cell(COLUMN_FAMILY_ID2, COL_NAME3, CELL_VAL4, timestamp=timestamp4) # Create the cells we will check. cell1 = Cell(CELL_VAL1, timestamp1) cell2 = Cell(CELL_VAL2, timestamp2) cell3 = Cell(CELL_VAL3, timestamp3) cell4 = Cell(CELL_VAL4, timestamp4) return cell1, cell2, cell3, cell4
def __init__(self, table, timestamp=None, batch_size=None, transaction=False, wal=_WAL_SENTINEL): if wal is not _WAL_SENTINEL: raise ValueError('The wal argument cannot be used with ' 'Cloud Bigtable.') if batch_size is not None: if transaction: raise TypeError('When batch_size is set, a Batch cannot be ' 'transactional') if batch_size <= 0: raise ValueError('batch_size must be positive') self._table = table self._batch_size = batch_size # Timestamp is in milliseconds, convert to microseconds. self._timestamp = self._delete_range = None if timestamp is not None: self._timestamp = _microseconds_to_timestamp(1000 * timestamp) # For deletes, we get the very next timestamp (assuming timestamp # granularity is milliseconds). This is because HappyBase users # expect HBase deletes to go **up to** and **including** the # timestamp while Cloud Bigtable Time Ranges **exclude** the # final timestamp. next_timestamp = self._timestamp + _ONE_MILLISECOND self._delete_range = TimestampRange(end=next_timestamp) self._transaction = transaction # Internal state for tracking mutations. self._row_map = {} self._mutation_count = 0
def test_with_timestamp(self): from gcloud_bigtable._helpers import _microseconds_to_timestamp from gcloud_bigtable.row_data import Cell value1 = 'foo' ts1_millis = 1221934570148 ts1 = _microseconds_to_timestamp(ts1_millis * 1000) cell1 = Cell(value=value1, timestamp=ts1) value2 = 'bar' ts2_millis = 1221955575548 ts2 = _microseconds_to_timestamp(ts2_millis * 1000) cell2 = Cell(value=value2, timestamp=ts2) result = self._callFUT([cell1, cell2], include_timestamp=True) self.assertEqual(result, [(value1, ts1_millis), (value2, ts2_millis)])
def test_success(self): from gcloud_bigtable._helpers import _microseconds_to_timestamp from gcloud_bigtable.row import TimestampRange timestamp = 1441928298571 ts_dt = _microseconds_to_timestamp(1000 * timestamp) result = self._callFUT(timestamp=timestamp) self.assertTrue(isinstance(result, TimestampRange)) self.assertEqual(result.start, None) self.assertEqual(result.end, ts_dt)
def test_constructor_explicit(self): from gcloud_bigtable._helpers import _microseconds_to_timestamp from gcloud_bigtable.row import TimestampRange table = object() timestamp = 144185290431 batch_size = 42 transaction = False # Must be False when batch_size is non-null batch = self._makeOne(table, timestamp=timestamp, batch_size=batch_size, transaction=transaction) self.assertEqual(batch._table, table) self.assertEqual(batch._batch_size, batch_size) self.assertEqual(batch._timestamp, _microseconds_to_timestamp(1000 * timestamp)) next_timestamp = _microseconds_to_timestamp(1000 * (timestamp + 1)) time_range = TimestampRange(end=next_timestamp) self.assertEqual(batch._delete_range, time_range) self.assertEqual(batch._transaction, transaction)
def from_pb(cls, cell_pb): """Create a new cell from a Cell protobuf. :type cell_pb: :class:`._generated.bigtable_data_pb2.Cell` :param cell_pb: The protobuf to convert. :rtype: :class:`Cell` :returns: The cell corresponding to the protobuf. """ timestamp = _microseconds_to_timestamp(cell_pb.timestamp_micros) return cls(cell_pb.value, timestamp)
def test_it(self): from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 from gcloud_bigtable._helpers import _microseconds_to_timestamp COL_FAM1 = u'col-fam-id' COL_NAME1 = b'col-name1' COL_NAME2 = b'col-name2' CELL_VAL1 = b'cell-val' CELL_VAL2 = b'cell-val-newer' CELL_VAL3 = b'altcol-cell-val' microseconds = 5554441037 timestamp = _microseconds_to_timestamp(microseconds) expected_dict = { COL_NAME1: [ (CELL_VAL1, timestamp), (CELL_VAL2, timestamp), ], COL_NAME2: [ (CELL_VAL3, timestamp), ], } expected_output = (COL_FAM1, expected_dict) sample_input = data_pb2.Family( name=COL_FAM1, columns=[ data_pb2.Column( qualifier=COL_NAME1, cells=[ data_pb2.Cell( value=CELL_VAL1, timestamp_micros=microseconds, ), data_pb2.Cell( value=CELL_VAL2, timestamp_micros=microseconds, ), ], ), data_pb2.Column( qualifier=COL_NAME2, cells=[ data_pb2.Cell( value=CELL_VAL3, timestamp_micros=microseconds, ), ], ), ], ) self.assertEqual(expected_output, self._callFUT(sample_input))
def test_with_timestamp(self): from gcloud_bigtable._helpers import _microseconds_to_timestamp from gcloud_bigtable.row import RowFilter from gcloud_bigtable.row import TimestampRange timestamp = 1441928298571 result = self._column_helper(num_filters=3, timestamp=timestamp) range_filter = result.filters[2] self.assertTrue(isinstance(range_filter, RowFilter)) # Relies on the fact that RowFilter instances can # only have one value set. time_range = range_filter.timestamp_range_filter self.assertTrue(isinstance(time_range, TimestampRange)) self.assertEqual(time_range.start, None) ts_dt = _microseconds_to_timestamp(1000 * timestamp) self.assertEqual(time_range.end, ts_dt)
def _convert_to_time_range(timestamp=None): """Create a timestamp range from an HBase / HappyBase timestamp. HBase uses timestamp as an argument to specify an inclusive end deadline. Since Cloud Bigtable uses exclusive end times, we increment by one millisecond (the lowest granularity allowed). :type timestamp: int :param timestamp: (Optional) Timestamp (in milliseconds since the epoch). Intended to be used as the end of an HBase time range, which is exclusive. :rtype: :class:`.TimestampRange`, :data:`NoneType <types.NoneType>` :returns: The timestamp range corresponding to the passed in ``timestamp``. """ if timestamp is None: return None next_timestamp = _microseconds_to_timestamp(1000 * timestamp) return TimestampRange(end=next_timestamp)
def _callFUT(self, microseconds): from gcloud_bigtable._helpers import _microseconds_to_timestamp return _microseconds_to_timestamp(microseconds)