def _write_to_row(self, row1=None, row2=None, row3=None, row4=None):
timestamp1 = datetime.datetime.utcnow().replace(tzinfo=pytz.utc)
# Must be millisecond granularity.
timestamp1 = _microseconds_to_timestamp(
_timestamp_to_microseconds(timestamp1))
# 1000 microseconds is a millisecond
timestamp2 = timestamp1 + datetime.timedelta(microseconds=1000)
timestamp3 = timestamp1 + datetime.timedelta(microseconds=2000)
timestamp4 = timestamp1 + datetime.timedelta(microseconds=3000)
if row1 is not None:
row1.set_cell(COLUMN_FAMILY_ID1, COL_NAME1, CELL_VAL1,
timestamp=timestamp1)
if row2 is not None:
row2.set_cell(COLUMN_FAMILY_ID1, COL_NAME1, CELL_VAL2,
timestamp=timestamp2)
if row3 is not None:
row3.set_cell(COLUMN_FAMILY_ID1, COL_NAME2, CELL_VAL3,
timestamp=timestamp3)
if row4 is not None:
row4.set_cell(COLUMN_FAMILY_ID2, COL_NAME3, CELL_VAL4,
timestamp=timestamp4)
# Create the cells we will check.
cell1 = Cell(CELL_VAL1, timestamp1)
cell2 = Cell(CELL_VAL2, timestamp2)
cell3 = Cell(CELL_VAL3, timestamp3)
cell4 = Cell(CELL_VAL4, timestamp4)
return cell1, cell2, cell3, cell4
def __init__(self, table, timestamp=None, batch_size=None,
transaction=False, wal=_WAL_SENTINEL):
if wal is not _WAL_SENTINEL:
raise ValueError('The wal argument cannot be used with '
'Cloud Bigtable.')
if batch_size is not None:
if transaction:
raise TypeError('When batch_size is set, a Batch cannot be '
'transactional')
if batch_size <= 0:
raise ValueError('batch_size must be positive')
self._table = table
self._batch_size = batch_size
# Timestamp is in milliseconds, convert to microseconds.
self._timestamp = self._delete_range = None
if timestamp is not None:
self._timestamp = _microseconds_to_timestamp(1000 * timestamp)
# For deletes, we get the very next timestamp (assuming timestamp
# granularity is milliseconds). This is because HappyBase users
# expect HBase deletes to go **up to** and **including** the
# timestamp while Cloud Bigtable Time Ranges **exclude** the
# final timestamp.
next_timestamp = self._timestamp + _ONE_MILLISECOND
self._delete_range = TimestampRange(end=next_timestamp)
self._transaction = transaction
# Internal state for tracking mutations.
self._row_map = {}
self._mutation_count = 0
def test_with_timestamp(self):
from gcloud_bigtable._helpers import _microseconds_to_timestamp
from gcloud_bigtable.row_data import Cell
value1 = 'foo'
ts1_millis = 1221934570148
ts1 = _microseconds_to_timestamp(ts1_millis * 1000)
cell1 = Cell(value=value1, timestamp=ts1)
value2 = 'bar'
ts2_millis = 1221955575548
ts2 = _microseconds_to_timestamp(ts2_millis * 1000)
cell2 = Cell(value=value2, timestamp=ts2)
result = self._callFUT([cell1, cell2], include_timestamp=True)
self.assertEqual(result,
[(value1, ts1_millis), (value2, ts2_millis)])
def test_success(self):
from gcloud_bigtable._helpers import _microseconds_to_timestamp
from gcloud_bigtable.row import TimestampRange
timestamp = 1441928298571
ts_dt = _microseconds_to_timestamp(1000 * timestamp)
result = self._callFUT(timestamp=timestamp)
self.assertTrue(isinstance(result, TimestampRange))
self.assertEqual(result.start, None)
self.assertEqual(result.end, ts_dt)
def test_constructor_explicit(self):
from gcloud_bigtable._helpers import _microseconds_to_timestamp
from gcloud_bigtable.row import TimestampRange
table = object()
timestamp = 144185290431
batch_size = 42
transaction = False # Must be False when batch_size is non-null
batch = self._makeOne(table, timestamp=timestamp,
batch_size=batch_size, transaction=transaction)
self.assertEqual(batch._table, table)
self.assertEqual(batch._batch_size, batch_size)
self.assertEqual(batch._timestamp,
_microseconds_to_timestamp(1000 * timestamp))
next_timestamp = _microseconds_to_timestamp(1000 * (timestamp + 1))
time_range = TimestampRange(end=next_timestamp)
self.assertEqual(batch._delete_range, time_range)
self.assertEqual(batch._transaction, transaction)
def from_pb(cls, cell_pb):
"""Create a new cell from a Cell protobuf.
:type cell_pb: :class:`._generated.bigtable_data_pb2.Cell`
:param cell_pb: The protobuf to convert.
:rtype: :class:`Cell`
:returns: The cell corresponding to the protobuf.
"""
timestamp = _microseconds_to_timestamp(cell_pb.timestamp_micros)
return cls(cell_pb.value, timestamp)
def test_it(self):
from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2
from gcloud_bigtable._helpers import _microseconds_to_timestamp
COL_FAM1 = u'col-fam-id'
COL_NAME1 = b'col-name1'
COL_NAME2 = b'col-name2'
CELL_VAL1 = b'cell-val'
CELL_VAL2 = b'cell-val-newer'
CELL_VAL3 = b'altcol-cell-val'
microseconds = 5554441037
timestamp = _microseconds_to_timestamp(microseconds)
expected_dict = {
COL_NAME1: [
(CELL_VAL1, timestamp),
(CELL_VAL2, timestamp),
],
COL_NAME2: [
(CELL_VAL3, timestamp),
],
}
expected_output = (COL_FAM1, expected_dict)
sample_input = data_pb2.Family(
name=COL_FAM1,
columns=[
data_pb2.Column(
qualifier=COL_NAME1,
cells=[
data_pb2.Cell(
value=CELL_VAL1,
timestamp_micros=microseconds,
),
data_pb2.Cell(
value=CELL_VAL2,
timestamp_micros=microseconds,
),
],
),
data_pb2.Column(
qualifier=COL_NAME2,
cells=[
data_pb2.Cell(
value=CELL_VAL3,
timestamp_micros=microseconds,
),
],
),
],
)
self.assertEqual(expected_output, self._callFUT(sample_input))
def test_with_timestamp(self):
from gcloud_bigtable._helpers import _microseconds_to_timestamp
from gcloud_bigtable.row import RowFilter
from gcloud_bigtable.row import TimestampRange
timestamp = 1441928298571
result = self._column_helper(num_filters=3, timestamp=timestamp)
range_filter = result.filters[2]
self.assertTrue(isinstance(range_filter, RowFilter))
# Relies on the fact that RowFilter instances can
# only have one value set.
time_range = range_filter.timestamp_range_filter
self.assertTrue(isinstance(time_range, TimestampRange))
self.assertEqual(time_range.start, None)
ts_dt = _microseconds_to_timestamp(1000 * timestamp)
self.assertEqual(time_range.end, ts_dt)
def _convert_to_time_range(timestamp=None):
"""Create a timestamp range from an HBase / HappyBase timestamp.
HBase uses timestamp as an argument to specify an inclusive end
deadline. Since Cloud Bigtable uses exclusive end times, we increment
by one millisecond (the lowest granularity allowed).
:type timestamp: int
:param timestamp: (Optional) Timestamp (in milliseconds since the
epoch). Intended to be used as the end of an HBase
time range, which is exclusive.
:rtype: :class:`.TimestampRange`, :data:`NoneType <types.NoneType>`
:returns: The timestamp range corresponding to the passed in
``timestamp``.
"""
if timestamp is None:
return None
next_timestamp = _microseconds_to_timestamp(1000 * timestamp)
return TimestampRange(end=next_timestamp)
def _callFUT(self, microseconds):
from gcloud_bigtable._helpers import _microseconds_to_timestamp
return _microseconds_to_timestamp(microseconds)
