def _parse_family_pb(family_pb):
"""Parses a Family protobuf into a dictionary.
:type family_pb: :class:`._generated.data_pb2.Family`
:param family_pb: A protobuf
:rtype: tuple
:returns: A string and dictionary. The string is the name of the
column family and the dictionary has column names (within the
family) as keys and cell lists as values. Each cell is
represented with a two-tuple with the value (in bytes) and the
timestamp for the cell. For example:
.. code:: python
{
b'col-name1': [
(b'cell-val', datetime.datetime(...)),
(b'cell-val-newer', datetime.datetime(...)),
],
b'col-name2': [
(b'altcol-cell-val', datetime.datetime(...)),
],
}
"""
result = {}
for column in family_pb.columns:
result[column.qualifier] = cells = []
for cell in column.cells:
val_pair = (cell.value, _datetime_from_microseconds(cell.timestamp_micros))
cells.append(val_pair)
return family_pb.name, result
def _write_to_row(self, row1=None, row2=None, row3=None, row4=None):
timestamp1 = datetime.datetime.utcnow().replace(tzinfo=UTC)
timestamp1_micros = _microseconds_from_datetime(timestamp1)
# Truncate to millisecond granularity.
timestamp1_micros -= timestamp1_micros % 1000
timestamp1 = _datetime_from_microseconds(timestamp1_micros)
# 1000 microseconds is a millisecond
timestamp2 = timestamp1 + datetime.timedelta(microseconds=1000)
timestamp2_micros = _microseconds_from_datetime(timestamp2)
timestamp3 = timestamp1 + datetime.timedelta(microseconds=2000)
timestamp3_micros = _microseconds_from_datetime(timestamp3)
timestamp4 = timestamp1 + datetime.timedelta(microseconds=3000)
timestamp4_micros = _microseconds_from_datetime(timestamp4)
if row1 is not None:
row1.set_cell(COLUMN_FAMILY_ID1, COL_NAME1, CELL_VAL1, timestamp=timestamp1)
if row2 is not None:
row2.set_cell(COLUMN_FAMILY_ID1, COL_NAME1, CELL_VAL2, timestamp=timestamp2)
if row3 is not None:
row3.set_cell(COLUMN_FAMILY_ID1, COL_NAME2, CELL_VAL3, timestamp=timestamp3)
if row4 is not None:
row4.set_cell(COLUMN_FAMILY_ID2, COL_NAME3, CELL_VAL4, timestamp=timestamp4)
# Create the cells we will check.
cell1 = Cell(CELL_VAL1, timestamp1_micros)
cell2 = Cell(CELL_VAL2, timestamp2_micros)
cell3 = Cell(CELL_VAL3, timestamp3_micros)
cell4 = Cell(CELL_VAL4, timestamp4_micros)
return cell1, cell2, cell3, cell4
def test_it(self):
from google.cloud._helpers import _datetime_from_microseconds
col_fam1 = u"col-fam-id"
col_name1 = b"col-name1"
col_name2 = b"col-name2"
cell_val1 = b"cell-val"
cell_val2 = b"cell-val-newer"
cell_val3 = b"altcol-cell-val"
microseconds = 5554441037
timestamp = _datetime_from_microseconds(microseconds)
expected_dict = {
col_name1: [(cell_val1, timestamp), (cell_val2, timestamp)],
col_name2: [(cell_val3, timestamp)],
}
expected_output = (col_fam1, expected_dict)
sample_input = _FamilyPB(
name=col_fam1,
columns=[
_ColumnPB(
qualifier=col_name1,
cells=[
_CellPB(value=cell_val1, timestamp_micros=microseconds),
_CellPB(value=cell_val2, timestamp_micros=microseconds),
],
),
_ColumnPB(qualifier=col_name2, cells=[_CellPB(value=cell_val3, timestamp_micros=microseconds)]),
],
)
self.assertEqual(expected_output, self._call_fut(sample_input))
def created(self):
"""Datetime at which the dataset was created.
:rtype: ``datetime.datetime``, or ``NoneType``
:returns: the creation time (None until set from the server).
"""
creation_time = self._properties.get('creationTime')
if creation_time is not None:
# creation_time will be in milliseconds.
return _datetime_from_microseconds(1000.0 * creation_time)
def expires(self):
"""Datetime at which the table will be removed.
:rtype: ``datetime.datetime``, or ``NoneType``
:returns: the expiration time, or None
"""
expiration_time = self._properties.get('expirationTime')
if expiration_time is not None:
# expiration_time will be in milliseconds.
return _datetime_from_microseconds(1000.0 * expiration_time)
def modified(self):
"""Datetime at which the dataset was last modified.
:rtype: ``datetime.datetime``, or ``NoneType``
:returns: the modification time (None until set from the server).
"""
modified_time = self._properties.get('lastModifiedTime')
if modified_time is not None:
# modified_time will be in milliseconds.
return _datetime_from_microseconds(1000.0 * modified_time)
def ended(self):
"""Datetime at which the job finished.
:rtype: ``datetime.datetime``, or ``NoneType``
:returns: the end time (None until set from the server).
"""
statistics = self._properties.get('statistics')
if statistics is not None:
millis = statistics.get('endTime')
if millis is not None:
return _datetime_from_microseconds(millis * 1000.0)
def created(self):
"""Datetime at which the job was created.
:rtype: ``datetime.datetime``, or ``NoneType``
:returns: the creation time (None until set from the server).
"""
statistics = self._properties.get('statistics')
if statistics is not None:
millis = statistics.get('creationTime')
if millis is not None:
return _datetime_from_microseconds(millis * 1000.0)
def from_pb(cls, cell_pb):
"""Create a new cell from a Cell protobuf.
:type cell_pb: :class:`._generated.data_pb2.Cell`
:param cell_pb: The protobuf to convert.
:rtype: :class:`Cell`
:returns: The cell corresponding to the protobuf.
"""
timestamp = _datetime_from_microseconds(cell_pb.timestamp_micros)
if cell_pb.labels:
return cls(cell_pb.value, timestamp, labels=cell_pb.labels)
else:
return cls(cell_pb.value, timestamp)
def from_pb(cls, cell_pb):
"""Create a new cell from a Cell protobuf.
:type cell_pb: :class:`._generated.data_pb2.Cell`
:param cell_pb: The protobuf to convert.
:rtype: :class:`Cell`
:returns: The cell corresponding to the protobuf.
"""
timestamp = _datetime_from_microseconds(cell_pb.timestamp_micros)
if cell_pb.labels:
return cls(cell_pb.value, timestamp, labels=cell_pb.labels)
else:
return cls(cell_pb.value, timestamp)
def test_it(self):
from google.cloud._helpers import _datetime_from_microseconds
col_fam1 = u'col-fam-id'
col_name1 = b'col-name1'
col_name2 = b'col-name2'
cell_val1 = b'cell-val'
cell_val2 = b'cell-val-newer'
cell_val3 = b'altcol-cell-val'
microseconds = 5554441037
timestamp = _datetime_from_microseconds(microseconds)
expected_dict = {
col_name1: [
(cell_val1, timestamp),
(cell_val2, timestamp),
],
col_name2: [
(cell_val3, timestamp),
],
}
expected_output = (col_fam1, expected_dict)
sample_input = _FamilyPB(
name=col_fam1,
columns=[
_ColumnPB(
qualifier=col_name1,
cells=[
_CellPB(
value=cell_val1,
timestamp_micros=microseconds,
),
_CellPB(
value=cell_val2,
timestamp_micros=microseconds,
),
],
),
_ColumnPB(
qualifier=col_name2,
cells=[
_CellPB(
value=cell_val3,
timestamp_micros=microseconds,
),
],
),
],
)
self.assertEqual(expected_output, self._call_fut(sample_input))
def test_constructor_explicit(self):
from google.cloud._helpers import _datetime_from_microseconds
from google.cloud.bigtable.row_filters import TimestampRange
table = object()
timestamp = 144185290431
batch_size = 42
transaction = False # Must be False when batch_size is non-null
batch = self._make_one(
table, timestamp=timestamp,
batch_size=batch_size, transaction=transaction)
self.assertEqual(batch._table, table)
self.assertEqual(batch._batch_size, batch_size)
self.assertEqual(batch._timestamp,
_datetime_from_microseconds(1000 * timestamp))
next_timestamp = _datetime_from_microseconds(1000 * (timestamp + 1))
time_range = TimestampRange(end=next_timestamp)
self.assertEqual(batch._delete_range, time_range)
self.assertEqual(batch._transaction, transaction)
self.assertEqual(batch._row_map, {})
self.assertEqual(batch._mutation_count, 0)
def _write_to_row(row1, row2, row3, row4):
from google.cloud._helpers import _datetime_from_microseconds
from google.cloud._helpers import _microseconds_from_datetime
from google.cloud._helpers import UTC
from google.cloud.bigtable.row_data import Cell
timestamp1 = datetime.datetime.utcnow().replace(tzinfo=UTC)
timestamp1_micros = _microseconds_from_datetime(timestamp1)
# Truncate to millisecond granularity.
timestamp1_micros -= timestamp1_micros % 1000
timestamp1 = _datetime_from_microseconds(timestamp1_micros)
# 1000 microseconds is a millisecond
timestamp2 = timestamp1 + datetime.timedelta(microseconds=1000)
timestamp2_micros = _microseconds_from_datetime(timestamp2)
timestamp3 = timestamp1 + datetime.timedelta(microseconds=2000)
timestamp3_micros = _microseconds_from_datetime(timestamp3)
timestamp4 = timestamp1 + datetime.timedelta(microseconds=3000)
timestamp4_micros = _microseconds_from_datetime(timestamp4)
if row1 is not None:
row1.set_cell(COLUMN_FAMILY_ID1,
COL_NAME1,
CELL_VAL1,
timestamp=timestamp1)
if row2 is not None:
row2.set_cell(COLUMN_FAMILY_ID1,
COL_NAME1,
CELL_VAL2,
timestamp=timestamp2)
if row3 is not None:
row3.set_cell(COLUMN_FAMILY_ID1,
COL_NAME2,
CELL_VAL3,
timestamp=timestamp3)
if row4 is not None:
row4.set_cell(COLUMN_FAMILY_ID2,
COL_NAME3,
CELL_VAL4,
timestamp=timestamp4)
# Create the cells we will check.
cell1 = Cell(CELL_VAL1, timestamp1_micros)
cell2 = Cell(CELL_VAL2, timestamp2_micros)
cell3 = Cell(CELL_VAL3, timestamp3_micros)
cell4 = Cell(CELL_VAL4, timestamp4_micros)
return cell1, cell2, cell3, cell4
def test_it(self):
from google.cloud._helpers import _datetime_from_microseconds
col_fam1 = u"col-fam-id"
col_fam2 = u"col-fam-id2"
col_name1 = b"col-name1"
col_name2 = b"col-name2"
col_name3 = b"col-name3-but-other-fam"
cell_val1 = b"cell-val"
cell_val2 = b"cell-val-newer"
cell_val3 = b"altcol-cell-val"
cell_val4 = b"foo"
microseconds = 1000871
timestamp = _datetime_from_microseconds(microseconds)
expected_output = {
col_fam1: {
col_name1: [(cell_val1, timestamp), (cell_val2, timestamp)],
col_name2: [(cell_val3, timestamp)],
},
col_fam2: {col_name3: [(cell_val4, timestamp)]},
}
response_row = _RowPB(
families=[
_FamilyPB(
name=col_fam1,
columns=[
_ColumnPB(
qualifier=col_name1,
cells=[
_CellPB(value=cell_val1, timestamp_micros=microseconds),
_CellPB(value=cell_val2, timestamp_micros=microseconds),
],
),
_ColumnPB(qualifier=col_name2, cells=[_CellPB(value=cell_val3, timestamp_micros=microseconds)]),
],
),
_FamilyPB(
name=col_fam2,
columns=[
_ColumnPB(qualifier=col_name3, cells=[_CellPB(value=cell_val4, timestamp_micros=microseconds)])
],
),
]
)
sample_input = _ReadModifyWriteRowResponsePB(row=response_row)
self.assertEqual(expected_output, self._call_fut(sample_input))
def test_with_timestamp(self):
from google.cloud._helpers import _datetime_from_microseconds
from google.cloud.bigtable.row_filters import TimestampRange
from google.cloud.bigtable.row_filters import TimestampRangeFilter
timestamp = 1441928298571
result = self._column_helper(num_filters=3, timestamp=timestamp)
range_filter = result.filters[2]
self.assertTrue(isinstance(range_filter, TimestampRangeFilter))
# Relies on the fact that RowFilter instances can
# only have one value set.
time_range = range_filter.range_
self.assertTrue(isinstance(time_range, TimestampRange))
self.assertEqual(time_range.start, None)
ts_dt = _datetime_from_microseconds(1000 * timestamp)
self.assertEqual(time_range.end, ts_dt)
def _write_to_row(self, row1=None, row2=None, row3=None, row4=None):
timestamp1 = datetime.datetime.utcnow().replace(tzinfo=UTC)
timestamp1_micros = _microseconds_from_datetime(timestamp1)
# Truncate to millisecond granularity.
timestamp1_micros -= timestamp1_micros % 1000
timestamp1 = _datetime_from_microseconds(timestamp1_micros)
# 1000 microseconds is a millisecond
timestamp2 = timestamp1 + datetime.timedelta(microseconds=1000)
timestamp2_micros = _microseconds_from_datetime(timestamp2)
timestamp3 = timestamp1 + datetime.timedelta(microseconds=2000)
timestamp3_micros = _microseconds_from_datetime(timestamp3)
timestamp4 = timestamp1 + datetime.timedelta(microseconds=3000)
timestamp4_micros = _microseconds_from_datetime(timestamp4)
if row1 is not None:
row1.set_cell(COLUMN_FAMILY_ID1,
COL_NAME1,
CELL_VAL1,
timestamp=timestamp1)
if row2 is not None:
row2.set_cell(COLUMN_FAMILY_ID1,
COL_NAME1,
CELL_VAL2,
timestamp=timestamp2)
if row3 is not None:
row3.set_cell(COLUMN_FAMILY_ID1,
COL_NAME2,
CELL_VAL3,
timestamp=timestamp3)
if row4 is not None:
row4.set_cell(COLUMN_FAMILY_ID2,
COL_NAME3,
CELL_VAL4,
timestamp=timestamp4)
# Create the cells we will check.
cell1 = Cell(CELL_VAL1, timestamp1_micros)
cell2 = Cell(CELL_VAL2, timestamp2_micros)
cell3 = Cell(CELL_VAL3, timestamp3_micros)
cell4 = Cell(CELL_VAL4, timestamp4_micros)
return cell1, cell2, cell3, cell4
def _convert_to_time_range(timestamp=None):
"""Create a timestamp range from an HBase / HappyBase timestamp.
HBase uses timestamp as an argument to specify an exclusive end
deadline. Cloud Bigtable also uses exclusive end times, so
the behavior matches.
:type timestamp: int
:param timestamp: (Optional) Timestamp (in milliseconds since the
epoch). Intended to be used as the end of an HBase
time range, which is exclusive.
:rtype: :class:`~google.cloud.bigtable.row.TimestampRange`,
:data:`NoneType <types.NoneType>`
:returns: The timestamp range corresponding to the passed in
``timestamp``.
"""
if timestamp is None:
return None
next_timestamp = _datetime_from_microseconds(1000 * timestamp)
return TimestampRange(end=next_timestamp)
def __init__(
self,
table,
timestamp=None,
batch_size=None,
transaction=False,
wal=_WAL_SENTINEL,
):
if wal is not _WAL_SENTINEL:
warnings.warn(_WAL_WARNING)
if batch_size is not None:
if transaction:
raise TypeError("When batch_size is set, a Batch cannot be "
"transactional")
if batch_size <= 0:
raise ValueError("batch_size must be positive")
self._table = table
self._batch_size = batch_size
self._timestamp = self._delete_range = None
# Timestamp is in milliseconds, convert to microseconds.
if timestamp is not None:
self._timestamp = _datetime_from_microseconds(1000 * timestamp)
# For deletes, we get the very next timestamp (assuming timestamp
# granularity is milliseconds). This is because HappyBase users
# expect HBase deletes to go **up to** and **including** the
# timestamp while Cloud Bigtable Time Ranges **exclude** the
# final timestamp.
next_timestamp = self._timestamp + _ONE_MILLISECOND
self._delete_range = TimestampRange(end=next_timestamp)
self._transaction = transaction
# Internal state for tracking mutations.
self._row_map = {}
self._mutation_count = 0
def test__parse_family_pb():
from google.cloud._helpers import _datetime_from_microseconds
from google.cloud.bigtable.row import _parse_family_pb
col_fam1 = u"col-fam-id"
col_name1 = b"col-name1"
col_name2 = b"col-name2"
cell_val1 = b"cell-val"
cell_val2 = b"cell-val-newer"
cell_val3 = b"altcol-cell-val"
microseconds = 5554441037
timestamp = _datetime_from_microseconds(microseconds)
expected_dict = {
col_name1: [(cell_val1, timestamp), (cell_val2, timestamp)],
col_name2: [(cell_val3, timestamp)],
}
expected_output = (col_fam1, expected_dict)
sample_input = _FamilyPB(
name=col_fam1,
columns=[
_ColumnPB(
qualifier=col_name1,
cells=[
_CellPB(value=cell_val1, timestamp_micros=microseconds),
_CellPB(value=cell_val2, timestamp_micros=microseconds),
],
),
_ColumnPB(
qualifier=col_name2,
cells=[
_CellPB(value=cell_val3, timestamp_micros=microseconds)
],
),
],
)
assert expected_output == _parse_family_pb(sample_input)
def _parse_family_pb(family_pb):
"""Parses a Family protobuf into a dictionary.
:type family_pb: :class:`._generated.data_pb2.Family`
:param family_pb: A protobuf
:rtype: tuple
:returns: A string and dictionary. The string is the name of the
column family and the dictionary has column names (within the
family) as keys and cell lists as values. Each cell is
represented with a two-tuple with the value (in bytes) and the
timestamp for the cell. For example:
.. code:: python
{
b'col-name1': [
(b'cell-val', datetime.datetime(...)),
(b'cell-val-newer', datetime.datetime(...)),
],
b'col-name2': [
(b'altcol-cell-val', datetime.datetime(...)),
],
}
"""
result = {}
for column in family_pb.columns:
result[column.qualifier] = cells = []
for cell in column.cells:
val_pair = (
cell.value,
_datetime_from_microseconds(cell.timestamp_micros),
)
cells.append(val_pair)
return family_pb.name, result
def _timestamp_from_json(value, field):
"""Coerce 'value' to a datetime, if set or not nullable."""
if _not_null(value, field):
# value will be a float in seconds, to microsecond precision, in UTC.
return _datetime_from_microseconds(1e6 * float(value))
def __init__(self, resource):
self.estimated_bytes = int(resource['estimatedBytes'])
self.estimated_rows = int(resource['estimatedRows'])
# time is in milliseconds since the epoch.
self.oldest_entry_time = _datetime_from_microseconds(
1000.0 * int(resource['oldestEntryTime']))
def test_it(self):
from google.cloud._helpers import _datetime_from_microseconds
col_fam1 = u'col-fam-id'
col_fam2 = u'col-fam-id2'
col_name1 = b'col-name1'
col_name2 = b'col-name2'
col_name3 = b'col-name3-but-other-fam'
cell_val1 = b'cell-val'
cell_val2 = b'cell-val-newer'
cell_val3 = b'altcol-cell-val'
cell_val4 = b'foo'
microseconds = 1000871
timestamp = _datetime_from_microseconds(microseconds)
expected_output = {
col_fam1: {
col_name1: [
(cell_val1, timestamp),
(cell_val2, timestamp),
],
col_name2: [
(cell_val3, timestamp),
],
},
col_fam2: {
col_name3: [
(cell_val4, timestamp),
],
},
}
response_row = _RowPB(families=[
_FamilyPB(
name=col_fam1,
columns=[
_ColumnPB(
qualifier=col_name1,
cells=[
_CellPB(
value=cell_val1,
timestamp_micros=microseconds,
),
_CellPB(
value=cell_val2,
timestamp_micros=microseconds,
),
],
),
_ColumnPB(
qualifier=col_name2,
cells=[
_CellPB(
value=cell_val3,
timestamp_micros=microseconds,
),
],
),
],
),
_FamilyPB(
name=col_fam2,
columns=[
_ColumnPB(
qualifier=col_name3,
cells=[
_CellPB(
value=cell_val4,
timestamp_micros=microseconds,
),
],
),
],
),
], )
sample_input = _ReadModifyWriteRowResponsePB(row=response_row)
self.assertEqual(expected_output, self._call_fut(sample_input))
def _call_fut(self, value):
from google.cloud._helpers import _datetime_from_microseconds
return _datetime_from_microseconds(value)
def __init__(self, resource):
self.estimated_bytes = int(resource['estimatedBytes'])
self.estimated_rows = int(resource['estimatedRows'])
# time is in milliseconds since the epoch.
self.oldest_entry_time = _datetime_from_microseconds(
1000.0 * int(resource['oldestEntryTime']))
def timestamp(self):
return _datetime_from_microseconds(self.timestamp_micros)
def _timestamp_from_json(value, field):
"""Coerce 'value' to a datetime, if set or not nullable."""
if _not_null(value, field):
# value will be a float in seconds, to microsecond precision, in UTC.
return _datetime_from_microseconds(1e6 * float(value))
def _call_fut(self, value):
from google.cloud._helpers import _datetime_from_microseconds
return _datetime_from_microseconds(value)
def timestamp(self):
return _datetime_from_microseconds(self.timestamp_micros)
def age_in_hours(micros):
return (
datetime.datetime.utcnow().replace(tzinfo=UTC) -
(_datetime_from_microseconds(micros))).total_seconds() // 3600
