def test_with_timestamp(self): from gcloud._helpers import _datetime_from_microseconds from gcloud.bigtable.row_data import Cell from gcloud.bigtable.row_data import PartialRowData row_data = PartialRowData(b"row-key") val1 = b"hi-im-bytes" ts1_millis = 1221934570148 ts1 = _datetime_from_microseconds(ts1_millis * 1000) val2 = b"bi-im-hytes" ts2_millis = 1331934880000 ts2 = _datetime_from_microseconds(ts2_millis * 1000) row_data._cells[u"fam1"] = {b"col1": [Cell(val1, ts1)], b"col2": [Cell(val2, ts2)]} result = self._callFUT(row_data, include_timestamp=True) expected_result = {b"fam1:col1": (val1, ts1_millis), b"fam1:col2": (val2, ts2_millis)} self.assertEqual(result, expected_result)
def _write_to_row(self, row1=None, row2=None, row3=None, row4=None): timestamp1 = datetime.datetime.utcnow().replace(tzinfo=UTC) timestamp1_micros = _microseconds_from_datetime(timestamp1) # Truncate to millisecond granularity. timestamp1_micros -= (timestamp1_micros % 1000) timestamp1 = _datetime_from_microseconds(timestamp1_micros) # 1000 microseconds is a millisecond timestamp2 = timestamp1 + datetime.timedelta(microseconds=1000) timestamp3 = timestamp1 + datetime.timedelta(microseconds=2000) timestamp4 = timestamp1 + datetime.timedelta(microseconds=3000) if row1 is not None: row1.set_cell(COLUMN_FAMILY_ID1, COL_NAME1, CELL_VAL1, timestamp=timestamp1) if row2 is not None: row2.set_cell(COLUMN_FAMILY_ID1, COL_NAME1, CELL_VAL2, timestamp=timestamp2) if row3 is not None: row3.set_cell(COLUMN_FAMILY_ID1, COL_NAME2, CELL_VAL3, timestamp=timestamp3) if row4 is not None: row4.set_cell(COLUMN_FAMILY_ID2, COL_NAME3, CELL_VAL4, timestamp=timestamp4) # Create the cells we will check. cell1 = Cell(CELL_VAL1, timestamp1) cell2 = Cell(CELL_VAL2, timestamp2) cell3 = Cell(CELL_VAL3, timestamp3) cell4 = Cell(CELL_VAL4, timestamp4) return cell1, cell2, cell3, cell4
def __init__(self, table, timestamp=None, batch_size=None, transaction=False, wal=_WAL_SENTINEL): if wal is not _WAL_SENTINEL: _WARN(_WAL_WARNING) if batch_size is not None: if transaction: raise TypeError('When batch_size is set, a Batch cannot be ' 'transactional') if batch_size <= 0: raise ValueError('batch_size must be positive') self._table = table self._batch_size = batch_size self._timestamp = self._delete_range = None # Timestamp is in milliseconds, convert to microseconds. if timestamp is not None: self._timestamp = _datetime_from_microseconds(1000 * timestamp) # For deletes, we get the very next timestamp (assuming timestamp # granularity is milliseconds). This is because HappyBase users # expect HBase deletes to go **up to** and **including** the # timestamp while Cloud Bigtable Time Ranges **exclude** the # final timestamp. next_timestamp = self._timestamp + _ONE_MILLISECOND self._delete_range = TimestampRange(end=next_timestamp) self._transaction = transaction # Internal state for tracking mutations. self._row_map = {} self._mutation_count = 0
def test_with_timestamp(self): from gcloud._helpers import _datetime_from_microseconds from gcloud.bigtable.row_data import Cell value1 = "foo" ts1_millis = 1221934570148 ts1 = _datetime_from_microseconds(ts1_millis * 1000) cell1 = Cell(value=value1, timestamp=ts1) value2 = "bar" ts2_millis = 1221955575548 ts2 = _datetime_from_microseconds(ts2_millis * 1000) cell2 = Cell(value=value2, timestamp=ts2) result = self._callFUT([cell1, cell2], include_timestamp=True) self.assertEqual(result, [(value1, ts1_millis), (value2, ts2_millis)])
def test_success(self): from gcloud._helpers import _datetime_from_microseconds from gcloud.bigtable.row import TimestampRange timestamp = 1441928298571 ts_dt = _datetime_from_microseconds(1000 * timestamp) result = self._callFUT(timestamp=timestamp) self.assertTrue(isinstance(result, TimestampRange)) self.assertEqual(result.start, None) self.assertEqual(result.end, ts_dt)
def expires(self): """Datetime at which the table will be removed. :rtype: ``datetime.datetime``, or ``NoneType`` :returns: the expiration time, or None """ expiration_time = self._properties.get('expirationTime') if expiration_time is not None: # expiration_time will be in milliseconds. return _datetime_from_microseconds(1000.0 * expiration_time)
def created(self): """Datetime at which the table was created. :rtype: ``datetime.datetime``, or ``NoneType`` :returns: the creation time (None until set from the server). """ creation_time = self._properties.get('creationTime') if creation_time is not None: # creation_time will be in milliseconds. return _datetime_from_microseconds(1000.0 * creation_time)
def modified(self): """Datetime at which the table was last modified. :rtype: ``datetime.datetime``, or ``NoneType`` :returns: the modification time (None until set from the server). """ modified_time = self._properties.get('lastModifiedTime') if modified_time is not None: # modified_time will be in milliseconds. return _datetime_from_microseconds(1000.0 * modified_time)
def created(self): """Datetime at which the job was created. :rtype: ``datetime.datetime``, or ``NoneType`` :returns: the creation time (None until set from the server). """ statistics = self._properties.get('statistics') if statistics is not None: millis = statistics.get('creationTime') if millis is not None: return _datetime_from_microseconds(millis * 1000.0)
def ended(self): """Datetime at which the job finished. :rtype: ``datetime.datetime``, or ``NoneType`` :returns: the end time (None until set from the server). """ statistics = self._properties.get('statistics') if statistics is not None: millis = statistics.get('endTime') if millis is not None: return _datetime_from_microseconds(millis * 1000.0)
def ended(self): """Datetime at which the job finished. :rtype: ``datetime.datetime``, or ``NoneType`` :returns: the end time (None until set from the server). """ statistics = self._properties.get("statistics") if statistics is not None: millis = statistics.get("endTime") if millis is not None: return _datetime_from_microseconds(millis * 1000.0)
def time_enqueued(self): """Retrieve the timestamp at which the task was enqueued. See: https://cloud.google.com/appengine/docs/python/taskqueue/rest/tasks :rtype: :class:`datetime.datetime` or ``NoneType`` :returns: Datetime object parsed from microsecond timestamp, or ``None`` if the property is not set locally. """ value = self._properties.get('enqueueTimestamp') if value is not None: return _datetime_from_microseconds(int(value))
def test_constructor_explicit(self): from gcloud._helpers import _datetime_from_microseconds from gcloud.bigtable.row_filters import TimestampRange table = object() timestamp = 144185290431 batch_size = 42 transaction = False # Must be False when batch_size is non-null batch = self._makeOne(table, timestamp=timestamp, batch_size=batch_size, transaction=transaction) self.assertEqual(batch._table, table) self.assertEqual(batch._batch_size, batch_size) self.assertEqual(batch._timestamp, _datetime_from_microseconds(1000 * timestamp)) next_timestamp = _datetime_from_microseconds(1000 * (timestamp + 1)) time_range = TimestampRange(end=next_timestamp) self.assertEqual(batch._delete_range, time_range) self.assertEqual(batch._transaction, transaction) self.assertEqual(batch._row_map, {}) self.assertEqual(batch._mutation_count, 0)
def test_it(self): from gcloud._helpers import _datetime_from_microseconds from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 col_fam1 = u'col-fam-id' col_name1 = b'col-name1' col_name2 = b'col-name2' cell_val1 = b'cell-val' cell_val2 = b'cell-val-newer' cell_val3 = b'altcol-cell-val' microseconds = 5554441037 timestamp = _datetime_from_microseconds(microseconds) expected_dict = { col_name1: [ (cell_val1, timestamp), (cell_val2, timestamp), ], col_name2: [ (cell_val3, timestamp), ], } expected_output = (col_fam1, expected_dict) sample_input = data_pb2.Family( name=col_fam1, columns=[ data_pb2.Column( qualifier=col_name1, cells=[ data_pb2.Cell( value=cell_val1, timestamp_micros=microseconds, ), data_pb2.Cell( value=cell_val2, timestamp_micros=microseconds, ), ], ), data_pb2.Column( qualifier=col_name2, cells=[ data_pb2.Cell( value=cell_val3, timestamp_micros=microseconds, ), ], ), ], ) self.assertEqual(expected_output, self._callFUT(sample_input))
def from_pb(cls, cell_pb): """Create a new cell from a Cell protobuf. :type cell_pb: :class:`._generated_v2.data_pb2.Cell` :param cell_pb: The protobuf to convert. :rtype: :class:`Cell` :returns: The cell corresponding to the protobuf. """ timestamp = _datetime_from_microseconds(cell_pb.timestamp_micros) if cell_pb.labels: return cls(cell_pb.value, timestamp, labels=cell_pb.labels) else: return cls(cell_pb.value, timestamp)
def leaseTimestamp(self): """Retrieve the timestamp at which the task lease will expire. If this task has never been leased, it will be None. If this this task has been previously leased and the lease has expired, this value will be < Now(). See: https://cloud.google.com/appengine/docs/python/taskqueue/rest/tasks :rtype: :class:`datetime.datetime` or ``NoneType`` :returns: Datetime object parsed from microsecond timestamp, or ``None`` if the property is not set locally. If the task has not been leased, this will never be set. """ value = self._properties.get('leaseTimestamp') if value is not None: return _datetime_from_microseconds(int(value))
def test_constructor_explicit(self): from gcloud._helpers import _datetime_from_microseconds from gcloud.bigtable.row import TimestampRange table = object() timestamp = 144185290431 batch_size = 42 transaction = False # Must be False when batch_size is non-null batch = self._makeOne(table, timestamp=timestamp, batch_size=batch_size, transaction=transaction) self.assertEqual(batch._table, table) self.assertEqual(batch._batch_size, batch_size) self.assertEqual(batch._timestamp, _datetime_from_microseconds(1000 * timestamp)) next_timestamp = _datetime_from_microseconds(1000 * (timestamp + 1)) time_range = TimestampRange(end=next_timestamp) self.assertEqual(batch._delete_range, time_range) self.assertEqual(batch._transaction, transaction) self.assertEqual(batch._row_map, {}) self.assertEqual(batch._mutation_count, 0)
def _get_value_from_value_pb(value_pb): """Given a protobuf for a Value, get the correct value. The Cloud Datastore Protobuf API returns a Property Protobuf which has one value set and the rest blank. This function retrieves the the one value provided. Some work is done to coerce the return value into a more useful type (particularly in the case of a timestamp value, or a key value). :type value_pb: :class:`gcloud.datastore._generated.entity_pb2.Value` :param value_pb: The Value Protobuf. :returns: The value provided by the Protobuf. """ result = None # Simple field (int64) if value_pb.HasField('timestamp_microseconds_value'): microseconds = value_pb.timestamp_microseconds_value result = _datetime_from_microseconds(microseconds) elif value_pb.HasField('key_value'): # Message field (Key) result = key_from_protobuf(value_pb.key_value) elif value_pb.HasField('boolean_value'): # Simple field (bool) result = value_pb.boolean_value elif value_pb.HasField('double_value'): # Simple field (double) result = value_pb.double_value elif value_pb.HasField('integer_value'): # Simple field (int64) result = value_pb.integer_value elif value_pb.HasField('string_value'): # Simple field (string) result = value_pb.string_value elif value_pb.HasField('blob_value'): # Simple field (bytes) result = value_pb.blob_value elif value_pb.HasField('entity_value'): # Message field (Entity) result = entity_from_protobuf(value_pb.entity_value) elif value_pb.list_value: result = [ _get_value_from_value_pb(value) for value in value_pb.list_value ] return result
def _get_value_from_value_pb(value_pb): """Given a protobuf for a Value, get the correct value. The Cloud Datastore Protobuf API returns a Property Protobuf which has one value set and the rest blank. This function retrieves the the one value provided. Some work is done to coerce the return value into a more useful type (particularly in the case of a timestamp value, or a key value). :type value_pb: :class:`gcloud.datastore._generated.entity_pb2.Value` :param value_pb: The Value Protobuf. :returns: The value provided by the Protobuf. """ result = None # Simple field (int64) if value_pb.HasField('timestamp_microseconds_value'): microseconds = value_pb.timestamp_microseconds_value result = _datetime_from_microseconds(microseconds) elif value_pb.HasField('key_value'): # Message field (Key) result = key_from_protobuf(value_pb.key_value) elif value_pb.HasField('boolean_value'): # Simple field (bool) result = value_pb.boolean_value elif value_pb.HasField('double_value'): # Simple field (double) result = value_pb.double_value elif value_pb.HasField('integer_value'): # Simple field (int64) result = value_pb.integer_value elif value_pb.HasField('string_value'): # Simple field (string) result = value_pb.string_value elif value_pb.HasField('blob_value'): # Simple field (bytes) result = value_pb.blob_value elif value_pb.HasField('entity_value'): # Message field (Entity) result = entity_from_protobuf(value_pb.entity_value) elif value_pb.list_value: result = [_get_value_from_value_pb(value) for value in value_pb.list_value] return result
def test_with_timestamp(self): from gcloud._helpers import _datetime_from_microseconds from gcloud.bigtable.row import TimestampRange from gcloud.bigtable.row import TimestampRangeFilter timestamp = 1441928298571 result = self._column_helper(num_filters=3, timestamp=timestamp) range_filter = result.filters[2] self.assertTrue(isinstance(range_filter, TimestampRangeFilter)) # Relies on the fact that RowFilter instances can # only have one value set. time_range = range_filter.range_ self.assertTrue(isinstance(time_range, TimestampRange)) self.assertEqual(time_range.start, None) ts_dt = _datetime_from_microseconds(1000 * timestamp) self.assertEqual(time_range.end, ts_dt)
def _convert_to_time_range(timestamp=None): """Create a timestamp range from an HBase / HappyBase timestamp. HBase uses timestamp as an argument to specify an exclusive end deadline. Cloud Bigtable also uses exclusive end times, so the behavior matches. :type timestamp: int :param timestamp: (Optional) Timestamp (in milliseconds since the epoch). Intended to be used as the end of an HBase time range, which is exclusive. :rtype: :class:`.TimestampRange`, :data:`NoneType <types.NoneType>` :returns: The timestamp range corresponding to the passed in ``timestamp``. """ if timestamp is None: return None next_timestamp = _datetime_from_microseconds(1000 * timestamp) return TimestampRange(end=next_timestamp)
def _parse_family_pb(family_pb): """Parses a Family protobuf into a dictionary. :type family_pb: :class:`._generated.bigtable_data_pb2.Family` :param family_pb: A protobuf :rtype: tuple :returns: A string and dictionary. The string is the name of the column family and the dictionary has column names (within the family) as keys and cell lists as values. Each cell is represented with a two-tuple with the value (in bytes) and the timestamp for the cell. For example: .. code:: python { b'col-name1': [ (b'cell-val', datetime.datetime(...)), (b'cell-val-newer', datetime.datetime(...)), ], b'col-name2': [ (b'altcol-cell-val', datetime.datetime(...)), ], } """ result = {} for column in family_pb.columns: result[column.qualifier] = cells = [] for cell in column.cells: val_pair = ( cell.value, _datetime_from_microseconds(cell.timestamp_micros), ) cells.append(val_pair) return family_pb.name, result
def _datetime_from_json(value, field): if _not_null(value, field): # Field value will be in milliseconds. return _datetime_from_microseconds(1000.0 * float(value))
def _callFUT(self, value): from gcloud._helpers import _datetime_from_microseconds return _datetime_from_microseconds(value)
def test_it(self): from gcloud._helpers import _datetime_from_microseconds col_fam1 = u'col-fam-id' col_fam2 = u'col-fam-id2' col_name1 = b'col-name1' col_name2 = b'col-name2' col_name3 = b'col-name3-but-other-fam' cell_val1 = b'cell-val' cell_val2 = b'cell-val-newer' cell_val3 = b'altcol-cell-val' cell_val4 = b'foo' microseconds = 1000871 timestamp = _datetime_from_microseconds(microseconds) expected_output = { col_fam1: { col_name1: [ (cell_val1, timestamp), (cell_val2, timestamp), ], col_name2: [ (cell_val3, timestamp), ], }, col_fam2: { col_name3: [ (cell_val4, timestamp), ], }, } response_row = _RowPB( families=[ _FamilyPB( name=col_fam1, columns=[ _ColumnPB( qualifier=col_name1, cells=[ _CellPB( value=cell_val1, timestamp_micros=microseconds, ), _CellPB( value=cell_val2, timestamp_micros=microseconds, ), ], ), _ColumnPB( qualifier=col_name2, cells=[ _CellPB( value=cell_val3, timestamp_micros=microseconds, ), ], ), ], ), _FamilyPB( name=col_fam2, columns=[ _ColumnPB( qualifier=col_name3, cells=[ _CellPB( value=cell_val4, timestamp_micros=microseconds, ), ], ), ], ), ], ) sample_input = _ReadModifyWriteRowResponsePB(row=response_row) self.assertEqual(expected_output, self._callFUT(sample_input))
def _datetime_from_json(value, field): if _not_null(value, field): # value will be a float in seconds, to microsecond precision, in UTC. return _datetime_from_microseconds(1e6 * float(value))
def test_it(self): from gcloud._helpers import _datetime_from_microseconds from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 col_fam1 = u'col-fam-id' col_fam2 = u'col-fam-id2' col_name1 = b'col-name1' col_name2 = b'col-name2' col_name3 = b'col-name3-but-other-fam' cell_val1 = b'cell-val' cell_val2 = b'cell-val-newer' cell_val3 = b'altcol-cell-val' cell_val4 = b'foo' microseconds = 1000871 timestamp = _datetime_from_microseconds(microseconds) expected_output = { col_fam1: { col_name1: [ (cell_val1, timestamp), (cell_val2, timestamp), ], col_name2: [ (cell_val3, timestamp), ], }, col_fam2: { col_name3: [ (cell_val4, timestamp), ], }, } sample_input = data_pb2.Row( families=[ data_pb2.Family( name=col_fam1, columns=[ data_pb2.Column( qualifier=col_name1, cells=[ data_pb2.Cell( value=cell_val1, timestamp_micros=microseconds, ), data_pb2.Cell( value=cell_val2, timestamp_micros=microseconds, ), ], ), data_pb2.Column( qualifier=col_name2, cells=[ data_pb2.Cell( value=cell_val3, timestamp_micros=microseconds, ), ], ), ], ), data_pb2.Family( name=col_fam2, columns=[ data_pb2.Column( qualifier=col_name3, cells=[ data_pb2.Cell( value=cell_val4, timestamp_micros=microseconds, ), ], ), ], ), ], ) self.assertEqual(expected_output, self._callFUT(sample_input))
def _datetime_from_json(value, field): """Coerce 'value' to a datetime, if set or not nullable.""" if _not_null(value, field): # value will be a float in seconds, to microsecond precision, in UTC. return _datetime_from_microseconds(1e6 * float(value))