def _write_to_row(self, row1=None, row2=None, row3=None, row4=None):
timestamp1 = datetime.datetime.utcnow().replace(tzinfo=UTC)
timestamp1_micros = _microseconds_from_datetime(timestamp1)
# Truncate to millisecond granularity.
timestamp1_micros -= timestamp1_micros % 1000
timestamp1 = _datetime_from_microseconds(timestamp1_micros)
# 1000 microseconds is a millisecond
timestamp2 = timestamp1 + datetime.timedelta(microseconds=1000)
timestamp2_micros = _microseconds_from_datetime(timestamp2)
timestamp3 = timestamp1 + datetime.timedelta(microseconds=2000)
timestamp3_micros = _microseconds_from_datetime(timestamp3)
timestamp4 = timestamp1 + datetime.timedelta(microseconds=3000)
timestamp4_micros = _microseconds_from_datetime(timestamp4)
if row1 is not None:
row1.set_cell(COLUMN_FAMILY_ID1, COL_NAME1, CELL_VAL1, timestamp=timestamp1)
if row2 is not None:
row2.set_cell(COLUMN_FAMILY_ID1, COL_NAME1, CELL_VAL2, timestamp=timestamp2)
if row3 is not None:
row3.set_cell(COLUMN_FAMILY_ID1, COL_NAME2, CELL_VAL3, timestamp=timestamp3)
if row4 is not None:
row4.set_cell(COLUMN_FAMILY_ID2, COL_NAME3, CELL_VAL4, timestamp=timestamp4)
# Create the cells we will check.
cell1 = Cell(CELL_VAL1, timestamp1_micros)
cell2 = Cell(CELL_VAL2, timestamp2_micros)
cell3 = Cell(CELL_VAL3, timestamp3_micros)
cell4 = Cell(CELL_VAL4, timestamp4_micros)
return cell1, cell2, cell3, cell4
def _write_to_row(self, row1=None, row2=None, row3=None, row4=None):
timestamp1 = datetime.datetime.utcnow().replace(tzinfo=UTC)
timestamp1_micros = _microseconds_from_datetime(timestamp1)
# Truncate to millisecond granularity.
timestamp1_micros -= timestamp1_micros % 1000
timestamp1 = _datetime_from_microseconds(timestamp1_micros)
# 1000 microseconds is a millisecond
timestamp2 = timestamp1 + datetime.timedelta(microseconds=1000)
timestamp2_micros = _microseconds_from_datetime(timestamp2)
timestamp3 = timestamp1 + datetime.timedelta(microseconds=2000)
timestamp3_micros = _microseconds_from_datetime(timestamp3)
timestamp4 = timestamp1 + datetime.timedelta(microseconds=3000)
timestamp4_micros = _microseconds_from_datetime(timestamp4)
if row1 is not None:
row1.set_cell(COLUMN_FAMILY_ID1, COL_NAME1, CELL_VAL1, timestamp=timestamp1)
if row2 is not None:
row2.set_cell(COLUMN_FAMILY_ID1, COL_NAME1, CELL_VAL2, timestamp=timestamp2)
if row3 is not None:
row3.set_cell(COLUMN_FAMILY_ID1, COL_NAME2, CELL_VAL3, timestamp=timestamp3)
if row4 is not None:
row4.set_cell(COLUMN_FAMILY_ID2, COL_NAME3, CELL_VAL4, timestamp=timestamp4)
# Create the cells we will check.
cell1 = Cell(CELL_VAL1, timestamp1_micros)
cell2 = Cell(CELL_VAL2, timestamp2_micros)
cell3 = Cell(CELL_VAL3, timestamp3_micros)
cell4 = Cell(CELL_VAL4, timestamp4_micros)
return cell1, cell2, cell3, cell4
def to_pb(self):
"""Converts the :class:`TimestampRange` to a protobuf.
:rtype: :class:`.data_v2_pb2.TimestampRange`
:returns: The converted current object.
"""
timestamp_range_kwargs = {}
if self.start is not None:
timestamp_range_kwargs['start_timestamp_micros'] = (
_microseconds_from_datetime(self.start))
if self.end is not None:
timestamp_range_kwargs['end_timestamp_micros'] = (
_microseconds_from_datetime(self.end))
return data_v2_pb2.TimestampRange(**timestamp_range_kwargs)
def to_pb(self):
"""Converts the :class:`TimestampRange` to a protobuf.
:rtype: :class:`.data_v2_pb2.TimestampRange`
:returns: The converted current object.
"""
timestamp_range_kwargs = {}
if self.start is not None:
timestamp_range_kwargs['start_timestamp_micros'] = (
_microseconds_from_datetime(self.start))
if self.end is not None:
timestamp_range_kwargs['end_timestamp_micros'] = (
_microseconds_from_datetime(self.end))
return data_v2_pb2.TimestampRange(**timestamp_range_kwargs)
def test_w_datetime(self):
import datetime
from google.cloud._helpers import _microseconds_from_datetime
when = datetime.datetime(2016, 12, 20, 15, 58, 27, 339328)
self.assertEqual(
self._call_fut(when), _microseconds_from_datetime(when) / 1e6)
def to_pb(self):
"""Converts the :class:`TimestampRange` to a protobuf.
:rtype: :class:`.data_v2_pb2.TimestampRange`
:returns: The converted current object.
"""
timestamp_range_kwargs = {}
if self.start is not None:
timestamp_range_kwargs["start_timestamp_micros"] = (
_microseconds_from_datetime(self.start) // 1000 * 1000)
if self.end is not None:
end_time = _microseconds_from_datetime(self.end)
if end_time % 1000 != 0:
end_time = end_time // 1000 * 1000 + 1000
timestamp_range_kwargs["end_timestamp_micros"] = end_time
return data_v2_pb2.TimestampRange(**timestamp_range_kwargs)
def get_expiration_seconds(expiration):
"""Convert 'expiration' to a number of seconds in the future.
:type expiration: int, long, datetime.datetime, datetime.timedelta
:param expiration: When the signed URL should expire.
:raises TypeError: When expiration is not an integer.
:rtype: int
:returns: a timestamp as an absolute number of seconds.
"""
# If it's a timedelta, add it to `now` in UTC.
if isinstance(expiration, datetime.timedelta):
now = NOW().replace(tzinfo=_helpers.UTC)
expiration = now + expiration
# If it's a datetime, convert to a timestamp.
if isinstance(expiration, datetime.datetime):
micros = _helpers._microseconds_from_datetime(expiration)
expiration = micros // 10**6
if not isinstance(expiration, six.integer_types):
raise TypeError('Expected an integer timestamp, datetime, or '
'timedelta. Got %s' % type(expiration))
return expiration
def _get_expiration_seconds(expiration):
"""Convert 'expiration' to a number of seconds in the future.
:type expiration: int, long, datetime.datetime, datetime.timedelta
:param expiration: When the signed URL should expire.
:raises TypeError: When expiration is not an integer.
:rtype: int
:returns: a timestamp as an absolute number of seconds.
"""
# If it's a timedelta, add it to `now` in UTC.
if isinstance(expiration, datetime.timedelta):
now = _NOW().replace(tzinfo=UTC)
expiration = now + expiration
# If it's a datetime, convert to a timestamp.
if isinstance(expiration, datetime.datetime):
micros = _microseconds_from_datetime(expiration)
expiration = micros // 10**6
if not isinstance(expiration, six.integer_types):
raise TypeError('Expected an integer timestamp, datetime, or '
'timedelta. Got %s' % type(expiration))
return expiration
def test_w_datetime(self):
import datetime
from google.cloud._helpers import UTC
from google.cloud._helpers import _microseconds_from_datetime
when = datetime.datetime(2016, 12, 3, 14, 11, 27, tzinfo=UTC)
self.assertEqual(self._call_fut(when),
_microseconds_from_datetime(when) / 1e6)
def get_expiration_seconds_v2(expiration):
"""Convert 'expiration' to a number of seconds in the future.
:type expiration: Union[Integer, datetime.datetime, datetime.timedelta]
:param expiration: Point in time when the signed URL should expire. If
a ``datetime`` instance is passed without an explicit
``tzinfo`` set, it will be assumed to be ``UTC``.
:raises: :exc:`TypeError` when expiration is not a valid type.
:rtype: int
:returns: a timestamp as an absolute number of seconds since epoch.
"""
# If it's a timedelta, add it to `now` in UTC.
if isinstance(expiration, datetime.timedelta):
now = NOW().replace(tzinfo=_helpers.UTC)
expiration = now + expiration
# If it's a datetime, convert to a timestamp.
if isinstance(expiration, datetime.datetime):
micros = _helpers._microseconds_from_datetime(expiration)
expiration = micros // 10**6
if not isinstance(expiration, six.integer_types):
raise TypeError("Expected an integer timestamp, datetime, or "
"timedelta. Got %s" % type(expiration))
return expiration
def test_w_datetime(self):
import datetime
from google.cloud._helpers import UTC
from google.cloud._helpers import _microseconds_from_datetime
when = datetime.datetime(2016, 12, 3, 14, 11, 27, tzinfo=UTC)
self.assertEqual(self._call_fut(when),
_microseconds_from_datetime(when) / 1e6)
def get_expiration_seconds_v2(expiration):
"""Convert 'expiration' to a number of seconds in the future.
:type expiration: Union[Integer, datetime.datetime, datetime.timedelta]
:param expiration: Point in time when the signed URL should expire.
:raises: :exc:`TypeError` when expiration is not a valid type.
:rtype: int
:returns: a timestamp as an absolute number of seconds since epoch.
"""
# If it's a timedelta, add it to `now` in UTC.
if isinstance(expiration, datetime.timedelta):
now = NOW().replace(tzinfo=_helpers.UTC)
expiration = now + expiration
# If it's a datetime, convert to a timestamp.
if isinstance(expiration, datetime.datetime):
micros = _helpers._microseconds_from_datetime(expiration)
expiration = micros // 10 ** 6
if not isinstance(expiration, six.integer_types):
raise TypeError(
"Expected an integer timestamp, datetime, or "
"timedelta. Got %s" % type(expiration)
)
return expiration
def _write_to_row(row1, row2, row3, row4):
from google.cloud._helpers import _datetime_from_microseconds
from google.cloud._helpers import _microseconds_from_datetime
from google.cloud._helpers import UTC
from google.cloud.bigtable.row_data import Cell
timestamp1 = datetime.datetime.utcnow().replace(tzinfo=UTC)
timestamp1_micros = _microseconds_from_datetime(timestamp1)
# Truncate to millisecond granularity.
timestamp1_micros -= timestamp1_micros % 1000
timestamp1 = _datetime_from_microseconds(timestamp1_micros)
# 1000 microseconds is a millisecond
timestamp2 = timestamp1 + datetime.timedelta(microseconds=1000)
timestamp2_micros = _microseconds_from_datetime(timestamp2)
timestamp3 = timestamp1 + datetime.timedelta(microseconds=2000)
timestamp3_micros = _microseconds_from_datetime(timestamp3)
timestamp4 = timestamp1 + datetime.timedelta(microseconds=3000)
timestamp4_micros = _microseconds_from_datetime(timestamp4)
if row1 is not None:
row1.set_cell(COLUMN_FAMILY_ID1,
COL_NAME1,
CELL_VAL1,
timestamp=timestamp1)
if row2 is not None:
row2.set_cell(COLUMN_FAMILY_ID1,
COL_NAME1,
CELL_VAL2,
timestamp=timestamp2)
if row3 is not None:
row3.set_cell(COLUMN_FAMILY_ID1,
COL_NAME2,
CELL_VAL3,
timestamp=timestamp3)
if row4 is not None:
row4.set_cell(COLUMN_FAMILY_ID2,
COL_NAME3,
CELL_VAL4,
timestamp=timestamp4)
# Create the cells we will check.
cell1 = Cell(CELL_VAL1, timestamp1_micros)
cell2 = Cell(CELL_VAL2, timestamp2_micros)
cell3 = Cell(CELL_VAL3, timestamp3_micros)
cell4 = Cell(CELL_VAL4, timestamp4_micros)
return cell1, cell2, cell3, cell4
def _timestamp_to_json_row(value):
"""Coerce 'value' to an JSON-compatible representation.
This version returns floating-point seconds value used in row data.
"""
if isinstance(value, datetime.datetime):
value = _microseconds_from_datetime(value) * 1e-6
return value
def to_pb(self):
"""Converts the :class:`TimestampRange` to a protobuf.
:rtype: :class:`.data_v2_pb2.TimestampRange`
:returns: The converted current object.
"""
timestamp_range_kwargs = {}
if self.start is not None:
timestamp_range_kwargs["start_timestamp_micros"] = (
_microseconds_from_datetime(self.start) // 1000 * 1000
)
if self.end is not None:
end_time = _microseconds_from_datetime(self.end)
if end_time % 1000 != 0:
end_time = end_time // 1000 * 1000 + 1000
timestamp_range_kwargs["end_timestamp_micros"] = end_time
return data_v2_pb2.TimestampRange(**timestamp_range_kwargs)
def test_it(self):
import datetime
from google.cloud._helpers import UTC
from google.cloud._helpers import _microseconds_from_datetime
NOW = datetime.datetime(2015, 7, 29, 17, 45, 21, 123456, tzinfo=UTC)
NOW_MICROS = _microseconds_from_datetime(NOW)
self.assertEqual(self._call_fut(NOW_MICROS), NOW)
def _timestamp_to_json_row(value):
"""Coerce 'value' to an JSON-compatible representation.
This version returns floating-point seconds value used in row data.
"""
if isinstance(value, datetime.datetime):
value = _microseconds_from_datetime(value) * 1e-6
return value
def test_it(self):
import datetime
from google.cloud._helpers import UTC
from google.cloud._helpers import _microseconds_from_datetime
NOW = datetime.datetime(2015, 7, 29, 17, 45, 21, 123456, tzinfo=UTC)
NOW_MICROS = _microseconds_from_datetime(NOW)
self.assertEqual(self._call_fut(NOW_MICROS), NOW)
def _set_cell(self,
column_family_id,
column,
value,
timestamp=None,
state=None):
"""Helper for :meth:`set_cell`
Adds a mutation to set the value in a specific cell.
``state`` is unused by :class:`DirectRow` but is used by
subclasses.
:type column_family_id: str
:param column_family_id: The column family that contains the column.
Must be of the form
``[_a-zA-Z0-9][-_.a-zA-Z0-9]*``.
:type column: bytes
:param column: The column within the column family where the cell
is located.
:type value: bytes or :class:`int`
:param value: The value to set in the cell. If an integer is used,
will be interpreted as a 64-bit big-endian signed
integer (8 bytes).
:type timestamp: :class:`datetime.datetime`
:param timestamp: (Optional) The timestamp of the operation.
:type state: bool
:param state: (Optional) The state that is passed along to
:meth:`_get_mutations`.
"""
column = _to_bytes(column)
if isinstance(value, six.integer_types):
value = _PACK_I64(value)
value = _to_bytes(value)
if timestamp is None:
# Use -1 for current Bigtable server time.
timestamp_micros = -1
else:
timestamp_micros = _microseconds_from_datetime(timestamp)
# Truncate to millisecond granularity.
timestamp_micros -= (timestamp_micros % 1000)
mutation_val = data_v2_pb2.Mutation.SetCell(
family_name=column_family_id,
column_qualifier=column,
timestamp_micros=timestamp_micros,
value=value,
)
mutation_pb = data_v2_pb2.Mutation(set_cell=mutation_val)
self._get_mutations(state).append(mutation_pb)
def test_w_utc_datetime(self):
import datetime
from google.cloud._helpers import UTC
from google.cloud._helpers import _microseconds_from_datetime
NOW = datetime.datetime.utcnow().replace(tzinfo=UTC)
NOW_MICROS = _microseconds_from_datetime(NOW)
MILLIS = NOW_MICROS // 1000
result = self._call_fut(NOW)
self.assertIsInstance(result, int)
self.assertEqual(result, MILLIS)
def test_w_non_utc_datetime(self):
import datetime
from google.cloud._helpers import _microseconds_from_datetime
offset = datetime.timedelta(hours=-1)
zone = datetime.timezone(offset=offset, name="CET")
NOW = datetime.datetime(2015, 7, 28, 16, 34, 47, tzinfo=zone)
NOW_MICROS = _microseconds_from_datetime(NOW)
MILLIS = NOW_MICROS // 1000
result = self._call_fut(NOW)
self.assertIsInstance(result, int)
self.assertEqual(result, MILLIS)
def test_w_utc_datetime(self):
import datetime
import six
from google.cloud._helpers import UTC
from google.cloud._helpers import _microseconds_from_datetime
NOW = datetime.datetime.utcnow().replace(tzinfo=UTC)
NOW_MICROS = _microseconds_from_datetime(NOW)
MILLIS = NOW_MICROS // 1000
result = self._call_fut(NOW)
self.assertIsInstance(result, six.integer_types)
self.assertEqual(result, MILLIS)
def get_expiration_seconds_v4(expiration, max_age):
"""Convert 'expiration' to a number of seconds offset from the current time.
:type expiration: Union[Integer, datetime.datetime, datetime.timedelta]
:param expiration: Point in time when the signed URL should expire.
Exclusive with :arg:`max_age`: exactly one of the
two must be passed.
:type max_age: Integer
:param max_age: Max number of seconds until the signature expires.
Exclusive with :arg:`expiration`: exactly one of the
two must be passed.
:raises: :exc:`ValueError` when both :arg:`expiration` and
:arg:`max_age` are passed, or when neither is passed.
:raises: :exc:`TypeError` when expiration is not a valid type.
:raises: :exc:`ValueError` when expiration is too large.
:rtype: Integer
:returns: seconds in the future when the signed URL will expire
"""
if (expiration is None and max_age is None) or (expiration is not None
and max_age is not None):
raise ValueError("Pass exactly one of 'expiration' or 'max_age'.")
if max_age is not None:
return max_age
if not isinstance(expiration, _EXPIRATION_TYPES):
raise TypeError("Expected an integer timestamp, datetime, or "
"timedelta. Got %s" % type(expiration))
now = NOW().replace(tzinfo=_helpers.UTC)
if isinstance(expiration, six.integer_types):
now_seconds = _helpers._microseconds_from_datetime(now) // 10**6
seconds = expiration - now_seconds
if isinstance(expiration, datetime.datetime):
if expiration.tzinfo is None:
expiration = expiration.replace(tzinfo=_helpers.UTC)
expiration = expiration - now
if isinstance(expiration, datetime.timedelta):
seconds = int(expiration.total_seconds())
if seconds > SEVEN_DAYS:
raise ValueError(
"Max allowed expiration interval is seven days (%d seconds)".
format(SEVEN_DAYS))
return seconds
def test_to_api_repr_w_timestamp_micros(self):
from google.cloud._helpers import _microseconds_from_datetime
now = datetime.datetime.utcnow()
seconds = _microseconds_from_datetime(now) / 1.0e6
EXPECTED = {
"parameterType": {"type": "TIMESTAMP"},
"parameterValue": {"value": seconds},
}
klass = self._get_target_class()
param = klass.positional(type_="TIMESTAMP", value=seconds)
self.assertEqual(param.to_api_repr(), EXPECTED)
def test_to_api_repr_w_timestamp_micros(self):
from google.cloud._helpers import _microseconds_from_datetime
now = datetime.datetime.utcnow()
seconds = _microseconds_from_datetime(now) / 1.0e6
EXPECTED = {
"parameterType": {"type": "TIMESTAMP"},
"parameterValue": {"value": seconds},
}
klass = self._get_target_class()
param = klass.positional(type_="TIMESTAMP", value=seconds)
self.assertEqual(param.to_api_repr(), EXPECTED)
def test_w_utc_datetime(self):
import datetime
import six
from google.cloud._helpers import UTC
from google.cloud._helpers import _microseconds_from_datetime
NOW = datetime.datetime.utcnow().replace(tzinfo=UTC)
NOW_MICROS = _microseconds_from_datetime(NOW)
MILLIS = NOW_MICROS // 1000
result = self._callFUT(NOW)
self.assertTrue(isinstance(result, six.integer_types))
self.assertEqual(result, MILLIS)
def test_w_naive_datetime(self):
import datetime
import six
from google.cloud._helpers import UTC
from google.cloud._helpers import _microseconds_from_datetime
NOW = datetime.datetime.utcnow()
UTC_NOW = NOW.replace(tzinfo=UTC)
UTC_NOW_MICROS = _microseconds_from_datetime(UTC_NOW)
MILLIS = UTC_NOW_MICROS // 1000
result = self._callFUT(NOW)
self.assertTrue(isinstance(result, six.integer_types))
self.assertEqual(result, MILLIS)
def test_w_naive_datetime(self):
import datetime
import six
from google.cloud._helpers import UTC
from google.cloud._helpers import _microseconds_from_datetime
NOW = datetime.datetime.utcnow()
UTC_NOW = NOW.replace(tzinfo=UTC)
UTC_NOW_MICROS = _microseconds_from_datetime(UTC_NOW)
MILLIS = UTC_NOW_MICROS // 1000
result = self._call_fut(NOW)
self.assertIsInstance(result, six.integer_types)
self.assertEqual(result, MILLIS)
def _set_cell(self, column_family_id, column, value, timestamp=None,
state=None):
"""Helper for :meth:`set_cell`
Adds a mutation to set the value in a specific cell.
``state`` is unused by :class:`DirectRow` but is used by
subclasses.
:type column_family_id: str
:param column_family_id: The column family that contains the column.
Must be of the form
``[_a-zA-Z0-9][-_.a-zA-Z0-9]*``.
:type column: bytes
:param column: The column within the column family where the cell
is located.
:type value: bytes or :class:`int`
:param value: The value to set in the cell. If an integer is used,
will be interpreted as a 64-bit big-endian signed
integer (8 bytes).
:type timestamp: :class:`datetime.datetime`
:param timestamp: (Optional) The timestamp of the operation.
:type state: bool
:param state: (Optional) The state that is passed along to
:meth:`_get_mutations`.
"""
column = _to_bytes(column)
if isinstance(value, six.integer_types):
value = _PACK_I64(value)
value = _to_bytes(value)
if timestamp is None:
# Use -1 for current Bigtable server time.
timestamp_micros = -1
else:
timestamp_micros = _microseconds_from_datetime(timestamp)
# Truncate to millisecond granularity.
timestamp_micros -= (timestamp_micros % 1000)
mutation_val = data_v2_pb2.Mutation.SetCell(
family_name=column_family_id,
column_qualifier=column,
timestamp_micros=timestamp_micros,
value=value,
)
mutation_pb = data_v2_pb2.Mutation(set_cell=mutation_val)
self._get_mutations(state).append(mutation_pb)
def test_to_api_repr_w_timestamp_micros(self):
import datetime
from google.cloud._helpers import _microseconds_from_datetime
now = datetime.datetime.utcnow()
seconds = _microseconds_from_datetime(now) / 1.0e6
EXPECTED = {
'parameterType': {
'type': 'TIMESTAMP',
},
'parameterValue': {
'value': seconds,
},
}
klass = self._get_target_class()
param = klass.positional(type_='TIMESTAMP', value=seconds)
self.assertEqual(param.to_api_repr(), EXPECTED)
def test_to_api_repr_w_timestamp_micros(self):
import datetime
from google.cloud._helpers import _microseconds_from_datetime
now = datetime.datetime.utcnow()
seconds = _microseconds_from_datetime(now) / 1.0e6
EXPECTED = {
'parameterType': {
'type': 'TIMESTAMP',
},
'parameterValue': {
'value': seconds,
},
}
klass = self._get_target_class()
param = klass.positional(type_='TIMESTAMP', value=seconds)
self.assertEqual(param.to_api_repr(), EXPECTED)
def test_w_non_utc_datetime(self):
import datetime
import six
from google.cloud._helpers import _UTC
from google.cloud._helpers import _microseconds_from_datetime
class CET(_UTC):
_tzname = 'CET'
_utcoffset = datetime.timedelta(hours=-1)
zone = CET()
NOW = datetime.datetime(2015, 7, 28, 16, 34, 47, tzinfo=zone)
NOW_MICROS = _microseconds_from_datetime(NOW)
MILLIS = NOW_MICROS // 1000
result = self._call_fut(NOW)
self.assertIsInstance(result, six.integer_types)
self.assertEqual(result, MILLIS)
def test_w_non_utc_datetime(self):
import datetime
import six
from google.cloud._helpers import _UTC
from google.cloud._helpers import _microseconds_from_datetime
class CET(_UTC):
_tzname = 'CET'
_utcoffset = datetime.timedelta(hours=-1)
zone = CET()
NOW = datetime.datetime(2015, 7, 28, 16, 34, 47, tzinfo=zone)
NOW_MICROS = _microseconds_from_datetime(NOW)
MILLIS = NOW_MICROS // 1000
result = self._call_fut(NOW)
self.assertIsInstance(result, six.integer_types)
self.assertEqual(result, MILLIS)
def _flatten_cells(prd):
# Match results format from JSON testcases.
# Doesn't handle error cases.
from google.cloud._helpers import _bytes_to_unicode
from google.cloud._helpers import _microseconds_from_datetime
for row_key, row in prd.rows.items():
for family_name, family in row.cells.items():
for qualifier, column in family.items():
for cell in column:
yield {
u'rk': _bytes_to_unicode(row_key),
u'fm': family_name,
u'qual': _bytes_to_unicode(qualifier),
u'ts': _microseconds_from_datetime(cell.timestamp),
u'value': _bytes_to_unicode(cell.value),
u'label': u' '.join(cell.labels),
u'error': False,
}
def _flatten_cells(prd):
# Match results format from JSON testcases.
# Doesn't handle error cases.
from google.cloud._helpers import _bytes_to_unicode
from google.cloud._helpers import _microseconds_from_datetime
for row_key, row in prd.rows.items():
for family_name, family in row.cells.items():
for qualifier, column in family.items():
for cell in column:
yield {
u'rk': _bytes_to_unicode(row_key),
u'fm': family_name,
u'qual': _bytes_to_unicode(qualifier),
u'ts': _microseconds_from_datetime(cell.timestamp),
u'value': _bytes_to_unicode(cell.value),
u'label': u' '.join(cell.labels),
u'error': False,
}
def _flatten_cells(prd):
# Match results format from JSON testcases.
# Doesn't handle error cases.
from google.cloud._helpers import _bytes_to_unicode
from google.cloud._helpers import _microseconds_from_datetime
for row_key, row in prd.rows.items():
for family_name, family in row.cells.items():
for qualifier, column in family.items():
for cell in column:
yield {
u"rk": _bytes_to_unicode(row_key),
u"fm": family_name,
u"qual": _bytes_to_unicode(qualifier),
u"ts": _microseconds_from_datetime(cell.timestamp),
u"value": _bytes_to_unicode(cell.value),
u"label": u" ".join(cell.labels),
u"error": False,
}
def _flatten_cells(prd):
# Match results format from JSON testcases.
# Doesn't handle error cases.
from google.cloud._helpers import _bytes_to_unicode
from google.cloud._helpers import _microseconds_from_datetime
for row_key, row in prd.rows.items():
for family_name, family in row.cells.items():
for qualifier, column in family.items():
for cell in column:
yield {
u"rk": _bytes_to_unicode(row_key),
u"fm": family_name,
u"qual": _bytes_to_unicode(qualifier),
u"ts": _microseconds_from_datetime(cell.timestamp),
u"value": _bytes_to_unicode(cell.value),
u"label": u" ".join(cell.labels),
u"error": False,
}
def get_expiration_seconds_v2(expiration, max_age):
"""Convert 'expiration' to a number of seconds in the future.
:type expiration: Union[Integer, datetime.datetime, datetime.timedelta]
:param expiration: Point in time when the signed URL should expire.
Exclusive with :arg:`max_age`: exactly one of the
two must be passed.
:type max_age: Integer
:param max_age: Max number of seconds until the signature expires.
Exclusive with :arg:`expiration`: exactly one of the
two must be passed.
:raises: :exc:`ValueError` when both :arg:`expiration` and
:arg:`max_age` are passed, or when neither is passed.
:raises: :exc:`TypeError` when expiration is not a valid type.
:rtype: int
:returns: a timestamp as an absolute number of seconds since epoch.
"""
if (expiration is None and max_age is None) or (expiration is not None
and max_age is not None):
raise ValueError("Pass exactly one of 'expiration' or 'max_age'.")
if max_age is not None:
expiration = datetime.timedelta(seconds=max_age)
# If it's a timedelta, add it to `now` in UTC.
if isinstance(expiration, datetime.timedelta):
now = NOW().replace(tzinfo=_helpers.UTC)
expiration = now + expiration
# If it's a datetime, convert to a timestamp.
if isinstance(expiration, datetime.datetime):
micros = _helpers._microseconds_from_datetime(expiration)
expiration = micros // 10**6
if not isinstance(expiration, six.integer_types):
raise TypeError("Expected an integer timestamp, datetime, or "
"timedelta. Got %s" % type(expiration))
return expiration
def _cells_to_pairs(cells, include_timestamp=False):
"""Converts list of cells to HappyBase format.
For example::
>>> import datetime
>>> from google.cloud.bigtable.row_data import Cell
>>> cell1 = Cell(b'val1', datetime.datetime.utcnow())
>>> cell2 = Cell(b'val2', datetime.datetime.utcnow())
>>> _cells_to_pairs([cell1, cell2])
[b'val1', b'val2']
>>> _cells_to_pairs([cell1, cell2], include_timestamp=True)
[(b'val1', 1456361486255), (b'val2', 1456361491927)]
:type cells: list
:param cells: List of :class:`~google.cloud.bigtable.row_data.Cell`
returned from a read request.
:type include_timestamp: bool
:param include_timestamp: Flag to indicate if cell timestamps should be
included with the output.
:rtype: list
:returns: List of values in the cell. If ``include_timestamp=True``, each
value will be a pair, with the first part the bytes value in
the cell and the second part the number of milliseconds in the
timestamp on the cell.
"""
result = []
for cell in cells:
if include_timestamp:
ts_millis = _microseconds_from_datetime(cell.timestamp) // 1000
result.append((cell.value, ts_millis))
else:
result.append(cell.value)
return result
def insert_data(self,
rows,
row_ids=None,
skip_invalid_rows=None,
ignore_unknown_values=None,
template_suffix=None,
client=None):
"""API call: insert table data via a POST request
See:
https://cloud.google.com/bigquery/docs/reference/v2/tabledata/insertAll
:type rows: list of tuples
:param rows: Row data to be inserted. Each tuple should contain data
for each schema field on the current table and in the
same order as the schema fields.
:type row_ids: list of string
:param row_ids: Unique ids, one per row being inserted. If not
passed, no de-duplication occurs.
:type skip_invalid_rows: boolean or ``NoneType``
:param skip_invalid_rows: skip rows w/ invalid data?
:type ignore_unknown_values: boolean or ``NoneType``
:param ignore_unknown_values: ignore columns beyond schema?
:type template_suffix: str or ``NoneType``
:param template_suffix: treat ``name`` as a template table and provide
a suffix. BigQuery will create the table
``<name> + <template_suffix>`` based on the
schema of the template table. See:
https://cloud.google.com/bigquery/streaming-data-into-bigquery#template-tables
:type client: :class:`~google.cloud.bigquery.client.Client` or
``NoneType``
:param client: the client to use. If not passed, falls back to the
``client`` stored on the current dataset.
:rtype: list of mappings
:returns: One mapping per row with insert errors: the "index" key
identifies the row, and the "errors" key contains a list
of the mappings describing one or more problems with the
row.
"""
client = self._require_client(client)
rows_info = []
data = {'rows': rows_info}
for index, row in enumerate(rows):
row_info = {}
for field, value in zip(self._schema, row):
if field.field_type == 'TIMESTAMP' and value is not None:
# BigQuery stores TIMESTAMP data internally as a
# UNIX timestamp with microsecond precision.
# Specifies the number of seconds since the epoch.
value = _microseconds_from_datetime(value) * 1e-6
row_info[field.name] = value
info = {'json': row_info}
if row_ids is not None:
info['insertId'] = row_ids[index]
rows_info.append(info)
if skip_invalid_rows is not None:
data['skipInvalidRows'] = skip_invalid_rows
if ignore_unknown_values is not None:
data['ignoreUnknownValues'] = ignore_unknown_values
if template_suffix is not None:
data['templateSuffix'] = template_suffix
response = client.connection.api_request(
method='POST',
path='%s/insertAll' % self.path,
data=data)
errors = []
for error in response.get('insertErrors', ()):
errors.append({'index': int(error['index']),
'errors': error['errors']})
return errors
def _call_fut(self, value):
from google.cloud._helpers import _microseconds_from_datetime
return _microseconds_from_datetime(value)
def _timestamp_to_json(value):
"""Coerce 'value' to an JSON-compatible representation."""
if isinstance(value, datetime.datetime):
value = _microseconds_from_datetime(value) / 1.0e6
return value
from google.cloud._helpers import _microseconds_from_datetime
from google.cloud._helpers import UTC
from google.cloud.bigtable import row, column_family, Client
except ImportError:
Client = None
UTC = pytz.utc
_microseconds_from_datetime = lambda label_stamp: label_stamp
_datetime_from_microseconds = lambda micro: micro
if TYPE_CHECKING:
import google.cloud.bigtable.instance
EXISTING_INSTANCES = [] # type: List[google.cloud.bigtable.instance.Instance]
LABEL_KEY = u'python-bigtable-beam'
label_stamp = datetime.datetime.utcnow().replace(tzinfo=UTC)
label_stamp_micros = _microseconds_from_datetime(label_stamp)
LABELS = {LABEL_KEY: str(label_stamp_micros)}
class GenerateTestRows(beam.PTransform):
""" A transform test to run write to the Bigtable Table.
A PTransform that generate a list of `DirectRow` to write it in
Bigtable Table.
"""
def __init__(self,
number,
project_id=None,
instance_id=None,
table_id=None):
def test_w_datetime(self):
from google.cloud._helpers import _microseconds_from_datetime
when = datetime.datetime(2016, 12, 20, 15, 58, 27, 339328)
self.assertEqual(self._call_fut(when), _microseconds_from_datetime(when) / 1e6)
def _call_fut(self, value):
from google.cloud._helpers import _microseconds_from_datetime
return _microseconds_from_datetime(value)
def insert_data(self,
rows,
row_ids=None,
skip_invalid_rows=None,
ignore_unknown_values=None,
template_suffix=None,
client=None):
"""API call: insert table data via a POST request
See:
https://cloud.google.com/bigquery/docs/reference/v2/tabledata/insertAll
:type rows: list of tuples
:param rows: Row data to be inserted. Each tuple should contain data
for each schema field on the current table and in the
same order as the schema fields.
:type row_ids: list of string
:param row_ids: Unique ids, one per row being inserted. If not
passed, no de-duplication occurs.
:type skip_invalid_rows: bool
:param skip_invalid_rows: (Optional) skip rows w/ invalid data?
:type ignore_unknown_values: bool
:param ignore_unknown_values: (Optional) ignore columns beyond schema?
:type template_suffix: str
:param template_suffix:
(Optional) treat ``name`` as a template table and provide a suffix.
BigQuery will create the table ``<name> + <template_suffix>`` based
on the schema of the template table. See:
https://cloud.google.com/bigquery/streaming-data-into-bigquery#template-tables
:type client: :class:`~google.cloud.bigquery.client.Client` or
``NoneType``
:param client: the client to use. If not passed, falls back to the
``client`` stored on the current dataset.
:rtype: list of mappings
:returns: One mapping per row with insert errors: the "index" key
identifies the row, and the "errors" key contains a list
of the mappings describing one or more problems with the
row.
:raises: ValueError if table's schema is not set
"""
if len(self._schema) == 0:
raise ValueError(_TABLE_HAS_NO_SCHEMA)
client = self._require_client(client)
rows_info = []
data = {'rows': rows_info}
for index, row in enumerate(rows):
row_info = {}
for field, value in zip(self._schema, row):
if field.field_type == 'TIMESTAMP' and value is not None:
# BigQuery stores TIMESTAMP data internally as a
# UNIX timestamp with microsecond precision.
# Specifies the number of seconds since the epoch.
value = _microseconds_from_datetime(value) * 1e-6
row_info[field.name] = value
info = {'json': row_info}
if row_ids is not None:
info['insertId'] = row_ids[index]
rows_info.append(info)
if skip_invalid_rows is not None:
data['skipInvalidRows'] = skip_invalid_rows
if ignore_unknown_values is not None:
data['ignoreUnknownValues'] = ignore_unknown_values
if template_suffix is not None:
data['templateSuffix'] = template_suffix
response = client.connection.api_request(method='POST',
path='%s/insertAll' %
self.path,
data=data)
errors = []
for error in response.get('insertErrors', ()):
errors.append({
'index': int(error['index']),
'errors': error['errors']
})
return errors
def _convert_timestamp(value):
"""Helper for :meth:`Table.insert_data`."""
if isinstance(value, datetime.datetime):
value = _microseconds_from_datetime(value) * 1e-6
return value
def _timestamp_to_json(value):
"""Coerce 'value' to an JSON-compatible representation."""
if isinstance(value, datetime.datetime):
value = _microseconds_from_datetime(value) / 1.0e6
return value
def _convert_timestamp(value):
"""Helper for :meth:`Table.insert_data`."""
if isinstance(value, datetime.datetime):
value = _microseconds_from_datetime(value) * 1e-6
return value