def test__merge_chunk_array_of_array_of_string(self):
from google.cloud.spanner_v1.proto.type_pb2 import StructType
from google.cloud.spanner_v1.proto.type_pb2 import Type
subarray_type = Type(code="ARRAY",
array_element_type=Type(code="STRING"))
array_type = Type(code="ARRAY", array_element_type=subarray_type)
iterator = _MockCancellableIterator()
streamed = self._make_one(iterator)
FIELDS = [StructType.Field(name="lolos", type=array_type)]
streamed._metadata = self._make_result_set_metadata(FIELDS)
streamed._pending_chunk = self._make_list_value(value_pbs=[
self._make_list_value([u"A", u"B"]),
self._make_list_value([u"C"]),
])
chunk = self._make_list_value(value_pbs=[
self._make_list_value([u"D"]),
self._make_list_value([u"E", u"F"]),
])
merged = streamed._merge_chunk(chunk)
expected = self._make_list_value(value_pbs=[
self._make_list_value([u"A", u"B"]),
self._make_list_value([u"CD"]),
self._make_list_value([u"E", u"F"]),
])
self.assertEqual(merged, expected)
self.assertIsNone(streamed._pending_chunk)
def test__merge_chunk_array_of_array_of_int(self):
from google.cloud.spanner_v1.proto.type_pb2 import StructType
from google.cloud.spanner_v1.proto.type_pb2 import Type
subarray_type = Type(code='ARRAY',
array_element_type=Type(code='INT64'))
array_type = Type(code='ARRAY', array_element_type=subarray_type)
iterator = _MockCancellableIterator()
streamed = self._make_one(iterator)
FIELDS = [StructType.Field(name='loloi', type=array_type)]
streamed._metadata = self._make_result_set_metadata(FIELDS)
streamed._pending_chunk = self._make_list_value(value_pbs=[
self._make_list_value([0, 1]),
self._make_list_value([2]),
])
chunk = self._make_list_value(value_pbs=[
self._make_list_value([3]),
self._make_list_value([4, 5]),
])
merged = streamed._merge_chunk(chunk)
expected = self._make_list_value(value_pbs=[
self._make_list_value([0, 1]),
self._make_list_value([23]),
self._make_list_value([4, 5]),
])
self.assertEqual(merged, expected)
self.assertIsNone(streamed._pending_chunk)
def _make_array_field(name, element_type_code=None, element_type=None):
from google.cloud.spanner_v1.proto.type_pb2 import StructType
from google.cloud.spanner_v1.proto.type_pb2 import Type
if element_type is None:
element_type = Type(code=element_type_code)
array_type = Type(code="ARRAY", array_element_type=element_type)
return StructType.Field(name=name, type=array_type)
def test_w_array_empty(self):
from google.protobuf.struct_pb2 import Value
from google.cloud.spanner_v1.proto.type_pb2 import Type, ARRAY, INT64
field_type = Type(code=ARRAY, array_element_type=Type(code=INT64))
value_pb = Value()
self.assertEqual(self._callFUT(value_pb, field_type), [])
def _make_struct_type(struct_type_fields):
from google.cloud.spanner_v1.proto.type_pb2 import StructType
from google.cloud.spanner_v1.proto.type_pb2 import Type
fields = [
StructType.Field(name=key, type=Type(code=value))
for key, value in struct_type_fields
]
struct_type = StructType(fields=fields)
return Type(code="STRUCT", struct_type=struct_type)
def test_empty(self):
from google.cloud.spanner_v1.proto.type_pb2 import Type, StructType
from google.cloud.spanner_v1.proto.type_pb2 import STRING, INT64
struct_type_pb = StructType(fields=[
StructType.Field(name="name", type=Type(code=STRING)),
StructType.Field(name="age", type=Type(code=INT64)),
])
self.assertEqual(self._callFUT(rows=[], row_type=struct_type_pb), [])
def test_w_array_non_empty(self):
from google.protobuf.struct_pb2 import Value, ListValue
from google.cloud.spanner_v1.proto.type_pb2 import Type, ARRAY, INT64
field_type = Type(code=ARRAY, array_element_type=Type(code=INT64))
VALUES = [32, 19, 5]
values_pb = ListValue(
values=[Value(string_value=str(value)) for value in VALUES])
value_pb = Value(list_value=values_pb)
self.assertEqual(self._callFUT(value_pb, field_type), VALUES)
def test_non_empty(self):
from google.cloud.spanner_v1.proto.type_pb2 import Type, StructType
from google.cloud.spanner_v1.proto.type_pb2 import STRING, INT64
from google.cloud.spanner_v1._helpers import _make_list_value_pbs
VALUES = [[u"phred", 32], [u"bharney", 31]]
struct_type_pb = StructType(fields=[
StructType.Field(name="name", type=Type(code=STRING)),
StructType.Field(name="age", type=Type(code=INT64)),
])
values_pbs = _make_list_value_pbs(VALUES)
self.assertEqual(
self._callFUT(rows=values_pbs, row_type=struct_type_pb), VALUES)
def test_w_struct(self):
from google.protobuf.struct_pb2 import Value
from google.cloud.spanner_v1.proto.type_pb2 import Type, StructType
from google.cloud.spanner_v1.proto.type_pb2 import STRUCT, STRING, INT64
from google.cloud.spanner_v1._helpers import _make_list_value_pb
VALUES = [u"phred", 32]
struct_type_pb = StructType(fields=[
StructType.Field(name="name", type=Type(code=STRING)),
StructType.Field(name="age", type=Type(code=INT64)),
])
field_type = Type(code=STRUCT, struct_type=struct_type_pb)
value_pb = Value(list_value=_make_list_value_pb(VALUES))
self.assertEqual(self._callFUT(value_pb, field_type), VALUES)
def __init__(self, db_column, enum_list, null=True, default=None):
self.data_type = Type(code=STRING)
self.enum_list = enum_list
DataType.__init__(self,
db_column=db_column,
null=null,
default=default)
def test_w_null(self):
from google.protobuf.struct_pb2 import Value, NULL_VALUE
from google.cloud.spanner_v1.proto.type_pb2 import Type, STRING
field_type = Type(code=STRING)
value_pb = Value(null_value=NULL_VALUE)
self.assertEqual(self._callFUT(value_pb, field_type), None)
def test_w_float(self):
from google.protobuf.struct_pb2 import Value
from google.cloud.spanner_v1.proto.type_pb2 import Type, FLOAT64
VALUE = 3.14159
field_type = Type(code=FLOAT64)
value_pb = Value(number_value=VALUE)
self.assertEqual(self._callFUT(value_pb, field_type), VALUE)
def test_w_int(self):
from google.protobuf.struct_pb2 import Value
from google.cloud.spanner_v1.proto.type_pb2 import Type, INT64
VALUE = 12345
field_type = Type(code=INT64)
value_pb = Value(string_value=str(VALUE))
self.assertEqual(self._callFUT(value_pb, field_type), VALUE)
def test_w_bool(self):
from google.protobuf.struct_pb2 import Value
from google.cloud.spanner_v1.proto.type_pb2 import Type, BOOL
VALUE = True
field_type = Type(code=BOOL)
value_pb = Value(bool_value=VALUE)
self.assertEqual(self._callFUT(value_pb, field_type), VALUE)
def test_w_bytes(self):
from google.protobuf.struct_pb2 import Value
from google.cloud.spanner_v1.proto.type_pb2 import Type, BYTES
VALUE = b"Value"
field_type = Type(code=BYTES)
value_pb = Value(string_value=VALUE)
self.assertEqual(self._callFUT(value_pb, field_type), VALUE)
def test_w_string(self):
from google.protobuf.struct_pb2 import Value
from google.cloud.spanner_v1.proto.type_pb2 import Type, STRING
VALUE = u'Value'
field_type = Type(code=STRING)
value_pb = Value(string_value=VALUE)
self.assertEqual(self._callFUT(value_pb, field_type), VALUE)
def test_execute_sql_w_query_param_transfinite(self):
session = self._db.session()
session.create()
self.to_delete.append(session)
with session.batch() as batch:
batch.delete(self.ALL_TYPES_TABLE, self.ALL)
batch.insert(
self.ALL_TYPES_TABLE,
self.ALL_TYPES_COLUMNS,
self.ALL_TYPES_ROWDATA)
snapshot = session.snapshot(
read_timestamp=batch.committed, multi_use=True)
# Find -inf
self._check_sql_results(
snapshot,
sql='SELECT eye_d FROM all_types WHERE approx_value = @pos_inf',
params={'pos_inf': float('+inf')},
param_types={'pos_inf': Type(code=FLOAT64)},
expected=[(107,)],
)
# Find +inf
self._check_sql_results(
snapshot,
sql='SELECT eye_d FROM all_types WHERE approx_value = @neg_inf',
params={'neg_inf': float('-inf')},
param_types={'neg_inf': Type(code=FLOAT64)},
expected=[(207,)],
)
rows = list(snapshot.execute_sql(
'SELECT'
' [CAST("-inf" AS FLOAT64),'
' CAST("+inf" AS FLOAT64),'
' CAST("NaN" AS FLOAT64)]'))
self.assertEqual(len(rows), 1)
float_array, = rows[0]
self.assertEqual(float_array[0], float('-inf'))
self.assertEqual(float_array[1], float('+inf'))
# NaNs cannot be searched for by equality.
self.assertTrue(math.isnan(float_array[2]))
def test_w_unknown_type(self):
from google.protobuf.struct_pb2 import Value
from google.cloud.spanner_v1.proto.type_pb2 import Type
from google.cloud.spanner_v1.proto.type_pb2 import TYPE_CODE_UNSPECIFIED
field_type = Type(code=TYPE_CODE_UNSPECIFIED)
value_pb = Value(string_value="Borked")
with self.assertRaises(ValueError):
self._callFUT(value_pb, field_type)
def test_w_numeric(self):
import decimal
from google.protobuf.struct_pb2 import Value
from google.cloud.spanner_v1.proto.type_pb2 import Type, NUMERIC
VALUE = decimal.Decimal("99999999999999999999999999999.999999999")
field_type = Type(code=NUMERIC)
value_pb = Value(string_value=str(VALUE))
self.assertEqual(self._callFUT(value_pb, field_type), VALUE)
def test_w_date(self):
import datetime
from google.protobuf.struct_pb2 import Value
from google.cloud.spanner_v1.proto.type_pb2 import Type, DATE
VALUE = datetime.date.today()
field_type = Type(code=DATE)
value_pb = Value(string_value=VALUE.isoformat())
self.assertEqual(self._callFUT(value_pb, field_type), VALUE)
def _query_w_concurrent_update(self, transaction, pkey):
SQL = 'SELECT * FROM counters WHERE name = @name'
rows = list(transaction.execute_sql(
SQL,
params={'name': pkey},
param_types={'name': Type(code=STRING)},
))
self.assertEqual(len(rows), 1)
pkey, value = rows[0]
transaction.update(
COUNTERS_TABLE, COUNTERS_COLUMNS, [[pkey, value + 1]])
def __init__(self,
db_column,
max_length=None,
null=True,
default=None,
reg_exr=None):
self.data_type = Type(code=STRING)
self.max_length = max_length
self.reg_exr = reg_exr
DataType.__init__(self,
db_column=db_column,
null=null,
default=default)
def __init__(self,
db_column,
max_value=None,
min_value=None,
null=True,
default=None):
self.data_type = Type(code=INT64)
self.max_value = max_value
self.min_value = min_value
DataType.__init__(self,
db_column=db_column,
null=null,
default=default)
def execute_query(cls, query_string, params=None, transaction=None):
"""
Execute query string
:type query_string: str
:param query_string:
:type params: dict
:param params:
:type transaction: Transaction
:param transaction:
:rtype: list
:return:
"""
param_types = None
if params is not None:
param_types = {}
for key in params:
value = params.get(key)
if isinstance(value, six.string_types):
param_types[key] = Type(code=STRING)
elif isinstance(value, int):
param_types[key] = Type(code=INT64)
elif isinstance(value, float):
param_types[key] = Type(code=FLOAT64)
elif isinstance(value, bool):
param_types[key] = Type(code=BOOL)
elif isinstance(value, date):
param_types[key] = Type(code=DATE)
response = Executor.execute_query(query_string=query_string,
params=params,
param_types=param_types,
transaction=transaction)
return Helper.process_result_set(response)
def test_w_timestamp_w_nanos(self):
from google.protobuf.struct_pb2 import Value
from google.cloud.spanner_v1.proto.type_pb2 import Type, TIMESTAMP
from google.cloud._helpers import UTC, _datetime_to_rfc3339
from google.cloud.spanner_v1._helpers import TimestampWithNanoseconds
VALUE = TimestampWithNanoseconds(
2016, 12, 20, 21, 13, 47, nanosecond=123456789, tzinfo=UTC)
field_type = Type(code=TIMESTAMP)
value_pb = Value(string_value=_datetime_to_rfc3339(VALUE))
parsed = self._callFUT(value_pb, field_type)
self.assertIsInstance(parsed, TimestampWithNanoseconds)
self.assertEqual(parsed, VALUE)
def __init__(self,
db_column,
max_value=None,
min_value=None,
null=True,
default=None,
decimal_places=2):
self.data_type = Type(code=FLOAT64)
self.max_value = max_value
self.min_value = min_value
self.decimal_places = decimal_places
DataType.__init__(self,
db_column=db_column,
null=null,
default=default)
def test_w_timestamp_w_nanos(self):
import pytz
from google.protobuf.struct_pb2 import Value
from google.api_core import datetime_helpers
from google.cloud.spanner_v1.proto.type_pb2 import TIMESTAMP
from google.cloud.spanner_v1.proto.type_pb2 import Type
value = datetime_helpers.DatetimeWithNanoseconds(
2016, 12, 20, 21, 13, 47, nanosecond=123456789, tzinfo=pytz.UTC
)
field_type = Type(code=TIMESTAMP)
value_pb = Value(string_value=datetime_helpers.to_rfc3339(value))
parsed = self._callFUT(value_pb, field_type)
self.assertIsInstance(parsed, datetime_helpers.DatetimeWithNanoseconds)
self.assertEqual(parsed, value)
def _make_scalar_field(name, type_):
from google.cloud.spanner_v1.proto.type_pb2 import StructType
from google.cloud.spanner_v1.proto.type_pb2 import Type
return StructType.Field(name=name, type=Type(code=type_))
def _execute_sql_helper(self,
multi_use,
first=True,
count=0,
partition=None):
from google.protobuf.struct_pb2 import Struct
from google.cloud.spanner_v1.proto.result_set_pb2 import (
PartialResultSet, ResultSetMetadata, ResultSetStats)
from google.cloud.spanner_v1.proto.transaction_pb2 import (
TransactionSelector)
from google.cloud.spanner_v1.proto.type_pb2 import Type, StructType
from google.cloud.spanner_v1.proto.type_pb2 import STRING, INT64
from google.cloud.spanner_v1._helpers import _make_value_pb
txn_id = b'DEADBEEF'
VALUES = [
[u'bharney', u'rhubbyl', 31],
[u'phred', u'phlyntstone', 32],
]
VALUE_PBS = [[_make_value_pb(item) for item in row] for row in VALUES]
MODE = 2 # PROFILE
struct_type_pb = StructType(fields=[
StructType.Field(name='first_name', type=Type(code=STRING)),
StructType.Field(name='last_name', type=Type(code=STRING)),
StructType.Field(name='age', type=Type(code=INT64)),
])
metadata_pb = ResultSetMetadata(row_type=struct_type_pb)
stats_pb = ResultSetStats(query_stats=Struct(
fields={
'rows_returned': _make_value_pb(2),
}))
result_sets = [
PartialResultSet(values=VALUE_PBS[0], metadata=metadata_pb),
PartialResultSet(values=VALUE_PBS[1], stats=stats_pb),
]
iterator = _MockIterator(*result_sets)
database = _Database()
api = database.spanner_api = _FauxSpannerAPI(
_execute_streaming_sql_response=iterator)
session = _Session(database)
derived = self._makeDerived(session)
derived._multi_use = multi_use
derived._read_request_count = count
if not first:
derived._transaction_id = txn_id
result_set = derived.execute_sql(SQL_QUERY_WITH_PARAM,
PARAMS,
PARAM_TYPES,
query_mode=MODE,
partition=partition)
self.assertEqual(derived._read_request_count, count + 1)
if multi_use:
self.assertIs(result_set._source, derived)
else:
self.assertIsNone(result_set._source)
self.assertEqual(list(result_set), VALUES)
self.assertEqual(result_set.metadata, metadata_pb)
self.assertEqual(result_set.stats, stats_pb)
(r_session, sql, transaction, params, param_types, resume_token,
query_mode, partition_token,
metadata) = api._executed_streaming_sql_with
self.assertEqual(r_session, self.SESSION_NAME)
self.assertEqual(sql, SQL_QUERY_WITH_PARAM)
self.assertIsInstance(transaction, TransactionSelector)
if multi_use:
if first:
self.assertTrue(transaction.begin.read_only.strong)
else:
self.assertEqual(transaction.id, txn_id)
else:
self.assertTrue(transaction.single_use.read_only.strong)
expected_params = Struct(fields={
key: _make_value_pb(value)
for (key, value) in PARAMS.items()
})
self.assertEqual(params, expected_params)
self.assertEqual(param_types, PARAM_TYPES)
self.assertEqual(query_mode, MODE)
self.assertEqual(resume_token, b'')
self.assertEqual(partition_token, partition)
self.assertEqual(metadata,
[('google-cloud-resource-prefix', database.name)])
def _read_helper(self, multi_use, first=True, count=0, partition=None):
from google.protobuf.struct_pb2 import Struct
from google.cloud.spanner_v1.proto.result_set_pb2 import (
PartialResultSet, ResultSetMetadata, ResultSetStats)
from google.cloud.spanner_v1.proto.transaction_pb2 import (
TransactionSelector)
from google.cloud.spanner_v1.proto.type_pb2 import Type, StructType
from google.cloud.spanner_v1.proto.type_pb2 import STRING, INT64
from google.cloud.spanner_v1.keyset import KeySet
from google.cloud.spanner_v1._helpers import _make_value_pb
txn_id = b'DEADBEEF'
VALUES = [
[u'bharney', 31],
[u'phred', 32],
]
VALUE_PBS = [[_make_value_pb(item) for item in row] for row in VALUES]
struct_type_pb = StructType(fields=[
StructType.Field(name='name', type=Type(code=STRING)),
StructType.Field(name='age', type=Type(code=INT64)),
])
metadata_pb = ResultSetMetadata(row_type=struct_type_pb)
stats_pb = ResultSetStats(query_stats=Struct(
fields={
'rows_returned': _make_value_pb(2),
}))
result_sets = [
PartialResultSet(values=VALUE_PBS[0], metadata=metadata_pb),
PartialResultSet(values=VALUE_PBS[1], stats=stats_pb),
]
KEYS = ['*****@*****.**', '*****@*****.**']
keyset = KeySet(keys=KEYS)
INDEX = 'email-address-index'
LIMIT = 20
database = _Database()
api = database.spanner_api = _FauxSpannerAPI(
_streaming_read_response=_MockIterator(*result_sets))
session = _Session(database)
derived = self._makeDerived(session)
derived._multi_use = multi_use
derived._read_request_count = count
if not first:
derived._transaction_id = txn_id
if partition is not None: # 'limit' and 'partition' incompatible
result_set = derived.read(TABLE_NAME,
COLUMNS,
keyset,
index=INDEX,
partition=partition)
else:
result_set = derived.read(TABLE_NAME,
COLUMNS,
keyset,
index=INDEX,
limit=LIMIT)
self.assertEqual(derived._read_request_count, count + 1)
if multi_use:
self.assertIs(result_set._source, derived)
else:
self.assertIsNone(result_set._source)
self.assertEqual(list(result_set), VALUES)
self.assertEqual(result_set.metadata, metadata_pb)
self.assertEqual(result_set.stats, stats_pb)
(r_session, table, columns, key_set, transaction, index, limit,
resume_token, r_partition, metadata) = api._streaming_read_with
self.assertEqual(r_session, self.SESSION_NAME)
self.assertEqual(table, TABLE_NAME)
self.assertEqual(columns, COLUMNS)
self.assertEqual(key_set, keyset._to_pb())
self.assertIsInstance(transaction, TransactionSelector)
if multi_use:
if first:
self.assertTrue(transaction.begin.read_only.strong)
else:
self.assertEqual(transaction.id, txn_id)
else:
self.assertTrue(transaction.single_use.read_only.strong)
self.assertEqual(index, INDEX)
if partition is not None:
self.assertEqual(limit, 0)
self.assertEqual(r_partition, partition)
else:
self.assertEqual(limit, LIMIT)
self.assertIsNone(r_partition)
self.assertEqual(resume_token, b'')
self.assertEqual(metadata,
[('google-cloud-resource-prefix', database.name)])
