def _merge_array(lhs, rhs, type_):
"""Helper for '_merge_by_type'."""
element_type = type_.array_element_type
if element_type.code in _UNMERGEABLE_TYPES:
# Individual values cannot be merged, just concatenate
lhs.list_value.values.extend(rhs.list_value.values)
return lhs
lhs, rhs = list(lhs.list_value.values), list(rhs.list_value.values)
# Sanity check: If either list is empty, short-circuit.
# This is effectively a no-op.
if not len(lhs) or not len(rhs):
return Value(list_value=ListValue(values=(lhs + rhs)))
first = rhs.pop(0)
if first.HasField('null_value'): # can't merge
lhs.append(first)
else:
last = lhs.pop()
try:
merged = _merge_by_type(last, first, element_type)
except Unmergeable:
lhs.append(last)
lhs.append(first)
else:
lhs.append(merged)
return Value(list_value=ListValue(values=(lhs + rhs)))
def _merge_struct(lhs, rhs, type_):
"""Helper for '_merge_by_type'."""
fields = type_.struct_type.fields
lhs, rhs = list(lhs.list_value.values), list(rhs.list_value.values)
# Sanity check: If either list is empty, short-circuit.
# This is effectively a no-op.
if not len(lhs) or not len(rhs):
return Value(list_value=ListValue(values=(lhs + rhs)))
candidate_type = fields[len(lhs) - 1].type_
first = rhs.pop(0)
if first.HasField(
"null_value") or candidate_type.code in _UNMERGEABLE_TYPES:
lhs.append(first)
else:
last = lhs.pop()
try:
merged = _merge_by_type(last, first, candidate_type)
except Unmergeable:
lhs.append(last)
lhs.append(first)
else:
lhs.append(merged)
return Value(list_value=ListValue(values=lhs + rhs))
def test_list_view_not_equals(list_proto):
view_1 = ListView(list_proto)
lv = ListValue()
l = [0, 1, 'heya', {'key': 'value'}, [0, 1, 2]]
lv.extend(l)
view_2 = ListView(lv)
assert view_2 != view_1
def test_list_reverse():
list = ListValue()
l = [0, 2, 1]
list.extend(l)
view = ListView(list)
view.reverse()
assert view[0] == 1
assert view[1] == 2
assert view[2] == 0
def array_to_list_value(array, lv=None):
if lv is None:
lv = ListValue()
if len(array.shape) == 1:
lv.extend(array)
else:
for sub_array in array:
sub_lv = lv.add_list()
array_to_list_value(sub_array, sub_lv)
return lv
def python_to_val_proto(raw_val, allow_collection=False):
"""
Converts a Python variable into a `protobuf` `Value` `Message` object.
Parameters
----------
raw_val
Python variable.
allow_collection : bool, default False
Whether to allow ``list``s and ``dict``s as `val`. This flag exists because some callers
ought to not support logging collections, so this function will perform the typecheck on `val`.
Returns
-------
google.protobuf.struct_pb2.Value
`protobuf` `Value` `Message` representing `val`.
"""
# TODO: check `allow_collection` before `to_builtin()` to avoid unnecessary processing
val = to_builtin(raw_val)
if val is None:
return Value(null_value=NULL_VALUE)
elif isinstance(val,
bool): # did you know that `bool` is a subclass of `int`?
return Value(bool_value=val)
elif isinstance(val, numbers.Real):
return Value(number_value=val)
elif isinstance(val, six.string_types):
return Value(string_value=val)
elif isinstance(val, (list, dict)):
if allow_collection:
if isinstance(val, list):
list_value = ListValue()
list_value.extend(val) # pylint: disable=no-member
return Value(list_value=list_value)
else: # isinstance(val, dict)
if all(
[isinstance(key, six.string_types) for key in val.keys()]):
struct_value = Struct()
struct_value.update(val) # pylint: disable=no-member
return Value(struct_value=struct_value)
else: # protobuf's fault
raise TypeError(
"struct keys must be strings; consider using log_artifact() instead"
)
else:
raise TypeError(
"unsupported type {}; consider using log_attribute() instead".
format(type(raw_val)))
else:
raise TypeError(
"unsupported type {}; consider using log_artifact() instead".
format(type(raw_val)))
def test_to_pb_w_empty_list(self):
from google.protobuf.struct_pb2 import ListValue
from google.protobuf.struct_pb2 import Value
from google.cloud.spanner_v1.proto.keys_pb2 import KeyRange
key = u'key'
key_range = self._make_one(start_closed=[], end_closed=[key])
key_range_pb = key_range._to_pb()
expected = KeyRange(
start_closed=ListValue(values=[]),
end_closed=ListValue(values=[Value(string_value=key)]),
)
self.assertEqual(key_range_pb, expected)
def test_to_pb_w_start_closed_and_end_open(self):
from google.protobuf.struct_pb2 import ListValue
from google.protobuf.struct_pb2 import Value
from google.cloud.spanner_v1.proto.keys_pb2 import KeyRange
key1 = u'key_1'
key2 = u'key_2'
key_range = self._make_one(start_closed=[key1], end_open=[key2])
key_range_pb = key_range._to_pb()
expected = KeyRange(
start_closed=ListValue(values=[Value(string_value=key1)]),
end_open=ListValue(values=[Value(string_value=key2)]),
)
self.assertEqual(key_range_pb, expected)
def _makeListValue(values=(), value_pbs=None):
from google.protobuf.struct_pb2 import ListValue
from google.protobuf.struct_pb2 import Value
from google.cloud.spanner._helpers import _make_list_value_pb
if value_pbs is not None:
return Value(list_value=ListValue(values=value_pbs))
return Value(list_value=_make_list_value_pb(values))
def test_list_view_append_empty():
lv = ListValue()
view = ListView(lv)
assert len(view) == 0
view.append(10)
assert len(view) == 1
assert view[0] == 10
def array_to_list_value(array: np.ndarray,
lv: Optional[ListValue] = None) -> ListValue:
"""
Construct a proto ListValue from numpy array
Parameters
----------
array
Numpy array
lv
Proto buffer ListValue to extend
Returns
-------
"""
if lv is None:
lv = ListValue()
if len(array.shape) == 1:
lv.extend(array.tolist())
else:
for sub_array in array:
sub_lv = lv.add_list()
array_to_list_value(sub_array, sub_lv)
return lv
def test_w_array_non_empty(self):
from google.protobuf.struct_pb2 import Value, ListValue
from google.cloud.proto.spanner.v1.type_pb2 import Type, ARRAY, INT64
field_type = Type(code=ARRAY, array_element_type=Type(code=INT64))
VALUES = [32, 19, 5]
values_pb = ListValue(
values=[Value(string_value=str(value)) for value in VALUES])
value_pb = Value(list_value=values_pb)
self.assertEqual(self._callFUT(value_pb, field_type), VALUES)
def _make_list_value_pb(values):
"""Construct of ListValue protobufs.
:type values: list of scalar
:param values: Row data
:rtype: :class:`~google.protobuf.struct_pb2.ListValue`
:returns: protobuf
"""
return ListValue(values=[_make_value_pb(value) for value in values])
def test_w_array_empty(self):
from google.protobuf.struct_pb2 import Value, ListValue
from google.cloud.spanner_v1 import Type
from google.cloud.spanner_v1 import TypeCode
field_type = Type(code=TypeCode.ARRAY,
array_element_type=Type(code=TypeCode.INT64))
value_pb = Value(list_value=ListValue(values=[]))
self.assertEqual(self._callFUT(value_pb, field_type), [])
def _merge_struct(lhs, rhs, type_):
"""Helper for '_merge_by_type'."""
fields = type_.struct_type.fields
lhs, rhs = list(lhs.list_value.values), list(rhs.list_value.values)
candidate_type = fields[len(lhs) - 1].type
first = rhs.pop(0)
if (first.HasField('null_value') or
candidate_type.code in _UNMERGEABLE_TYPES):
lhs.append(first)
else:
last = lhs.pop()
lhs.append(_merge_by_type(last, first, candidate_type))
return Value(list_value=ListValue(values=lhs + rhs))
def test_list_view_insert():
lv = ListValue()
view = ListView(lv)
assert len(view) == 0
view.insert(0, 10)
assert len(view) == 1
assert view[0] == 10
view.insert(20, 20)
assert len(view) == 2
assert view[1] == 20
view.insert(0, 30)
assert len(view) == 2
assert view[0] == 30
def _to_value(data):
if data is None:
return Value(null_value=NullValue.NULL_VALUE)
elif isinstance(data, numbers.Number):
return Value(number_value=data)
elif isinstance(data, str):
return Value(string_value=data)
elif isinstance(data, bool):
return Value(bool_value=data)
elif isinstance(data, (list, tuple, set)):
return Value(list_value=ListValue(values=(TestLogging._to_value(e)
for e in data)))
elif isinstance(data, dict):
return Value(struct_value=TestLogging._dict_to_struct(data))
else:
raise TypeError("Unknown data type: %r" % type(data))
def to_value(value):
if value is None:
return Value(null_value=NULL_VALUE)
elif isinstance(value, bool):
# This check needs to happen before isinstance(value, int),
# isinstance(value, int) returns True when value is bool.
return Value(bool_value=value)
elif isinstance(value, six.integer_types) or isinstance(value, float):
return Value(number_value=value)
elif isinstance(value, six.string_types) or isinstance(
value, six.text_type):
return Value(string_value=value)
elif isinstance(value, dict):
return Value(struct_value=Struct(
fields={k: to_value(v)
for k, v in value.items()}))
elif isinstance(value, list):
return Value(list_value=ListValue(
values=[to_value(item) for item in value]))
else:
raise ValueError('Unsupported data type: {}'.format(type(value)))
def _merge_array(lhs, rhs, type_):
"""Helper for '_merge_by_type'."""
element_type = type_.array_element_type
if element_type.code in _UNMERGEABLE_TYPES:
# Individual values cannot be merged, just concatenate
lhs.list_value.values.extend(rhs.list_value.values)
return lhs
lhs, rhs = list(lhs.list_value.values), list(rhs.list_value.values)
first = rhs.pop(0)
if first.HasField('null_value'): # can't merge
lhs.append(first)
else:
last = lhs.pop()
try:
merged = _merge_by_type(last, first, element_type)
except Unmergeable:
lhs.append(last)
lhs.append(first)
else:
lhs.append(merged)
return Value(list_value=ListValue(values=(lhs + rhs)))
def test_list_view_dict():
list = ListValue()
l = [0, 1, 'hey', {'key': 'value'}, [0, 1, 2]]
list.extend(l)
view = ListView(list)
assert view.dict() == l
def list_proto():
lv = ListValue()
l = [0, 1, 'hey', {'key': 'value'}, [0, 1, 2]]
lv.extend(l)
return lv
def test_empty_struct_view():
l = ListValue()
view = ListView(l)
assert len(view) == 0
