def _merge_array(lhs, rhs, type_): """Helper for '_merge_by_type'.""" element_type = type_.array_element_type if element_type.code in _UNMERGEABLE_TYPES: # Individual values cannot be merged, just concatenate lhs.list_value.values.extend(rhs.list_value.values) return lhs lhs, rhs = list(lhs.list_value.values), list(rhs.list_value.values) # Sanity check: If either list is empty, short-circuit. # This is effectively a no-op. if not len(lhs) or not len(rhs): return Value(list_value=ListValue(values=(lhs + rhs))) first = rhs.pop(0) if first.HasField('null_value'): # can't merge lhs.append(first) else: last = lhs.pop() try: merged = _merge_by_type(last, first, element_type) except Unmergeable: lhs.append(last) lhs.append(first) else: lhs.append(merged) return Value(list_value=ListValue(values=(lhs + rhs)))
def _merge_struct(lhs, rhs, type_): """Helper for '_merge_by_type'.""" fields = type_.struct_type.fields lhs, rhs = list(lhs.list_value.values), list(rhs.list_value.values) # Sanity check: If either list is empty, short-circuit. # This is effectively a no-op. if not len(lhs) or not len(rhs): return Value(list_value=ListValue(values=(lhs + rhs))) candidate_type = fields[len(lhs) - 1].type_ first = rhs.pop(0) if first.HasField( "null_value") or candidate_type.code in _UNMERGEABLE_TYPES: lhs.append(first) else: last = lhs.pop() try: merged = _merge_by_type(last, first, candidate_type) except Unmergeable: lhs.append(last) lhs.append(first) else: lhs.append(merged) return Value(list_value=ListValue(values=lhs + rhs))
def test_list_view_not_equals(list_proto): view_1 = ListView(list_proto) lv = ListValue() l = [0, 1, 'heya', {'key': 'value'}, [0, 1, 2]] lv.extend(l) view_2 = ListView(lv) assert view_2 != view_1
def test_list_reverse(): list = ListValue() l = [0, 2, 1] list.extend(l) view = ListView(list) view.reverse() assert view[0] == 1 assert view[1] == 2 assert view[2] == 0
def array_to_list_value(array, lv=None): if lv is None: lv = ListValue() if len(array.shape) == 1: lv.extend(array) else: for sub_array in array: sub_lv = lv.add_list() array_to_list_value(sub_array, sub_lv) return lv
def python_to_val_proto(raw_val, allow_collection=False): """ Converts a Python variable into a `protobuf` `Value` `Message` object. Parameters ---------- raw_val Python variable. allow_collection : bool, default False Whether to allow ``list``s and ``dict``s as `val`. This flag exists because some callers ought to not support logging collections, so this function will perform the typecheck on `val`. Returns ------- google.protobuf.struct_pb2.Value `protobuf` `Value` `Message` representing `val`. """ # TODO: check `allow_collection` before `to_builtin()` to avoid unnecessary processing val = to_builtin(raw_val) if val is None: return Value(null_value=NULL_VALUE) elif isinstance(val, bool): # did you know that `bool` is a subclass of `int`? return Value(bool_value=val) elif isinstance(val, numbers.Real): return Value(number_value=val) elif isinstance(val, six.string_types): return Value(string_value=val) elif isinstance(val, (list, dict)): if allow_collection: if isinstance(val, list): list_value = ListValue() list_value.extend(val) # pylint: disable=no-member return Value(list_value=list_value) else: # isinstance(val, dict) if all( [isinstance(key, six.string_types) for key in val.keys()]): struct_value = Struct() struct_value.update(val) # pylint: disable=no-member return Value(struct_value=struct_value) else: # protobuf's fault raise TypeError( "struct keys must be strings; consider using log_artifact() instead" ) else: raise TypeError( "unsupported type {}; consider using log_attribute() instead". format(type(raw_val))) else: raise TypeError( "unsupported type {}; consider using log_artifact() instead". format(type(raw_val)))
def test_to_pb_w_empty_list(self): from google.protobuf.struct_pb2 import ListValue from google.protobuf.struct_pb2 import Value from google.cloud.spanner_v1.proto.keys_pb2 import KeyRange key = u'key' key_range = self._make_one(start_closed=[], end_closed=[key]) key_range_pb = key_range._to_pb() expected = KeyRange( start_closed=ListValue(values=[]), end_closed=ListValue(values=[Value(string_value=key)]), ) self.assertEqual(key_range_pb, expected)
def test_to_pb_w_start_closed_and_end_open(self): from google.protobuf.struct_pb2 import ListValue from google.protobuf.struct_pb2 import Value from google.cloud.spanner_v1.proto.keys_pb2 import KeyRange key1 = u'key_1' key2 = u'key_2' key_range = self._make_one(start_closed=[key1], end_open=[key2]) key_range_pb = key_range._to_pb() expected = KeyRange( start_closed=ListValue(values=[Value(string_value=key1)]), end_open=ListValue(values=[Value(string_value=key2)]), ) self.assertEqual(key_range_pb, expected)
def _makeListValue(values=(), value_pbs=None): from google.protobuf.struct_pb2 import ListValue from google.protobuf.struct_pb2 import Value from google.cloud.spanner._helpers import _make_list_value_pb if value_pbs is not None: return Value(list_value=ListValue(values=value_pbs)) return Value(list_value=_make_list_value_pb(values))
def test_list_view_append_empty(): lv = ListValue() view = ListView(lv) assert len(view) == 0 view.append(10) assert len(view) == 1 assert view[0] == 10
def array_to_list_value(array: np.ndarray, lv: Optional[ListValue] = None) -> ListValue: """ Construct a proto ListValue from numpy array Parameters ---------- array Numpy array lv Proto buffer ListValue to extend Returns ------- """ if lv is None: lv = ListValue() if len(array.shape) == 1: lv.extend(array.tolist()) else: for sub_array in array: sub_lv = lv.add_list() array_to_list_value(sub_array, sub_lv) return lv
def test_w_array_non_empty(self): from google.protobuf.struct_pb2 import Value, ListValue from google.cloud.proto.spanner.v1.type_pb2 import Type, ARRAY, INT64 field_type = Type(code=ARRAY, array_element_type=Type(code=INT64)) VALUES = [32, 19, 5] values_pb = ListValue( values=[Value(string_value=str(value)) for value in VALUES]) value_pb = Value(list_value=values_pb) self.assertEqual(self._callFUT(value_pb, field_type), VALUES)
def _make_list_value_pb(values): """Construct of ListValue protobufs. :type values: list of scalar :param values: Row data :rtype: :class:`~google.protobuf.struct_pb2.ListValue` :returns: protobuf """ return ListValue(values=[_make_value_pb(value) for value in values])
def test_w_array_empty(self): from google.protobuf.struct_pb2 import Value, ListValue from google.cloud.spanner_v1 import Type from google.cloud.spanner_v1 import TypeCode field_type = Type(code=TypeCode.ARRAY, array_element_type=Type(code=TypeCode.INT64)) value_pb = Value(list_value=ListValue(values=[])) self.assertEqual(self._callFUT(value_pb, field_type), [])
def _merge_struct(lhs, rhs, type_): """Helper for '_merge_by_type'.""" fields = type_.struct_type.fields lhs, rhs = list(lhs.list_value.values), list(rhs.list_value.values) candidate_type = fields[len(lhs) - 1].type first = rhs.pop(0) if (first.HasField('null_value') or candidate_type.code in _UNMERGEABLE_TYPES): lhs.append(first) else: last = lhs.pop() lhs.append(_merge_by_type(last, first, candidate_type)) return Value(list_value=ListValue(values=lhs + rhs))
def test_list_view_insert(): lv = ListValue() view = ListView(lv) assert len(view) == 0 view.insert(0, 10) assert len(view) == 1 assert view[0] == 10 view.insert(20, 20) assert len(view) == 2 assert view[1] == 20 view.insert(0, 30) assert len(view) == 2 assert view[0] == 30
def _to_value(data): if data is None: return Value(null_value=NullValue.NULL_VALUE) elif isinstance(data, numbers.Number): return Value(number_value=data) elif isinstance(data, str): return Value(string_value=data) elif isinstance(data, bool): return Value(bool_value=data) elif isinstance(data, (list, tuple, set)): return Value(list_value=ListValue(values=(TestLogging._to_value(e) for e in data))) elif isinstance(data, dict): return Value(struct_value=TestLogging._dict_to_struct(data)) else: raise TypeError("Unknown data type: %r" % type(data))
def to_value(value): if value is None: return Value(null_value=NULL_VALUE) elif isinstance(value, bool): # This check needs to happen before isinstance(value, int), # isinstance(value, int) returns True when value is bool. return Value(bool_value=value) elif isinstance(value, six.integer_types) or isinstance(value, float): return Value(number_value=value) elif isinstance(value, six.string_types) or isinstance( value, six.text_type): return Value(string_value=value) elif isinstance(value, dict): return Value(struct_value=Struct( fields={k: to_value(v) for k, v in value.items()})) elif isinstance(value, list): return Value(list_value=ListValue( values=[to_value(item) for item in value])) else: raise ValueError('Unsupported data type: {}'.format(type(value)))
def _merge_array(lhs, rhs, type_): """Helper for '_merge_by_type'.""" element_type = type_.array_element_type if element_type.code in _UNMERGEABLE_TYPES: # Individual values cannot be merged, just concatenate lhs.list_value.values.extend(rhs.list_value.values) return lhs lhs, rhs = list(lhs.list_value.values), list(rhs.list_value.values) first = rhs.pop(0) if first.HasField('null_value'): # can't merge lhs.append(first) else: last = lhs.pop() try: merged = _merge_by_type(last, first, element_type) except Unmergeable: lhs.append(last) lhs.append(first) else: lhs.append(merged) return Value(list_value=ListValue(values=(lhs + rhs)))
def test_list_view_dict(): list = ListValue() l = [0, 1, 'hey', {'key': 'value'}, [0, 1, 2]] list.extend(l) view = ListView(list) assert view.dict() == l
def list_proto(): lv = ListValue() l = [0, 1, 'hey', {'key': 'value'}, [0, 1, 2]] lv.extend(l) return lv
def test_empty_struct_view(): l = ListValue() view = ListView(l) assert len(view) == 0