def decode_value(blob, pos, reverse=False):
prop_value = entity_pb.PropertyValue()
marker = blob[pos]
pos += 1
code = ord(marker) ^ 0xFF if reverse else ord(marker)
if code == NULL_CODE:
pass
elif INT64_ZERO_CODE - 8 <= code <= INT64_ZERO_CODE + 8:
int_val, pos = Int64.decode(marker, blob, pos, reverse)
prop_value.set_int64value(int_val)
elif code in (TRUE_CODE, FALSE_CODE):
prop_value.set_booleanvalue(code == TRUE_CODE)
elif code == BYTES_CODE:
bytes_val, pos = Bytes.decode(blob, pos, reverse)
prop_value.set_stringvalue(bytes_val)
elif code == DOUBLE_CODE:
double_val, pos = Double.decode(blob, pos, reverse)
prop_value.set_doublevalue(double_val)
elif code == POINT_CODE:
point_val, pos = Point.decode(blob, pos, reverse)
prop_value.mutable_pointvalue().MergeFrom(point_val)
elif code == USER_CODE:
user_val, pos = User.decode(blob, pos, reverse)
prop_value.mutable_uservalue().MergeFrom(user_val)
elif code == REFERENCE_CODE:
ref_val, pos = Reference.decode(blob, pos, reverse)
prop_value.mutable_referencevalue().MergeFrom(ref_val)
return prop_value, pos
def __EncodeIndexPB(pb):
if isinstance(pb, entity_pb.PropertyValue) and pb.has_uservalue():
userval = entity_pb.PropertyValue()
userval.mutable_uservalue().set_email(pb.uservalue().email())
userval.mutable_uservalue().set_auth_domain(pb.uservalue().auth_domain())
userval.mutable_uservalue().set_gaiaid(0)
pb = userval
encoder = sortable_pb_encoder.Encoder()
pb.Output(encoder)
return buffer(encoder.buffer().tostring())
def test_extract_entities_from_composite_indexes(self):
project_id = 'guestbook'
props = ['prop1', 'prop2']
db_batch = flexmock()
db_batch.should_receive('valid_data_version_sync').and_return(True)
transaction_manager = flexmock()
dd = DatastoreDistributed(db_batch, transaction_manager,
self.get_zookeeper())
query = datastore_pb.Query()
for prop_name in props:
query.add_property_name(prop_name)
index = query.add_composite_index()
definition = index.mutable_definition()
for prop_name in props:
prop = definition.add_property()
prop.set_name(prop_name)
entity_id = 1524699263329044
val1 = entity_pb.PropertyValue()
val1.set_int64value(5)
val2 = entity_pb.PropertyValue()
val2.set_stringvalue('test')
index_key = '\x00'.join(
[project_id, 'namespace', 'index1', str(encode_index_pb(val1)),
str(encode_index_pb(val2)), 'Greeting:{}\x01'.format(entity_id)])
index_results = [{index_key: {'reference': 'ignored-ref'}}]
entities = dd._extract_entities_from_composite_indexes(
query, index_results, index)
self.assertEqual(len(entities), 1)
returned_entity = entity_pb.EntityProto(entities[0])
self.assertEqual(returned_entity.property_size(), 2)
self.assertEqual(returned_entity.key().path().element(0).type(), 'Greeting')
self.assertEqual(returned_entity.key().path().element(0).id(), entity_id)
self.assertEqual(returned_entity.property(0).name(), 'prop1')
self.assertEqual(returned_entity.property(0).value().int64value(), 5)
self.assertEqual(returned_entity.property(1).name(), 'prop2')
self.assertEqual(returned_entity.property(1).value().stringvalue(), 'test')
def _Dynamic_GetSchema(self, app_str, schema):
minint = -sys.maxint - 1
app_str = self.ResolveAppId(app_str.value())
kinds = []
for app, kind in self.__entities:
if app == app_str:
kind_pb = entity_pb.EntityProto()
kind_pb.mutable_key().set_app('')
kind_pb.mutable_key().mutable_path().add_element().set_type(
kind)
kind_pb.mutable_entity_group()
kinds.append(kind_pb)
props = {}
for entity in self.__entities[(app, kind)].values():
for prop in entity.property_list():
if prop.name() not in props:
props[prop.name()] = entity_pb.PropertyValue()
props[prop.name()].MergeFrom(prop.value())
for value_pb in props.values():
if value_pb.has_int64value():
value_pb.set_int64value(minint)
if value_pb.has_booleanvalue():
value_pb.set_booleanvalue(False)
if value_pb.has_stringvalue():
value_pb.set_stringvalue('')
if value_pb.has_doublevalue():
value_pb.set_doublevalue(float('-inf'))
if value_pb.has_pointvalue():
value_pb.mutable_pointvalue().set_x(float('-inf'))
value_pb.mutable_pointvalue().set_y(float('-inf'))
if value_pb.has_uservalue():
value_pb.mutable_uservalue().set_gaiaid(minint)
value_pb.mutable_uservalue().set_email('')
value_pb.mutable_uservalue().set_auth_domain('')
value_pb.mutable_uservalue().set_nickname('')
elif value_pb.has_referencevalue():
value_pb.clear_referencevalue()
value_pb.mutable_referencevalue().set_app('')
for name, value_pb in props.items():
prop_pb = kind_pb.add_property()
prop_pb.set_name(name)
prop_pb.mutable_value().CopyFrom(value_pb)
schema.kind_list().extend(kinds)
def encode_index_pb(pb):
""" Returns an encoded protocol buffer.
Args:
pb: The protocol buffer to encode.
Returns:
An encoded protocol buffer.
"""
def _encode_path(pb):
""" Takes a protocol buffer and returns the encoded path. """
path = []
for e in pb.element_list():
if e.has_name():
key_id = e.name()
elif e.has_id():
key_id = str(e.id()).zfill(ID_KEY_LENGTH)
else:
raise BadRequest('Entity path must contain name or ID')
if ID_SEPARATOR in e.type():
raise BadRequest('Kind names must not include ":"')
path.append(ID_SEPARATOR.join([e.type(), key_id]))
val = dbconstants.KIND_SEPARATOR.join(path)
val += dbconstants.KIND_SEPARATOR
return val
if isinstance(pb, entity_pb.PropertyValue) and pb.has_uservalue():
userval = entity_pb.PropertyValue()
userval.mutable_uservalue().set_email(pb.uservalue().email())
userval.mutable_uservalue().set_auth_domain("")
userval.mutable_uservalue().set_gaiaid(0)
pb = userval
def remove_nulls(value):
""" Remove null values from a given string and byte stuff encode. """
return buffer(
str(value).replace('\x01', '\x01\x02').replace('\x00', '\x01\x01'))
encoder = sortable_pb_encoder.Encoder()
pb.Output(encoder)
if isinstance(pb, entity_pb.PropertyValue):
value = encoder.buffer().tostring()
# We strip off null strings because it is our delimiter.
value = remove_nulls(value)
return buffer(value)
elif isinstance(pb, entity_pb.Path):
return buffer(_encode_path(pb))
def next_page(self):
if self._done:
raise gen.Return(([], False))
# TODO: This can be made async.
ns_dir = self._project_dir.open(
self._tr, (SinglePropIndex.DIR_NAME, self._namespace))
kinds = ns_dir.list(self._tr)
# TODO: Check if stat entities belong in kinds.
kind_dirs = [ns_dir.open(self._tr, (kind,)) for kind in kinds]
results = []
for kind, kind_dir in zip(kinds, kind_dirs):
# TODO: This can be made async.
prop_names = kind_dir.list(self._tr)
for prop_name in prop_names:
prop_dir = kind_dir.open(self._tr, (prop_name,))
index = SinglePropIndex(prop_dir)
populated_map = yield [self._populated(index, type_name)
for type_name in self.PROPERTY_TYPES]
populated_types = tuple(
type_ for type_, populated in zip(self.PROPERTY_TYPES, populated_map)
if populated)
if not populated_types:
continue
project_id = self._project_dir.get_path()[-1]
path = (u'__kind__', kind, u'__property__', prop_name)
prop_values = []
for prop_type in populated_types:
prop_value = entity_pb.PropertyValue()
prop_value.set_stringvalue(prop_type)
prop_values.append(prop_value)
# TODO: Consider giving metadata results their own entry class.
entry = CompositeEntry(
project_id, self._namespace, path,
[(u'property_representation', prop_values)], None, None)
results.append(entry)
self._done = True
raise gen.Return((results, False))
def __EncodeIndexPB(pb):
"""Encodes a protobuf using sortable_pb_encoder to preserve entity order.
Using sortable_pb_encoder, encodes the protobuf, while using
the ordering semantics for the datastore, and validating for the special
case of uservalues ordering.
Args:
pb: An Entity protobuf.
Returns:
A buffer holding the encoded protobuf.
"""
if isinstance(pb, entity_pb.PropertyValue) and pb.has_uservalue():
userval = entity_pb.PropertyValue()
userval.mutable_uservalue().set_email(pb.uservalue().email())
userval.mutable_uservalue().set_auth_domain(pb.uservalue().auth_domain())
userval.mutable_uservalue().set_gaiaid(0)
pb = userval
encoder = sortable_pb_encoder.Encoder()
pb.Output(encoder)
return buffer(encoder.buffer().tostring())
def Query(self, query, filters, orders):
"""Perform a query on this pseudo-kind.
Args:
query: the original datastore_pb.Query
filters: the filters from query
orders: the orders from query
Returns:
A query cursor to iterate over the query results, or None if the query
is invalid.
"""
property_range = datastore_stub_util.ParsePropertyQuery(
query, filters, orders)
keys_only = query.keys_only()
conn = self._stub._GetConnection()
cursor = None
try:
prefix = self._stub._GetTablePrefix(query)
filters = []
def AddExtremeFilter(extreme, inclusive, is_end):
"""Add filter for kind start/end."""
if not is_end:
op = datastore_pb.Query_Filter.GREATER_THAN_OR_EQUAL
else:
op = datastore_pb.Query_Filter.LESS_THAN_OR_EQUAL
filters.append(('kind', op, extreme[0]))
property_range.MapExtremes(AddExtremeFilter)
for name in datastore_stub_util.GetInvisibleSpecialPropertyNames():
filters.append(('name', '!=', name))
params = []
sql_filters = self._stub._CreateFilterString(filters, params)
if not keys_only:
sql_stmt = (
'SELECT kind, name, value FROM "%s!EntitiesByProperty" %s '
'GROUP BY kind, name, substr(value, 1, 1) '
'ORDER BY kind, name' % (prefix, sql_filters))
else:
sql_stmt = (
'SELECT kind, name FROM "%s!EntitiesByProperty" %s '
'GROUP BY kind, name ORDER BY kind, name' %
(prefix, sql_filters))
c = conn.execute(sql_stmt, params)
properties = []
kind = None
name = None
property_pb = None
for row in c.fetchall():
if not (row[0] == kind and row[1] == name):
if not property_range.Contains((row[0], row[1])):
continue
kind, name = row[:2]
if property_pb:
properties.append(property_pb)
property_pb = MakeEntityForQuery(query,
KindPseudoKind.name,
ToUtf8(kind), self.name,
ToUtf8(name))
if not keys_only:
value_data = row[2]
value_decoder = sortable_pb_encoder.Decoder(
array.array('B', str(value_data)))
raw_value_pb = entity_pb.PropertyValue()
raw_value_pb.Merge(value_decoder)
tag = datastore_types.GetPropertyValueTag(raw_value_pb)
tag_name = datastore_stub_util._PROPERTY_TYPE_NAMES[tag]
representation_pb = property_pb.add_property()
representation_pb.set_name('property_representation')
representation_pb.set_multiple(True)
representation_pb.mutable_value().set_stringvalue(tag_name)
if property_pb:
properties.append(property_pb)
cursor = datastore_stub_util._ExecuteQuery(properties, query, [],
[], [])
finally:
self._stub._ReleaseConnection(conn)
return cursor
def _Dynamic_GetSchema(self, app_str, schema):
minint = -sys.maxint - 1
try:
minfloat = float('-inf')
except ValueError:
minfloat = -1e300000
app_str = app_str.value()
kinds = []
for app, kind in self.__entities:
if app == app_str:
app_kind = (app, kind)
if app_kind in self.__schema_cache:
kinds.append(self.__schema_cache[app_kind])
continue
kind_pb = entity_pb.EntityProto()
kind_pb.mutable_key().set_app('')
kind_pb.mutable_key().mutable_path().add_element().set_type(kind)
kind_pb.mutable_entity_group()
props = {}
for entity in self.__entities[app_kind].values():
for prop in entity.protobuf.property_list():
if prop.name() not in props:
props[prop.name()] = entity_pb.PropertyValue()
props[prop.name()].MergeFrom(prop.value())
for value_pb in props.values():
if value_pb.has_int64value():
value_pb.set_int64value(minint)
if value_pb.has_booleanvalue():
value_pb.set_booleanvalue(False)
if value_pb.has_stringvalue():
value_pb.set_stringvalue('')
if value_pb.has_doublevalue():
value_pb.set_doublevalue(minfloat)
if value_pb.has_pointvalue():
value_pb.mutable_pointvalue().set_x(minfloat)
value_pb.mutable_pointvalue().set_y(minfloat)
if value_pb.has_uservalue():
value_pb.mutable_uservalue().set_gaiaid(minint)
value_pb.mutable_uservalue().set_email('')
value_pb.mutable_uservalue().set_auth_domain('')
value_pb.mutable_uservalue().clear_nickname()
elif value_pb.has_referencevalue():
value_pb.clear_referencevalue()
value_pb.mutable_referencevalue().set_app('')
for name, value_pb in props.items():
prop_pb = kind_pb.add_property()
prop_pb.set_name(name)
prop_pb.set_multiple(False)
prop_pb.mutable_value().CopyFrom(value_pb)
kinds.append(kind_pb)
self.__schema_cache[app_kind] = kind_pb
for kind_pb in kinds:
schema.add_kind().CopyFrom(kind_pb)
def _Dynamic_GetSchema(self, req, schema):
""" Get the schema of a particular kind of entity. """
app_str = req.app()
self.__ValidateAppId(app_str)
namespace_str = req.name_space()
app_namespace_str = datastore_types.EncodeAppIdNamespace(
app_str, namespace_str)
kinds = []
for app_namespace, kind in self.__entities:
if (app_namespace != app_namespace_str
or (req.has_start_kind() and kind < req.start_kind())
or (req.has_end_kind() and kind > req.end_kind())):
continue
app_kind = (app_namespace_str, kind)
if app_kind in self.__schema_cache:
kinds.append(self.__schema_cache[app_kind])
continue
kind_pb = entity_pb.EntityProto()
kind_pb.mutable_key().set_app('')
kind_pb.mutable_key().mutable_path().add_element().set_type(kind)
kind_pb.mutable_entity_group()
props = {}
for entity in self.__entities[app_kind].values():
for prop in entity.protobuf.property_list():
if prop.name() not in props:
props[prop.name()] = entity_pb.PropertyValue()
props[prop.name()].MergeFrom(prop.value())
for value_pb in props.values():
if value_pb.has_int64value():
value_pb.set_int64value(0)
if value_pb.has_booleanvalue():
value_pb.set_booleanvalue(False)
if value_pb.has_stringvalue():
value_pb.set_stringvalue('none')
if value_pb.has_doublevalue():
value_pb.set_doublevalue(0.0)
if value_pb.has_pointvalue():
value_pb.mutable_pointvalue().set_x(0.0)
value_pb.mutable_pointvalue().set_y(0.0)
if value_pb.has_uservalue():
value_pb.mutable_uservalue().set_gaiaid(0)
value_pb.mutable_uservalue().set_email('none')
value_pb.mutable_uservalue().set_auth_domain('none')
value_pb.mutable_uservalue().clear_nickname()
value_pb.mutable_uservalue().clear_obfuscated_gaiaid()
if value_pb.has_referencevalue():
value_pb.clear_referencevalue()
value_pb.mutable_referencevalue().set_app('none')
pathelem = value_pb.mutable_referencevalue(
).add_pathelement()
pathelem.set_type('none')
pathelem.set_name('none')
for name, value_pb in props.items():
prop_pb = kind_pb.add_property()
prop_pb.set_name(name)
prop_pb.set_multiple(False)
prop_pb.mutable_value().CopyFrom(value_pb)
kinds.append(kind_pb)
self.__schema_cache[app_kind] = kind_pb
for kind_pb in kinds:
kind = schema.add_kind()
kind.CopyFrom(kind_pb)
if not req.properties():
kind.clear_property()
schema.set_more_results(False)