def Query(self, query, filters, orders):
"""Perform a query on this pseudo-kind.
Args:
query: the original datastore_pb.Query.
filters: the filters from query.
orders: the orders from query.
Returns:
(results, remaining_filters, remaining_orders)
results is a list of entity_pb.EntityProto
remaining_filters and remaining_orders are the filters and orders that
should be applied in memory
"""
property_range = datastore_stub_util.ParsePropertyQuery(
query, filters, orders)
keys_only = query.keys_only()
app_namespace_str = datastore_types.EncodeAppIdNamespace(
query.app(), query.name_space())
properties = []
if keys_only:
usekey = '__property__keys'
else:
usekey = '__property__'
entities = self._stub._GetAllEntities()
for app_namespace, kind in entities:
if app_namespace != app_namespace_str: continue
app_kind = (app_namespace_str, kind)
kind = kind.decode('utf-8')
(start_cmp, end_cmp) = property_range.MapExtremes(
lambda extreme, inclusive, is_end: cmp(kind, extreme[0]))
if not ((start_cmp is None or start_cmp >= 0) and
(end_cmp is None or end_cmp <= 0)):
continue
kind_properties = self._stub._GetSchemaCache(app_kind, usekey)
if not kind_properties:
kind_properties = []
kind_key = datastore_types.Key.from_path(
KindPseudoKind.name,
kind,
_app=query.app(),
namespace=query.name_space())
props = collections.defaultdict(set)
for entity in entities[app_kind].values():
for prop in entity.protobuf.property_list():
prop_name = prop.name()
if (prop_name in datastore_stub_util.
GetInvisibleSpecialPropertyNames()):
continue
value_pb = prop.value()
props[prop_name].add(
datastore_types.GetPropertyValueTag(value_pb))
for prop in sorted(props):
property_e = datastore.Entity(self.name,
name=prop,
parent=kind_key,
_app=query.app(),
namespace=query.name_space())
if not keys_only and props[prop]:
property_e['property_representation'] = [
datastore_stub_util._PROPERTY_TYPE_NAMES[tag]
for tag in sorted(props[prop])
]
kind_properties.append(property_e._ToPb())
self._stub._SetSchemaCache(app_kind, usekey, kind_properties)
def InQuery(property_e):
return property_range.Contains(
(kind, _FinalElement(property_e.key()).name()))
properties += filter(InQuery, kind_properties)
return (properties, [], [])
def Query(self, query, filters, orders):
"""Perform a query on this pseudo-kind.
Args:
query: the original datastore_pb.Query
filters: the filters from query
orders: the orders from query
Returns:
A query cursor to iterate over the query results, or None if the query
is invalid.
"""
property_range = datastore_stub_util.ParsePropertyQuery(
query, filters, orders)
keys_only = query.keys_only()
conn = self._stub._GetConnection()
cursor = None
try:
prefix = self._stub._GetTablePrefix(query)
filters = []
def AddExtremeFilter(extreme, inclusive, is_end):
"""Add filter for kind start/end."""
if not is_end:
op = datastore_pb.Query_Filter.GREATER_THAN_OR_EQUAL
else:
op = datastore_pb.Query_Filter.LESS_THAN_OR_EQUAL
filters.append(('kind', op, extreme[0]))
property_range.MapExtremes(AddExtremeFilter)
for name in datastore_stub_util.GetInvisibleSpecialPropertyNames():
filters.append(('name', '!=', name))
params = []
sql_filters = self._stub._CreateFilterString(filters, params)
if not keys_only:
sql_stmt = (
'SELECT kind, name, value FROM "%s!EntitiesByProperty" %s '
'GROUP BY kind, name, substr(value, 1, 1) '
'ORDER BY kind, name' % (prefix, sql_filters))
else:
sql_stmt = (
'SELECT kind, name FROM "%s!EntitiesByProperty" %s '
'GROUP BY kind, name ORDER BY kind, name' %
(prefix, sql_filters))
c = conn.execute(sql_stmt, params)
properties = []
kind = None
name = None
property_pb = None
for row in c.fetchall():
if not (row[0] == kind and row[1] == name):
if not property_range.Contains((row[0], row[1])):
continue
kind, name = row[:2]
if property_pb:
properties.append(property_pb)
property_pb = MakeEntityForQuery(query,
KindPseudoKind.name,
ToUtf8(kind), self.name,
ToUtf8(name))
if not keys_only:
value_data = row[2]
value_decoder = sortable_pb_encoder.Decoder(
array.array('B', str(value_data)))
raw_value_pb = entity_pb.PropertyValue()
raw_value_pb.Merge(value_decoder)
tag = datastore_types.GetPropertyValueTag(raw_value_pb)
tag_name = datastore_stub_util._PROPERTY_TYPE_NAMES[tag]
representation_pb = property_pb.add_property()
representation_pb.set_name('property_representation')
representation_pb.set_multiple(True)
representation_pb.mutable_value().set_stringvalue(tag_name)
if property_pb:
properties.append(property_pb)
cursor = datastore_stub_util._ExecuteQuery(properties, query, [],
[], [])
finally:
self._stub._ReleaseConnection(conn)
return cursor
def Query(self, entities, query, filters, orders):
"""Perform a query on this pseudo-kind.
Args:
entities: all the app's entities.
query: the original datastore_pb.Query.
filters: the filters from query.
orders: the orders from query.
Returns:
(results, remaining_filters, remaining_orders)
results is a list of datastore.Entity
remaining_filters and remaining_orders are the filters and orders that
should be applied in memory
"""
property_range = datastore_stub_util.ParsePropertyQuery(
query, filters, orders)
keys_only = query.keys_only()
app_namespace_str = datastore_types.EncodeAppIdNamespace(
query.app(), query.name_space())
properties = []
if keys_only:
usekey = '__property__keys'
else:
usekey = '__property__'
for app_namespace, kind in entities:
if app_namespace != app_namespace_str: continue
(start_cmp, end_cmp) = property_range.MapExtremes(
lambda extreme, inclusive, is_end: cmp(kind, extreme[0]))
if not ((start_cmp is None or start_cmp >= 0) and
(end_cmp is None or end_cmp <= 0)):
continue
app_kind = (app_namespace_str, kind)
kind_properties = self.filestub._GetSchemaCache(app_kind, usekey)
if not kind_properties:
kind_properties = []
kind_key = datastore_types.Key.from_path(
KindPseudoKind.name, kind)
props = {}
for entity in entities[app_kind].values():
for prop in entity.protobuf.property_list():
prop_name = prop.name()
if (prop_name in datastore_stub_util.
GetInvisibleSpecialPropertyNames()):
continue
if prop_name not in props:
props[prop_name] = set()
native_value = entity.native[prop_name]
if not isinstance(native_value, list):
native_value = [native_value]
for value in native_value:
tag = self.filestub._PROPERTY_TYPE_TAGS.get(
value.__class__)
if tag is not None:
props[prop_name].add(tag)
else:
logging.warning(
'Unexpected value of class %s in datastore',
value.__class__)
for prop in sorted(props):
property_e = datastore.Entity(self.name,
name=prop,
parent=kind_key)
kind_properties.append(property_e)
if not keys_only and props[prop]:
property_e['property_representation'] = [
datastore_stub_util._PROPERTY_TYPE_NAMES[tag]
for tag in sorted(props[prop])
]
self.filestub._SetSchemaCache(app_kind, usekey,
kind_properties)
def InQuery(property_e):
return property_range.Contains((kind, property_e.key().name()))
properties += filter(InQuery, kind_properties)
return (properties, [], [])
