def _GetQueryCursor(self, query, filters, orders, index_list):
app_id = query.app()
namespace = query.name_space()
pseudo_kind = None
if query.has_kind() and query.kind() in self._pseudo_kinds:
pseudo_kind = self._pseudo_kinds[query.kind()]
self.__entities_lock.acquire()
try:
app_ns = datastore_types.EncodeAppIdNamespace(app_id, namespace)
if pseudo_kind:
(results, filters,
orders) = pseudo_kind.Query(query, filters, orders)
elif query.has_kind():
results = [
entity.protobuf for entity in self.__entities_by_kind[
app_ns, query.kind()].values()
]
else:
results = []
for (cur_app_ns,
_), entities in self.__entities_by_kind.iteritems():
if cur_app_ns == app_ns:
results.extend(entity.protobuf
for entity in entities.itervalues())
except KeyError:
results = []
finally:
self.__entities_lock.release()
return datastore_stub_util._ExecuteQuery(results, query, filters,
orders, index_list)
def Query(self, query, filters, orders):
"""Perform a query on this pseudo-kind.
Args:
query: the original datastore_pb.Query.
filters: the filters from query.
orders: the orders from query.
Returns:
(results, remaining_filters, remaining_orders)
results is a list of entity_pb.EntityProto
remaining_filters and remaining_orders are the filters and orders that
should be applied in memory
"""
kind_range = datastore_stub_util.ParseKindQuery(query, filters, orders)
app_namespace_str = datastore_types.EncodeAppIdNamespace(
query.app(), query.name_space())
kinds = []
for app_namespace, kind in self._stub._GetAllEntities():
if app_namespace != app_namespace_str: continue
kind = kind.decode('utf-8')
if not kind_range.Contains(kind): continue
kinds.append(
datastore.Entity(self.name,
name=kind,
_app=query.app(),
namespace=query.name_space())._ToPb())
return (kinds, [], [])
def Query(self, entities, query, filters, orders):
"""Perform a query on this pseudo-kind.
Args:
entities: all the app's entities.
query: the original datastore_pb.Query.
filters: the filters from query.
orders: the orders from query.
Returns:
(results, remaining_filters, remaining_orders)
results is a list of datastore.Entity
remaining_filters and remaining_orders are the filters and orders that
should be applied in memory
"""
kind_range = datastore_stub_util.ParseKindQuery(query, filters, orders)
app_namespace_str = datastore_types.EncodeAppIdNamespace(
query.app(), query.name_space())
kinds = []
for app_namespace, kind in entities:
if app_namespace != app_namespace_str: continue
if not kind_range.Contains(kind): continue
kinds.append(datastore.Entity(self.name, name=kind))
return (kinds, [], [])
def _AppIdNamespaceKindForKey(self, key):
""" Get (app, kind) tuple from given key.
The (app, kind) tuple is used as an index into several internal
dictionaries, e.g. __entities.
Args:
key: entity_pb.Reference
Returns:
Tuple (app, kind), both are unicode strings.
"""
last_path = key.path().element_list()[-1]
return (datastore_types.EncodeAppIdNamespace(key.app(), key.name_space()),
last_path.type())
def _GetEntityLocation(self, key):
"""Get keys to self.__entities_by_* from the given key.
Example usage:
app_kind, eg_k, k = self._GetEntityLocation(key)
self.__entities_by_kind[app_kind][k]
self.__entities_by_entity_group[eg_k][k]
Args:
key: entity_pb.Reference
Returns:
Tuple (by_kind key, by_entity_group key, entity key)
"""
app_ns = datastore_types.EncodeAppIdNamespace(key.app(), key.name_space())
kind = key.path().element_list()[-1].type()
entity_group = datastore_stub_util._GetEntityGroup(key)
eg_k = datastore_types.ReferenceToKeyValue(entity_group)
k = datastore_types.ReferenceToKeyValue(key)
return ((app_ns, kind), eg_k, k)
def Query(self, query, filters, orders):
"""Perform a query on this pseudo-kind.
Args:
query: the original datastore_pb.Query.
filters: the filters from query.
orders: the orders from query.
Returns:
(results, remaining_filters, remaining_orders)
results is a list of entity_pb.EntityProto
remaining_filters and remaining_orders are the filters and orders that
should be applied in memory
"""
property_range = datastore_stub_util.ParsePropertyQuery(
query, filters, orders)
keys_only = query.keys_only()
app_namespace_str = datastore_types.EncodeAppIdNamespace(
query.app(), query.name_space())
properties = []
if keys_only:
usekey = '__property__keys'
else:
usekey = '__property__'
entities = self._stub._GetAllEntities()
for app_namespace, kind in entities:
if app_namespace != app_namespace_str: continue
app_kind = (app_namespace_str, kind)
kind = kind.decode('utf-8')
(start_cmp, end_cmp) = property_range.MapExtremes(
lambda extreme, inclusive, is_end: cmp(kind, extreme[0]))
if not ((start_cmp is None or start_cmp >= 0) and
(end_cmp is None or end_cmp <= 0)):
continue
kind_properties = self._stub._GetSchemaCache(app_kind, usekey)
if not kind_properties:
kind_properties = []
kind_key = datastore_types.Key.from_path(
KindPseudoKind.name,
kind,
_app=query.app(),
namespace=query.name_space())
props = collections.defaultdict(set)
for entity in entities[app_kind].values():
for prop in entity.protobuf.property_list():
prop_name = prop.name()
if (prop_name in datastore_stub_util.
GetInvisibleSpecialPropertyNames()):
continue
value_pb = prop.value()
props[prop_name].add(
datastore_types.GetPropertyValueTag(value_pb))
for prop in sorted(props):
property_e = datastore.Entity(self.name,
name=prop,
parent=kind_key,
_app=query.app(),
namespace=query.name_space())
if not keys_only and props[prop]:
property_e['property_representation'] = [
datastore_stub_util._PROPERTY_TYPE_NAMES[tag]
for tag in sorted(props[prop])
]
kind_properties.append(property_e._ToPb())
self._stub._SetSchemaCache(app_kind, usekey, kind_properties)
def InQuery(property_e):
return property_range.Contains(
(kind, _FinalElement(property_e.key()).name()))
properties += filter(InQuery, kind_properties)
return (properties, [], [])
def _Dynamic_GetSchema(self, req, schema):
""" Get the schema of a particular kind of entity. """
app_str = req.app()
self.__ValidateAppId(app_str)
namespace_str = req.name_space()
app_namespace_str = datastore_types.EncodeAppIdNamespace(
app_str, namespace_str)
kinds = []
for app_namespace, kind in self.__entities:
if (app_namespace != app_namespace_str
or (req.has_start_kind() and kind < req.start_kind())
or (req.has_end_kind() and kind > req.end_kind())):
continue
app_kind = (app_namespace_str, kind)
if app_kind in self.__schema_cache:
kinds.append(self.__schema_cache[app_kind])
continue
kind_pb = entity_pb.EntityProto()
kind_pb.mutable_key().set_app('')
kind_pb.mutable_key().mutable_path().add_element().set_type(kind)
kind_pb.mutable_entity_group()
props = {}
for entity in self.__entities[app_kind].values():
for prop in entity.protobuf.property_list():
if prop.name() not in props:
props[prop.name()] = entity_pb.PropertyValue()
props[prop.name()].MergeFrom(prop.value())
for value_pb in props.values():
if value_pb.has_int64value():
value_pb.set_int64value(0)
if value_pb.has_booleanvalue():
value_pb.set_booleanvalue(False)
if value_pb.has_stringvalue():
value_pb.set_stringvalue('none')
if value_pb.has_doublevalue():
value_pb.set_doublevalue(0.0)
if value_pb.has_pointvalue():
value_pb.mutable_pointvalue().set_x(0.0)
value_pb.mutable_pointvalue().set_y(0.0)
if value_pb.has_uservalue():
value_pb.mutable_uservalue().set_gaiaid(0)
value_pb.mutable_uservalue().set_email('none')
value_pb.mutable_uservalue().set_auth_domain('none')
value_pb.mutable_uservalue().clear_nickname()
value_pb.mutable_uservalue().clear_obfuscated_gaiaid()
if value_pb.has_referencevalue():
value_pb.clear_referencevalue()
value_pb.mutable_referencevalue().set_app('none')
pathelem = value_pb.mutable_referencevalue(
).add_pathelement()
pathelem.set_type('none')
pathelem.set_name('none')
for name, value_pb in props.items():
prop_pb = kind_pb.add_property()
prop_pb.set_name(name)
prop_pb.set_multiple(False)
prop_pb.mutable_value().CopyFrom(value_pb)
kinds.append(kind_pb)
self.__schema_cache[app_kind] = kind_pb
for kind_pb in kinds:
kind = schema.add_kind()
kind.CopyFrom(kind_pb)
if not req.properties():
kind.clear_property()
schema.set_more_results(False)
def _Dynamic_RunQuery(self, query, query_result):
if query.has_transaction():
self.__ValidateTransaction(query.transaction())
if not query.has_ancestor():
raise apiproxy_errors.ApplicationError(
datastore_pb.Error.BAD_REQUEST,
'Only ancestor queries are allowed inside transactions.')
entities = self.__tx_snapshot
else:
entities = self.__entities
app_id = query.app()
namespace = query.name_space()
self.__ValidateAppId(app_id)
if query.has_offset() and query.offset() > _MAX_QUERY_OFFSET:
raise apiproxy_errors.ApplicationError(
datastore_pb.Error.BAD_REQUEST, 'Too big query offset.')
num_components = len(query.filter_list()) + len(query.order_list())
if query.has_ancestor():
num_components += 1
if num_components > _MAX_QUERY_COMPONENTS:
raise apiproxy_errors.ApplicationError(
datastore_pb.Error.BAD_REQUEST,
('query is too large. may not have more than %s filters'
' + sort orders ancestor total' % _MAX_QUERY_COMPONENTS))
(filters, orders) = datastore_index.Normalize(query.filter_list(),
query.order_list())
if self.__require_indexes:
required, kind, ancestor, props, num_eq_filters = datastore_index.CompositeIndexForQuery(query)
if required:
required_key = kind, ancestor, props
indexes = self.__indexes.get(app_id)
if not indexes:
raise apiproxy_errors.ApplicationError(
datastore_pb.Error.NEED_INDEX,
"This query requires a composite index, but none are defined. "
"You must create an index.yaml file in your application root.")
eq_filters_set = set(props[:num_eq_filters])
remaining_filters = props[num_eq_filters:]
for index in indexes:
definition = datastore_index.ProtoToIndexDefinition(index)
index_key = datastore_index.IndexToKey(definition)
if required_key == index_key:
break
if num_eq_filters > 1 and (kind, ancestor) == index_key[:2]:
this_props = index_key[2]
this_eq_filters_set = set(this_props[:num_eq_filters])
this_remaining_filters = this_props[num_eq_filters:]
if (eq_filters_set == this_eq_filters_set and
remaining_filters == this_remaining_filters):
break
else:
raise apiproxy_errors.ApplicationError(
datastore_pb.Error.NEED_INDEX,
"This query requires a composite index that is not defined. "
"You must update the index.yaml file in your application root.")
try:
query.set_app(app_id)
datastore_types.SetNamespace(query, namespace)
encoded = datastore_types.EncodeAppIdNamespace(app_id, namespace)
if query.has_kind():
results = entities[encoded, query.kind()].values()
results = [entity.native for entity in results]
else:
results = []
for key in entities:
if key[0] == encoded:
results += [entity.native for entity in entities[key].values()]
except KeyError:
results = []
if query.has_ancestor():
ancestor_path = query.ancestor().path().element_list()
def is_descendant(entity):
path = entity.key()._Key__reference.path().element_list()
return path[:len(ancestor_path)] == ancestor_path
results = filter(is_descendant, results)
operators = {datastore_pb.Query_Filter.LESS_THAN: '<',
datastore_pb.Query_Filter.LESS_THAN_OR_EQUAL: '<=',
datastore_pb.Query_Filter.GREATER_THAN: '>',
datastore_pb.Query_Filter.GREATER_THAN_OR_EQUAL: '>=',
datastore_pb.Query_Filter.EQUAL: '==',
}
def has_prop_indexed(entity, prop):
"""Returns True if prop is in the entity and is indexed."""
if prop in datastore_types._SPECIAL_PROPERTIES:
return True
elif prop in entity.unindexed_properties():
return False
values = entity.get(prop, [])
if not isinstance(values, (tuple, list)):
values = [values]
for value in values:
if type(value) not in datastore_types._RAW_PROPERTY_TYPES:
return True
return False
for filt in filters:
assert filt.op() != datastore_pb.Query_Filter.IN
prop = filt.property(0).name().decode('utf-8')
op = operators[filt.op()]
filter_val_list = [datastore_types.FromPropertyPb(filter_prop)
for filter_prop in filt.property_list()]
def passes_filter(entity):
"""Returns True if the entity passes the filter, False otherwise.
The filter being evaluated is filt, the current filter that we're on
in the list of filters in the query.
"""
if not has_prop_indexed(entity, prop):
return False
try:
entity_vals = datastore._GetPropertyValue(entity, prop)
except KeyError:
entity_vals = []
if not isinstance(entity_vals, list):
entity_vals = [entity_vals]
for fixed_entity_val in entity_vals:
for filter_val in filter_val_list:
fixed_entity_type = self._PROPERTY_TYPE_TAGS.get(
fixed_entity_val.__class__)
filter_type = self._PROPERTY_TYPE_TAGS.get(filter_val.__class__)
if fixed_entity_type == filter_type:
comp = u'%r %s %r' % (fixed_entity_val, op, filter_val)
elif op != '==':
comp = '%r %s %r' % (fixed_entity_type, op, filter_type)
else:
continue
logging.log(logging.DEBUG - 1,
'Evaling filter expression "%s"', comp)
try:
ret = eval(comp)
if ret and ret != NotImplementedError:
return True
except TypeError:
pass
return False
results = filter(passes_filter, results)
for order in orders:
prop = order.property().decode('utf-8')
results = [entity for entity in results if has_prop_indexed(entity, prop)]
def order_compare_entities(a, b):
""" Return a negative, zero or positive number depending on whether
entity a is considered smaller than, equal to, or larger than b,
according to the query's orderings. """
cmped = 0
for o in orders:
prop = o.property().decode('utf-8')
reverse = (o.direction() is datastore_pb.Query_Order.DESCENDING)
a_val = datastore._GetPropertyValue(a, prop)
if isinstance(a_val, list):
a_val = sorted(a_val, order_compare_properties, reverse=reverse)[0]
b_val = datastore._GetPropertyValue(b, prop)
if isinstance(b_val, list):
b_val = sorted(b_val, order_compare_properties, reverse=reverse)[0]
cmped = order_compare_properties(a_val, b_val)
if o.direction() is datastore_pb.Query_Order.DESCENDING:
cmped = -cmped
if cmped != 0:
return cmped
if cmped == 0:
return cmp(a.key(), b.key())
def order_compare_properties(x, y):
"""Return a negative, zero or positive number depending on whether
property value x is considered smaller than, equal to, or larger than
property value y. If x and y are different types, they're compared based
on the type ordering used in the real datastore, which is based on the
tag numbers in the PropertyValue PB.
"""
if isinstance(x, datetime.datetime):
x = datastore_types.DatetimeToTimestamp(x)
if isinstance(y, datetime.datetime):
y = datastore_types.DatetimeToTimestamp(y)
x_type = self._PROPERTY_TYPE_TAGS.get(x.__class__)
y_type = self._PROPERTY_TYPE_TAGS.get(y.__class__)
if x_type == y_type:
try:
return cmp(x, y)
except TypeError:
return 0
else:
return cmp(x_type, y_type)
results.sort(order_compare_entities)
clone = datastore_pb.Query()
clone.CopyFrom(query)
clone.clear_hint()
if clone in self.__query_history:
self.__query_history[clone] += 1
else:
self.__query_history[clone] = 1
cursor = _Cursor(query, results, order_compare_entities)
self.__queries[cursor.cursor] = cursor
if query.has_count():
count = query.count()
elif query.has_limit():
count = query.limit()
else:
count = _BATCH_SIZE
cursor.PopulateQueryResult(query_result, count, compile=query.compile())
if query.compile():
compiled_query = query_result.mutable_compiled_query()
compiled_query.set_keys_only(query.keys_only())
compiled_query.mutable_primaryscan().set_index_name(query.Encode())
def Query(self, entities, query, filters, orders):
"""Perform a query on this pseudo-kind.
Args:
entities: all the app's entities.
query: the original datastore_pb.Query.
filters: the filters from query.
orders: the orders from query.
Returns:
(results, remaining_filters, remaining_orders)
results is a list of datastore.Entity
remaining_filters and remaining_orders are the filters and orders that
should be applied in memory
"""
property_range = datastore_stub_util.ParsePropertyQuery(
query, filters, orders)
keys_only = query.keys_only()
app_namespace_str = datastore_types.EncodeAppIdNamespace(
query.app(), query.name_space())
properties = []
if keys_only:
usekey = '__property__keys'
else:
usekey = '__property__'
for app_namespace, kind in entities:
if app_namespace != app_namespace_str: continue
(start_cmp, end_cmp) = property_range.MapExtremes(
lambda extreme, inclusive, is_end: cmp(kind, extreme[0]))
if not ((start_cmp is None or start_cmp >= 0) and
(end_cmp is None or end_cmp <= 0)):
continue
app_kind = (app_namespace_str, kind)
kind_properties = self.filestub._GetSchemaCache(app_kind, usekey)
if not kind_properties:
kind_properties = []
kind_key = datastore_types.Key.from_path(
KindPseudoKind.name, kind)
props = {}
for entity in entities[app_kind].values():
for prop in entity.protobuf.property_list():
prop_name = prop.name()
if (prop_name in datastore_stub_util.
GetInvisibleSpecialPropertyNames()):
continue
if prop_name not in props:
props[prop_name] = set()
native_value = entity.native[prop_name]
if not isinstance(native_value, list):
native_value = [native_value]
for value in native_value:
tag = self.filestub._PROPERTY_TYPE_TAGS.get(
value.__class__)
if tag is not None:
props[prop_name].add(tag)
else:
logging.warning(
'Unexpected value of class %s in datastore',
value.__class__)
for prop in sorted(props):
property_e = datastore.Entity(self.name,
name=prop,
parent=kind_key)
kind_properties.append(property_e)
if not keys_only and props[prop]:
property_e['property_representation'] = [
datastore_stub_util._PROPERTY_TYPE_NAMES[tag]
for tag in sorted(props[prop])
]
self.filestub._SetSchemaCache(app_kind, usekey,
kind_properties)
def InQuery(property_e):
return property_range.Contains((kind, property_e.key().name()))
properties += filter(InQuery, kind_properties)
return (properties, [], [])
