def _ProjectionPartialEntityGenerator(cursor):
"""Generator that creates partial entities for projection.
Generate partial datastore entities from a cursor, holding only the values
being projected. These entities might share a key.
Args:
cursor: a SQLite3.Cursor or subclass.
Yields:
Partial entities resulting from the projection.
"""
for row in cursor:
entity_original = entity_pb.EntityProto(row[1])
entity = entity_pb.EntityProto()
entity.mutable_key().MergeFrom(entity_original.key())
entity.mutable_entity_group().MergeFrom(entity_original.entity_group())
for name, value_data in zip(row[2::2], row[3::2]):
prop_to_add = entity.add_property()
prop_to_add.set_name(ToUtf8(name))
value_decoder = sortable_pb_encoder.Decoder(
array.array('B', str(value_data)))
prop_to_add.mutable_value().Merge(value_decoder)
prop_to_add.set_multiple(False)
datastore_stub_util.PrepareSpecialPropertiesForLoad(entity)
yield entity
def next(self):
row = self.__cursor.next()
while str(row[0]) in self.__seen:
row = self.__cursor.next()
self.__seen.add(str(row[0]))
entity = entity_pb.EntityProto(row[1])
datastore_stub_util.PrepareSpecialPropertiesForLoad(entity)
return entity
def _Next(self):
"""Fetches the next unique result from the result set.
Returns:
A datastore_pb.EntityProto instance.
"""
if self._HasNext():
self.__seen.add(self.__next_result[0])
entity = entity_pb.EntityProto(self.__next_result[1])
datastore_stub_util.PrepareSpecialPropertiesForLoad(entity)
self.__next_result = None, None
return entity
return None
def _Dynamic_Get(self, get_request, get_response):
if get_request.has_transaction():
self.__ValidateTransaction(get_request.transaction())
entities = self.__tx_snapshot
else:
entities = self.__entities
for key in get_request.key_list():
self.__ValidateAppId(key.app())
app_kind = self._AppIdNamespaceKindForKey(key)
group = get_response.add_entity()
try:
entity = entities[app_kind][key].protobuf
except KeyError:
entity = None
if entity:
group.mutable_entity().CopyFrom(entity)
datastore_stub_util.PrepareSpecialPropertiesForLoad(
group.mutable_entity())
def _DedupingEntityGenerator(cursor):
"""Generator that removes duplicate entities from the results.
Generate datastore entities from a cursor, skipping the duplicates
Args:
cursor: a SQLite3.Cursor or subclass.
Yields:
Entities that do not share a key.
"""
seen = set()
for row in cursor:
row_key, row_entity = row[:2]
encoded_row_key = str(row_key)
if encoded_row_key in seen:
continue
seen.add(encoded_row_key)
entity = entity_pb.EntityProto(row_entity)
datastore_stub_util.PrepareSpecialPropertiesForLoad(entity)
yield entity
def _Dynamic_RunQuery(self, query, query_result):
"""Send a query request to the datastore server. """
if query.has_transaction():
if not query.has_ancestor():
raise apiproxy_errors.ApplicationError(
datastore_pb.Error.BAD_REQUEST,
'Only ancestor queries are allowed inside transactions.')
(filters, orders) = datastore_index.Normalize(query.filter_list(),
query.order_list(), [])
datastore_stub_util.FillUsersInQuery(filters)
query_response = datastore_pb.QueryResult()
if not query.has_app():
query.set_app(self.__app_id)
self.__ValidateAppId(query.app())
self._RemoteSend(query, query_response, "RunQuery")
skipped_results = 0
if query_response.has_skipped_results():
skipped_results = query_response.skipped_results()
def has_prop_indexed(entity, prop):
"""Returns True if prop is in the entity and is indexed."""
if prop in datastore_types._SPECIAL_PROPERTIES:
return True
elif prop in entity.unindexed_properties():
return False
values = entity.get(prop, [])
if not isinstance(values, (tuple, list)):
values = [values]
for value in values:
if type(value) not in datastore_types._RAW_PROPERTY_TYPES:
return True
return False
def order_compare_entities(a, b):
""" Return a negative, zero or positive number depending on whether
entity a is considered smaller than, equal to, or larger than b,
according to the query's orderings. """
cmped = 0
for o in orders:
prop = o.property().decode('utf-8')
reverse = (o.direction() is
datastore_pb.Query_Order.DESCENDING)
a_val = datastore._GetPropertyValue(a, prop)
if isinstance(a_val, list):
a_val = sorted(a_val,
order_compare_properties,
reverse=reverse)[0]
b_val = datastore._GetPropertyValue(b, prop)
if isinstance(b_val, list):
b_val = sorted(b_val,
order_compare_properties,
reverse=reverse)[0]
cmped = order_compare_properties(a_val, b_val)
if o.direction() is datastore_pb.Query_Order.DESCENDING:
cmped = -cmped
if cmped != 0:
return cmped
if cmped == 0:
return cmp(a.key(), b.key())
def order_compare_entities_pb(a, b):
""" Return a negative, zero or positive number depending on whether
entity a is considered smaller than, equal to, or larger than b,
according to the query's orderings. a and b are protobuf-encoded
entities."""
return order_compare_entities(datastore.Entity.FromPb(a),
datastore.Entity.FromPb(b))
def order_compare_properties(x, y):
"""Return a negative, zero or positive number depending on whether
property value x is considered smaller than, equal to, or larger than
property value y. If x and y are different types, they're compared based
on the type ordering used in the real datastore, which is based on the
tag numbers in the PropertyValue PB.
"""
if isinstance(x, datetime.datetime):
x = datastore_types.DatetimeToTimestamp(x)
if isinstance(y, datetime.datetime):
y = datastore_types.DatetimeToTimestamp(y)
x_type = self._PROPERTY_TYPE_TAGS.get(x.__class__)
y_type = self._PROPERTY_TYPE_TAGS.get(y.__class__)
if x_type == y_type:
try:
return cmp(x, y)
except TypeError:
return 0
else:
return cmp(x_type, y_type)
results = query_response.result_list()
results = [datastore.Entity._FromPb(r) for r in results]
results = [r._ToPb() for r in results]
for result in results:
datastore_stub_util.PrepareSpecialPropertiesForLoad(result)
datastore_stub_util.ValidateQuery(query, filters, orders,
_MAX_QUERY_COMPONENTS)
cursor = datastore_stub_util.ListCursor(query, results,
order_compare_entities_pb)
self.__cleanup_old_cursors()
self.__queries[cursor.cursor] = cursor, datetime.datetime.now()
if query.has_count():
count = query.count()
elif query.has_limit():
count = query.limit()
else:
count = _BATCH_SIZE
cursor.PopulateQueryResult(query_result,
count,
query.offset(),
compile=query.compile())
query_result.set_skipped_results(skipped_results)
if query.compile():
compiled_query = query_result.mutable_compiled_query()
compiled_query.set_keys_only(query.keys_only())
compiled_query.mutable_primaryscan().set_index_name(query.Encode())
def _Dynamic_RunQuery(self, query, query_result):
if query.has_transaction():
self.__ValidateTransaction(query.transaction())
entities = self.__tx_snapshot
else:
entities = self.__entities
app_id = query.app()
namespace = query.name_space()
self.__ValidateAppId(app_id)
(filters, orders) = datastore_index.Normalize(query.filter_list(),
query.order_list())
datastore_stub_util.ValidateQuery(query, filters, orders,
_MAX_QUERY_COMPONENTS)
datastore_stub_util.FillUsersInQuery(filters)
pseudo_kind = None
if query.has_kind() and query.kind() in self.__pseudo_kinds:
pseudo_kind = self.__pseudo_kinds[query.kind()]
if not pseudo_kind and self.__require_indexes:
required, kind, ancestor, props, num_eq_filters = datastore_index.CompositeIndexForQuery(
query)
if required:
required_key = kind, ancestor, props
indexes = self.__indexes.get(app_id)
if not indexes:
raise apiproxy_errors.ApplicationError(
datastore_pb.Error.NEED_INDEX,
"This query requires a composite index, but none are defined. "
"You must create an index.yaml file in your application root."
)
eq_filters_set = set(props[:num_eq_filters])
remaining_filters = props[num_eq_filters:]
for index in indexes:
definition = datastore_index.ProtoToIndexDefinition(index)
index_key = datastore_index.IndexToKey(definition)
if required_key == index_key:
break
if num_eq_filters > 1 and (kind,
ancestor) == index_key[:2]:
this_props = index_key[2]
this_eq_filters_set = set(this_props[:num_eq_filters])
this_remaining_filters = this_props[num_eq_filters:]
if (eq_filters_set == this_eq_filters_set and
remaining_filters == this_remaining_filters):
break
else:
raise apiproxy_errors.ApplicationError(
datastore_pb.Error.NEED_INDEX,
"This query requires a composite index that is not defined. "
"You must update the index.yaml file in your application root."
)
try:
query.set_app(app_id)
datastore_types.SetNamespace(query, namespace)
encoded = datastore_types.EncodeAppIdNamespace(app_id, namespace)
if pseudo_kind:
(results, filters,
orders) = pseudo_kind.Query(entities, query, filters, orders)
elif query.has_kind():
results = entities[encoded, query.kind()].values()
results = [entity.native for entity in results]
else:
results = []
for key in entities:
if key[0] == encoded:
results += [
entity.native for entity in entities[key].values()
]
except KeyError:
results = []
if query.has_ancestor():
ancestor_path = query.ancestor().path().element_list()
def is_descendant(entity):
path = entity.key()._Key__reference.path().element_list()
return path[:len(ancestor_path)] == ancestor_path
results = filter(is_descendant, results)
operators = {
datastore_pb.Query_Filter.LESS_THAN: '<',
datastore_pb.Query_Filter.LESS_THAN_OR_EQUAL: '<=',
datastore_pb.Query_Filter.GREATER_THAN: '>',
datastore_pb.Query_Filter.GREATER_THAN_OR_EQUAL: '>=',
datastore_pb.Query_Filter.EQUAL: '==',
}
def has_prop_indexed(entity, prop):
"""Returns True if prop is in the entity and is indexed."""
if prop in datastore_types._SPECIAL_PROPERTIES:
return True
elif prop in entity.unindexed_properties():
return False
values = entity.get(prop, [])
if not isinstance(values, (tuple, list)):
values = [values]
for value in values:
if type(value) not in datastore_types._RAW_PROPERTY_TYPES:
return True
return False
for filt in filters:
assert filt.op() != datastore_pb.Query_Filter.IN
prop = filt.property(0).name().decode('utf-8')
op = operators[filt.op()]
filter_val_list = [
datastore_types.FromPropertyPb(filter_prop)
for filter_prop in filt.property_list()
]
def passes_filter(entity):
"""Returns True if the entity passes the filter, False otherwise.
The filter being evaluated is filt, the current filter that we're on
in the list of filters in the query.
"""
if not has_prop_indexed(entity, prop):
return False
try:
entity_vals = datastore._GetPropertyValue(entity, prop)
except KeyError:
entity_vals = []
if not isinstance(entity_vals, list):
entity_vals = [entity_vals]
for fixed_entity_val in entity_vals:
for filter_val in filter_val_list:
fixed_entity_type = self._PROPERTY_TYPE_TAGS.get(
fixed_entity_val.__class__)
filter_type = self._PROPERTY_TYPE_TAGS.get(
filter_val.__class__)
if fixed_entity_type == filter_type:
comp = u'%r %s %r' % (fixed_entity_val, op,
filter_val)
elif op != '==':
comp = '%r %s %r' % (fixed_entity_type, op,
filter_type)
else:
continue
logging.log(logging.DEBUG - 1,
'Evaling filter expression "%s"', comp)
try:
ret = eval(comp)
if ret and ret != NotImplementedError:
return True
except TypeError:
pass
return False
results = filter(passes_filter, results)
for order in orders:
prop = order.property().decode('utf-8')
results = [
entity for entity in results if has_prop_indexed(entity, prop)
]
def order_compare_entities(a, b):
""" Return a negative, zero or positive number depending on whether
entity a is considered smaller than, equal to, or larger than b,
according to the query's orderings. """
cmped = 0
for o in orders:
prop = o.property().decode('utf-8')
reverse = (o.direction() is
datastore_pb.Query_Order.DESCENDING)
a_val = datastore._GetPropertyValue(a, prop)
if isinstance(a_val, list):
a_val = sorted(a_val,
order_compare_properties,
reverse=reverse)[0]
b_val = datastore._GetPropertyValue(b, prop)
if isinstance(b_val, list):
b_val = sorted(b_val,
order_compare_properties,
reverse=reverse)[0]
cmped = order_compare_properties(a_val, b_val)
if o.direction() is datastore_pb.Query_Order.DESCENDING:
cmped = -cmped
if cmped != 0:
return cmped
if cmped == 0:
return cmp(a.key(), b.key())
def order_compare_entities_pb(a, b):
""" Return a negative, zero or positive number depending on whether
entity a is considered smaller than, equal to, or larger than b,
according to the query's orderings. a and b are protobuf-encoded
entities."""
return order_compare_entities(datastore.Entity.FromPb(a),
datastore.Entity.FromPb(b))
def order_compare_properties(x, y):
"""Return a negative, zero or positive number depending on whether
property value x is considered smaller than, equal to, or larger than
property value y. If x and y are different types, they're compared based
on the type ordering used in the real datastore, which is based on the
tag numbers in the PropertyValue PB.
"""
if isinstance(x, datetime.datetime):
x = datastore_types.DatetimeToTimestamp(x)
if isinstance(y, datetime.datetime):
y = datastore_types.DatetimeToTimestamp(y)
x_type = self._PROPERTY_TYPE_TAGS.get(x.__class__)
y_type = self._PROPERTY_TYPE_TAGS.get(y.__class__)
if x_type == y_type:
try:
return cmp(x, y)
except TypeError:
return 0
else:
return cmp(x_type, y_type)
results.sort(order_compare_entities)
clone = datastore_pb.Query()
clone.CopyFrom(query)
clone.clear_hint()
clone.clear_limit()
clone.clear_offset()
if clone in self.__query_history:
self.__query_history[clone] += 1
else:
self.__query_history[clone] = 1
results = [r._ToPb() for r in results]
for result in results:
datastore_stub_util.PrepareSpecialPropertiesForLoad(result)
cursor = datastore_stub_util.ListCursor(query, results,
order_compare_entities_pb)
if query.has_count():
count = query.count()
elif query.has_limit():
count = query.limit()
else:
count = _BATCH_SIZE
cursor.PopulateQueryResult(query_result,
count,
query.offset(),
compile=query.compile())
if query.compile():
compiled_query = query_result.mutable_compiled_query()
compiled_query.set_keys_only(query.keys_only())
compiled_query.mutable_primaryscan().set_index_name(query.Encode())
if query_result.has_cursor():
self.__queries[query_result.cursor().cursor()] = cursor
def _Dynamic_RunQuery(self, query, query_result):
if query.has_transaction():
if not query.has_ancestor():
raise apiproxy_errors.ApplicationError(
datastore_pb.Error.BAD_REQUEST,
'Only ancestor queries are allowed inside transactions.')
(filters, orders) = datastore_index.Normalize(query.filter_list(),
query.order_list())
datastore_stub_util.FillUsersInQuery(filters)
query_response = datastore_pb.QueryResult()
query.set_app(self.__app_id)
self._RemoteSend(query, query_response, "RunQuery")
results = query_response.result_list()
results = [datastore.Entity._FromPb(r) for r in results]
if query.has_ancestor():
ancestor_path = query.ancestor().path().element_list()
def is_descendant(entity):
path = entity.key()._Key__reference.path().element_list()
return path[:len(ancestor_path)] == ancestor_path
results = filter(is_descendant, results)
operators = {
datastore_pb.Query_Filter.LESS_THAN: '<',
datastore_pb.Query_Filter.LESS_THAN_OR_EQUAL: '<=',
datastore_pb.Query_Filter.GREATER_THAN: '>',
datastore_pb.Query_Filter.GREATER_THAN_OR_EQUAL: '>=',
datastore_pb.Query_Filter.EQUAL: '==',
}
def has_prop_indexed(entity, prop):
"""Returns True if prop is in the entity and is indexed."""
if prop in datastore_types._SPECIAL_PROPERTIES:
return True
elif prop in entity.unindexed_properties():
return False
values = entity.get(prop, [])
if not isinstance(values, (tuple, list)):
values = [values]
for value in values:
if type(value) not in datastore_types._RAW_PROPERTY_TYPES:
return True
return False
for filt in filters:
assert filt.op() != datastore_pb.Query_Filter.IN
prop = filt.property(0).name().decode('utf-8')
op = operators[filt.op()]
filter_val_list = [
datastore_types.FromPropertyPb(filter_prop)
for filter_prop in filt.property_list()
]
for order in orders:
prop = order.property().decode('utf-8')
results = [
entity for entity in results if has_prop_indexed(entity, prop)
]
def order_compare_entities(a, b):
""" Return a negative, zero or positive number depending on whether
entity a is considered smaller than, equal to, or larger than b,
according to the query's orderings. """
cmped = 0
for o in orders:
prop = o.property().decode('utf-8')
reverse = (o.direction() is
datastore_pb.Query_Order.DESCENDING)
a_val = datastore._GetPropertyValue(a, prop)
if isinstance(a_val, list):
a_val = sorted(a_val,
order_compare_properties,
reverse=reverse)[0]
b_val = datastore._GetPropertyValue(b, prop)
if isinstance(b_val, list):
b_val = sorted(b_val,
order_compare_properties,
reverse=reverse)[0]
cmped = order_compare_properties(a_val, b_val)
if o.direction() is datastore_pb.Query_Order.DESCENDING:
cmped = -cmped
if cmped != 0:
return cmped
if cmped == 0:
return cmp(a.key(), b.key())
def order_compare_entities_pb(a, b):
""" Return a negative, zero or positive number depending on whether
entity a is considered smaller than, equal to, or larger than b,
according to the query's orderings. a and b are protobuf-encoded
entities."""
return order_compare_entities(datastore.Entity.FromPb(a),
datastore.Entity.FromPb(b))
def order_compare_properties(x, y):
"""Return a negative, zero or positive number depending on whether
property value x is considered smaller than, equal to, or larger than
property value y. If x and y are different types, they're compared based
on the type ordering used in the real datastore, which is based on the
tag numbers in the PropertyValue PB.
"""
if isinstance(x, datetime.datetime):
x = datastore_types.DatetimeToTimestamp(x)
if isinstance(y, datetime.datetime):
y = datastore_types.DatetimeToTimestamp(y)
x_type = self._PROPERTY_TYPE_TAGS.get(x.__class__)
y_type = self._PROPERTY_TYPE_TAGS.get(y.__class__)
if x_type == y_type:
try:
return cmp(x, y)
except TypeError:
return 0
else:
return cmp(x_type, y_type)
results.sort(order_compare_entities)
clone = datastore_pb.Query()
clone.CopyFrom(query)
clone.clear_hint()
clone.clear_limit()
clone.clear_offset()
#if clone in self.__query_history:
# self.__query_history[clone] += 1
#else:
# self.__query_history[clone] = 1
results = [r._ToPb() for r in results]
for result in results:
datastore_stub_util.PrepareSpecialPropertiesForLoad(result)
datastore_stub_util.ValidateQuery(query, filters, orders,
_MAX_QUERY_COMPONENTS)
cursor = datastore_stub_util.ListCursor(query, results,
order_compare_entities_pb)
self.__queries[cursor.cursor] = cursor
if query.has_count():
count = query.count()
elif query.has_limit():
count = query.limit()
else:
count = _BATCH_SIZE
cursor.PopulateQueryResult(query_result,
count,
query.offset(),
compile=query.compile())
if query.compile():
compiled_query = query_result.mutable_compiled_query()
compiled_query.set_keys_only(query.keys_only())
compiled_query.mutable_primaryscan().set_index_name(query.Encode())
