def __ValidateAppId(self, app_id):
"""Verify that this is the stub for app_id.
Args:
app_id: An application ID.
Raises:
datastore_errors.BadRequestError: if this is not the stub for app_id.
"""
assert app_id
if not self.__trusted and app_id != self.__app_id:
raise datastore_errors.BadRequestError(
'app %s cannot access app %s\'s data' % (self.__app_id, app_id))
def __ValidateKey(self, key):
"""Validate this key.
Args:
key: entity_pb.Reference
Raises:
datastore_errors.BadRequestError: if the key is invalid
"""
assert isinstance(key, entity_pb.Reference)
self.__ValidateAppId(key.app())
for elem in key.path().element_list():
if elem.has_id() == elem.has_name():
raise datastore_errors.BadRequestError(
'each key path element should have id or name but not both: %r' % key)
def _Dynamic_RunQuery(self, query, query_result):
if query.keys_only():
query_result.set_keys_only(True)
num_components = len(query.filter_list()) + len(query.order_list())
if query.has_ancestor():
num_components += 1
if num_components > _MAX_QUERY_COMPONENTS:
raise apiproxy_errors.ApplicationError(
datastore_pb.Error.BAD_REQUEST,
('query is too large. may not have more than %s filters'
' + sort orders ancestor total' % _MAX_QUERY_COMPONENTS))
app = query.app()
query_result.mutable_cursor().set_cursor(0)
query_result.set_more_results(False)
if self.__require_indexes:
(required, kind, ancestor, props,
num_eq_filters) = (datastore_index.CompositeIndexForQuery(query))
if required:
index = entity_pb.CompositeIndex()
index.mutable_definition().set_entity_type(kind)
index.mutable_definition().set_ancestor(ancestor)
for (k, v) in props:
p = index.mutable_definition().add_property()
p.set_name(k)
p.set_direction(v)
if props and not self.__has_index(index):
raise apiproxy_errors.ApplicationError(
datastore_pb.Error.NEED_INDEX,
"This query requires a composite index that is not defined. "
"You must update the index.yaml file in your application root."
)
collection = query.kind()
if query.has_name_space():
collection = query.name_space(
) + _NAMESPACE_CONCAT_STR + collection
clone = datastore_pb.Query()
clone.CopyFrom(query)
clone.clear_hint()
if clone in self.__query_history:
self.__query_history[clone] += 1
else:
self.__query_history[clone] = 1
# HACK we need to get one Entity from this collection so we know what the
# property types are (because we need to construct queries that depend on
# the types of the properties)...
try:
prototype = self.__db[collection].find_one()
except pymongo.errors.InvalidName:
raise datastore_errors.BadRequestError('query without kind')
if prototype is None:
return
prototype = datastore.Entity._FromPb(
self.__entity_for_mongo_document(prototype))
spec = {}
if query.has_ancestor():
spec["_id"] = re.compile("^%s.*$" %
self.__id_for_key(query.ancestor()))
operators = {
datastore_pb.Query_Filter.LESS_THAN: '<',
datastore_pb.Query_Filter.LESS_THAN_OR_EQUAL: '<=',
datastore_pb.Query_Filter.GREATER_THAN: '>',
datastore_pb.Query_Filter.GREATER_THAN_OR_EQUAL: '>=',
datastore_pb.Query_Filter.EQUAL: '==',
}
for filt in query.filter_list():
assert filt.op() != datastore_pb.Query_Filter.IN
prop = filt.property(0).name().decode('utf-8')
op = operators[filt.op()]
filter_val_list = [
datastore_types.FromPropertyPb(filter_prop)
for filter_prop in filt.property_list()
]
(key, value) = self.__filter_binding(prop, filter_val_list[0], op,
prototype)
if key in spec:
if (not isinstance(spec[key], types.DictType)
and not isinstance(value, types.DictType)):
if spec[key] != value:
return
elif not isinstance(spec[key], types.DictType):
value["$in"] = [spec[key]]
spec[key] = value
elif not isinstance(value, types.DictType):
spec[key]["$in"] = [value]
else:
spec[key].update(value)
else:
spec[key] = value
offset = 0
# Cursor magic
if query.has_compiled_cursor():
offset, query_pb, unused_spec, incl = self._DecodeCompiledCursor(
query.compiled_cursor())
cursor = self.__db[collection].find(spec)
order = self.__translate_order_for_mongo(query.order_list(), prototype)
if order is None:
return
if order:
cursor = cursor.sort(order)
if query.offset() == datastore._MAX_INT_32:
query.set_offset(0)
query.set_limit(datastore._MAX_INT_32)
if offset:
cursor = cursor.skip(int(offset))
elif query.has_offset() and query.offset() != _MAX_QUERY_OFFSET:
cursor = cursor.skip(int(query.offset()))
if query.has_limit():
cursor = cursor.limit(int(query.limit()))
self.__cursor_lock.acquire()
cursor_index = self.__next_cursor
self.__next_cursor += 1
self.__cursor_lock.release()
self.__queries[cursor_index] = cursor
# Cursor magic
compiled_cursor = query_result.mutable_compiled_cursor()
position = compiled_cursor.add_position()
query_info = self._MinimalQueryInfo(query)
cloned_cursor = cursor.clone()
results = list(cloned_cursor)
if results:
start_key = _CURSOR_CONCAT_STR.join(
(str(len(results) + offset), query_info.Encode(),
self.__entity_for_mongo_document(results[-1]).Encode()))
# Populate query result
result_list = query_result.result_list()
for doc in results:
result_list.append(self.__entity_for_mongo_document(doc))
query_result.set_skipped_results(len(results))
position.set_start_key(str(start_key))
position.set_start_inclusive(False)
del cloned_cursor
query_result.mutable_cursor().set_cursor(cursor_index)
query_result.set_more_results(False)
def transaction(self, callback, **ctx_options):
options = _make_ctx_options(ctx_options, TransactionOptions)
propagation = TransactionOptions.propagation(options)
if propagation is None:
propagation = TransactionOptions.NESTED
mode = datastore_rpc.TransactionMode.READ_WRITE
if ctx_options.get('read_only', False):
mode = datastore_rpc.TransactionMode.READ_ONLY
parent = self
if propagation == TransactionOptions.NESTED:
if self.in_transaction():
raise datastore_errors.BadRequestError(
'Nested transactions are not supported.')
elif propagation == TransactionOptions.MANDATORY:
if not self.in_transaction():
raise datastore_errors.BadRequestError(
'Requires an existing transaction.')
result = callback()
if isinstance(result, tasklets.Future):
result = yield result
raise tasklets.Return(result)
elif propagation == TransactionOptions.ALLOWED:
if self.in_transaction():
result = callback()
if isinstance(result, tasklets.Future):
result = yield result
raise tasklets.Return(result)
elif propagation == TransactionOptions.INDEPENDENT:
while parent.in_transaction():
parent = parent._parent_context
if parent is None:
raise datastore_errors.BadRequestError(
'Context without non-transactional ancestor')
else:
raise datastore_errors.BadArgumentError(
'Invalid propagation value (%s).' % (propagation, ))
app = TransactionOptions.app(options) or key_module._DefaultAppId()
retries = TransactionOptions.retries(options)
if retries is None:
retries = 3
yield parent.flush()
transaction = None
tconn = None
for _ in range(1 + max(0, retries)):
previous_transaction = (transaction if mode
== datastore_rpc.TransactionMode.READ_WRITE
else None)
transaction = yield (parent._conn.async_begin_transaction(
options, app, previous_transaction, mode))
tconn = datastore_rpc.TransactionalConnection(
adapter=parent._conn.adapter,
config=parent._conn.config,
transaction=transaction,
_api_version=parent._conn._api_version)
tctx = parent.__class__(
conn=tconn,
auto_batcher_class=parent._auto_batcher_class,
parent_context=parent)
tctx._old_ds_conn = datastore._GetConnection()
ok = False
try:
tctx.set_memcache_policy(parent.get_memcache_policy())
tctx.set_memcache_timeout_policy(
parent.get_memcache_timeout_policy())
tasklets.set_context(tctx)
datastore._SetConnection(tconn)
try:
try:
result = callback()
if isinstance(result, tasklets.Future):
result = yield result
finally:
yield tctx.flush()
except GeneratorExit:
raise
except Exception:
t, e, tb = sys.exc_info()
tconn.async_rollback(options)
if issubclass(t, datastore_errors.Rollback):
return
else:
six.reraise(t, e, tb)
else:
ok = yield tconn.async_commit(options)
if ok:
parent._cache.update(tctx._cache)
yield parent._clear_memcache(tctx._cache)
raise tasklets.Return(result)
finally:
datastore._SetConnection(tctx._old_ds_conn)
del tctx._old_ds_conn
if ok:
for on_commit_callback in tctx._on_commit_queue:
on_commit_callback()
tconn.async_rollback(options)
raise datastore_errors.TransactionFailedError(
'The transaction could not be committed. Please try again.')
def MockNdbTransaction(func, **options):
if len(calls) < 1:
calls.append(1)
raise datastore_errors.BadRequestError('Other reason')
return original_step_put(func, **options)
def MockNdbTransaction(func, **options):
if len(calls) < 1:
calls.append(1)
raise datastore_errors.BadRequestError()
return original_ndb_transaction(func, **options)
