def __init__(self, encoded=None):
"""Constructor. Creates a Key from a string.
Args:
# a base64-encoded primary key, generated by Key.__str__
encoded: str
"""
if encoded is not None:
if not isinstance(encoded, basestring):
try:
repr_encoded = repr(encoded)
except:
repr_encoded = "<couldn't encode>"
raise datastore_errors.BadArgumentError(
'Key() expects a string; received %s (a %s).' %
(repr_encoded, typename(encoded)))
try:
modulo = len(encoded) % 4
if modulo != 0:
encoded += ('=' * (4 - modulo))
encoded_pb = base64.urlsafe_b64decode(str(encoded))
self.__reference = entity_pb.Reference(encoded_pb)
assert self.__reference.IsInitialized()
except (AssertionError, TypeError), e:
raise datastore_errors.BadKeyError(
'Invalid string key %s. Details: %s' % (encoded, e))
except Exception, e:
if e.__class__.__name__ == 'ProtocolBufferDecodeError':
raise datastore_errors.BadKeyError('Invalid string key %s.' % encoded)
else:
raise
def _put_tasklet(self, todo):
assert todo
# TODO: What if the same entity is being put twice?
# TODO: What if two entities with the same key are being put?
# TODO: Clear entities from memcache before starting the write?
# TODO: Attempt to prevent dogpile effect while keeping cache consistent?
ents = [ent for (_, ent) in todo]
results = yield self._conn.async_put(None, ents)
for key, (fut, ent) in zip(results, todo):
if key != ent.key:
if ent._has_complete_key():
raise datastore_errors.BadKeyError(
'Entity key differs from the one returned by the datastore. '
'Expected %r, got %r' % (key, ent.key))
ent.key = key
fut.set_result(key)
# Now update memcache.
# TODO: Could we update memcache *before* calling async_put()?
# (Hm, not for new entities but possibly for updated ones.)
mapping = {}
for _, ent in todo:
if self.should_memcache(ent.key):
pb = self._conn.adapter.entity_to_pb(ent)
mapping[ent.key.urlsafe()] = pb
if mapping:
# TODO: Optionally set the memcache expiration time;
# maybe configurable based on key (or even entity).
failures = memcache.set_multi(mapping)
if failures:
badkeys = []
for failure in failures:
badkeys.append(mapping[failure].key)
logging.info('memcache failed to set %d out of %d keys: %s',
len(failures), len(mapping), badkeys)
def ToTagUri(self):
"""Returns a tag: URI for this entity for use in XML output.
Foreign keys for entities may be represented in XML output as tag URIs.
RFC 4151 describes the tag URI scheme. From http://taguri.org/:
The tag algorithm lets people mint - create - identifiers that no one
else using the same algorithm could ever mint. It is simple enough to do
in your head, and the resulting identifiers can be easy to read, write,
and remember. The identifiers conform to the URI (URL) Syntax.
Tag URIs for entities use the app's auth domain and the date that the URI
is generated. The namespace-specific part is <kind>[<key>].
For example, here is the tag URI for a Kitten with the key "Fluffy" in the
catsinsinks app:
tag:catsinsinks.googleapps.com,2006-08-29:Kitten[Fluffy]
Raises a BadKeyError if this entity's key is incomplete.
"""
if not self.has_id_or_name():
raise datastore_errors.BadKeyError(
'ToTagUri() called for an entity with an incomplete key.')
return u'tag:%s.%s,%s:%s[%s]' % (saxutils.escape(self.app()),
os.environ['AUTH_DOMAIN'],
datetime.date.today().isoformat(),
saxutils.escape(self.kind()),
saxutils.escape(str(self)))
def _put_tasklet(self, todo, options):
if not todo:
raise RuntimeError('Nothing to do.')
# TODO: What if the same entity is being put twice?
# TODO: What if two entities with the same key are being put?
datastore_entities = []
for unused_fut, ent in todo:
datastore_entities.append(ent)
# Wait for datastore RPC(s).
keys = yield self._conn.async_put(options, datastore_entities)
for key, (fut, ent) in zip(keys, todo):
if key != ent._key:
if ent._has_complete_key():
raise datastore_errors.BadKeyError(
'Entity key differs from the one returned by the datastore. '
'Expected %r, got %r' % (key, ent._key))
ent._key = key
fut.set_result(key)
def _put_tasklet(self, todo, options):
if not todo:
raise RuntimeError('Nothing to do.')
datastore_entities = []
for unused_fut, ent in todo:
datastore_entities.append(ent)
keys = yield self._conn.async_put(options, datastore_entities)
for key, (fut, ent) in zip(keys, todo):
if key != ent._key:
if ent._has_complete_key():
ent_key = ent._key
raise datastore_errors.BadKeyError(
'Entity Key differs from the one returned by Datastore. '
'Returned Key: %r, Entity Key: %r' % (key, ent_key))
ent._key = key
fut.set_result(key)
def _put_tasklet(self, todo):
assert todo
# TODO: What if the same entity is being put twice?
# TODO: What if two entities with the same key are being put?
# TODO: Clear entities from memcache before starting the write?
# TODO: Attempt to prevent dogpile effect while keeping cache consistent?
ents = [ent for (_, ent) in todo]
results = yield self._conn.async_put(None, ents)
for key, (fut, ent) in zip(results, todo):
if key != ent._key:
if ent._has_complete_key():
raise datastore_errors.BadKeyError(
'Entity key differs from the one returned by the datastore. '
'Expected %r, got %r' % (key, ent._key))
ent._key = key
fut.set_result(key)
# Now update memcache.
# TODO: Could we update memcache *before* calling async_put()?
# (Hm, not for new entities but possibly for updated ones.)
mappings = {} # Maps timeout value to {urlsafe_key: pb} mapping.
for _, ent in todo:
if self.should_memcache(ent._key):
pb = self._conn.adapter.entity_to_pb(ent)
timeout = self._memcache_timeout_policy(ent._key)
mapping = mappings.get(timeout)
if mapping is None:
mapping = mappings[timeout] = {}
mapping[ent._key.urlsafe()] = pb
if mappings:
# If the timeouts are not uniform, make a separate call for each
# distinct timeout value.
for timeout, mapping in mappings.iteritems():
failures = memcache.set_multi(mapping,
time=timeout,
key_prefix=self._memcache_prefix)
if failures:
badkeys = []
for failure in failures:
badkeys.append(mapping[failure].key)
logging.info(
'memcache failed to set %d out of %d keys: %s',
len(failures), len(mapping), badkeys)
def __str__(self):
"""Encodes this Key as an opaque string.
Returns a string representation of this key, suitable for use in HTML,
URLs, and other similar use cases. If the entity's key is incomplete,
raises a BadKeyError.
Unfortunately, this string encoding isn't particularly compact, and its
length varies with the length of the path. If you want a shorter identifier
and you know the kind and parent (if any) ahead of time, consider using just
the entity's id or name.
Returns:
string
"""
if (self.has_id_or_name()):
encoded = base64.urlsafe_b64encode(self.__reference.Encode())
return encoded.replace('=', '')
else:
raise datastore_errors.BadKeyError(
'Cannot string encode an incomplete key!\n%s' % self.__reference)
def _put_tasklet(self, todo):
assert todo
# TODO: What if the same entity is being put twice?
# TODO: What if two entities with the same key are being put?
by_options = {}
delete_keys = [] # For memcache.delete_multi().
mappings = {} # For memcache.set_multi(), segregated by timeout.
for fut, ent, options in todo:
if ent._has_complete_key():
if self._use_memcache(ent._key, options):
if self._use_datastore(ent._key, options):
delete_keys.append(ent._key.urlsafe())
else:
pb = self._conn.adapter.entity_to_pb(ent)
timeout = self._get_memcache_timeout(ent._key, options)
mapping = mappings.get(timeout)
if mapping is None:
mapping = mappings[timeout] = {}
mapping[ent._key.urlsafe()] = pb
else:
key = ent._key
if key is None:
# Create a dummy Key to call _use_datastore().
key = model.Key(ent.__class__, None)
if not self._use_datastore(key, options):
raise datastore_errors.BadKeyError(
'Cannot put incomplete key when use_datastore=False.')
if options in by_options:
futures, entities = by_options[options]
else:
futures, entities = by_options[options] = [], []
futures.append(fut)
entities.append(ent)
if delete_keys: # Pre-emptively delete from memcache.
memcache.delete_multi(delete_keys,
seconds=_LOCK_TIME,
key_prefix=self._memcache_prefix)
if mappings: # Write to memcache (only if use_datastore=False).
# If the timeouts are not uniform, make a separate call for each
# distinct timeout value.
for timeout, mapping in mappings.iteritems():
# Use add, not set. This is a no-op within _LOCK_TIME seconds
# of the delete done by the most recent write.
memcache.add_multi(mapping,
time=timeout,
key_prefix=self._memcache_prefix)
for options, (futures, entities) in by_options.iteritems():
datastore_futures = []
datastore_entities = []
for fut, ent in zip(futures, entities):
key = ent._key
if key is None:
# Pass a dummy Key to _use_datastore().
key = model.Key(ent.__class__, None)
if self._use_datastore(key, options):
datastore_futures.append(fut)
datastore_entities.append(ent)
else:
# TODO: If ent._key is None, this is really lame.
fut.set_result(ent._key)
if datastore_entities:
keys = yield self._conn.async_put(options, datastore_entities)
for key, fut, ent in zip(keys, datastore_futures,
datastore_entities):
if key != ent._key:
if ent._has_complete_key():
raise datastore_errors.BadKeyError(
'Entity key differs from the one returned by the datastore. '
'Expected %r, got %r' % (key, ent._key))
ent._key = key
fut.set_result(key)
def from_path(*args, **kwds):
"""Static method to construct a Key out of a "path" (kind, id or name, ...).
This is useful when an application wants to use just the id or name portion
of a key in e.g. a URL, where the rest of the URL provides enough context to
fill in the rest, i.e. the app id (always implicit), the entity kind, and
possibly an ancestor key. Since ids and names are usually small, they're
more attractive for use in end-user-visible URLs than the full string
representation of a key.
Args:
kind: the entity kind (a str or unicode instance)
id_or_name: the id (an int or long) or name (a str or unicode instance)
Additional positional arguments are allowed and should be
alternating kind and id/name.
Keyword args:
parent: optional parent Key; default None.
Returns:
A new Key instance whose .kind() and .id() or .name() methods return
the *last* kind and id or name positional arguments passed.
Raises:
BadArgumentError for invalid arguments.
BadKeyError if the parent key is incomplete.
"""
parent = kwds.pop('parent', None)
_app = kwds.pop('_app', None)
if kwds:
raise datastore_errors.BadArgumentError(
'Excess keyword arguments ' + repr(kwds))
if not args or len(args) % 2:
raise datastore_errors.BadArgumentError(
'A non-zero even number of positional arguments is required '
'(kind, id or name, kind, id or name, ...); received %s' % repr(args))
if _app is not None:
if not isinstance(_app, basestring):
raise datastore_errors.BadArgumentError(
'Expected a string _app; received %r (a %s).' %
(_app, typename(_app)))
if parent is not None:
if not isinstance(parent, Key):
raise datastore_errors.BadArgumentError(
'Expected None or a Key as parent; received %r (a %s).' %
(parent, typename(parent)))
if not parent.has_id_or_name():
raise datastore_errors.BadKeyError(
'The parent Key is incomplete.')
if _app is not None and _app != parent.app():
raise datastore_errors.BadArgumentError(
'The _app argument (%r) should match parent.app() (%s)' %
(_app, parent.app()))
key = Key()
ref = key.__reference
if parent is not None:
ref.CopyFrom(parent.__reference)
elif _app is not None:
ref.set_app(_app)
else:
ref.set_app(_LOCAL_APP_ID)
path = ref.mutable_path()
for i in xrange(0, len(args), 2):
kind, id_or_name = args[i:i+2]
if isinstance(kind, basestring):
kind = kind.encode('utf-8')
else:
raise datastore_errors.BadArgumentError(
'Expected a string kind as argument %d; received %r (a %s).' %
(i + 1, kind, typename(kind)))
elem = path.add_element()
elem.set_type(kind)
if isinstance(id_or_name, (int, long)):
elem.set_id(id_or_name)
elif isinstance(id_or_name, basestring):
ValidateString(id_or_name, 'name')
if id_or_name and id_or_name[0] in string.digits:
raise datastore_errors.BadArgumentError(
'Names may not begin with a digit; received %s.' % id_or_name)
elem.set_name(id_or_name.encode('utf-8'))
else:
raise datastore_errors.BadArgumentError(
'Expected an integer id or string name as argument %d; '
'received %r (a %s).' % (i + 2, id_or_name, typename(id_or_name)))
assert ref.IsInitialized()
return key
