def _get_tasklet(self, todo):
assert todo
# First check memcache.
keys = set(key for _, key in todo)
memkeymap = dict((key, key.urlsafe())
for key in keys if self.should_memcache(key))
if memkeymap:
results = memcache.get_multi(memkeymap.values())
leftover = []
## del todo[1:] # Uncommenting this creates an interesting bug.
for fut, key in todo:
mkey = memkeymap[key]
if mkey in results:
pb = results[mkey]
ent = self._conn.adapter.pb_to_entity(pb)
fut.set_result(ent)
else:
leftover.append((fut, key))
todo = leftover
if todo:
keys = [key for (_, key) in todo]
# TODO: What if async_get() created a non-trivial MultiRpc?
results = yield self._conn.async_get(None, keys)
for ent, (fut, _) in zip(results, todo):
fut.set_result(ent)
def _delete_tasklet(self, todo):
assert todo
by_options = {}
delete_keys = [] # For memcache.delete_multi()
for fut, key, options in todo:
if self._use_memcache(key, options):
delete_keys.append(key.urlsafe())
if options in by_options:
futures, keys = by_options[options]
else:
futures, keys = by_options[options] = [], []
futures.append(fut)
keys.append(key)
if delete_keys: # Pre-emptively delete from memcache.
memcache.delete_multi(delete_keys, seconds=_LOCK_TIME,
key_prefix=self._memcache_prefix)
for options, (futures, keys) in by_options.iteritems():
datastore_keys = []
for key in keys:
if self._use_datastore(key, options):
datastore_keys.append(key)
if datastore_keys:
yield self._conn.async_delete(options, datastore_keys)
for fut in futures:
fut.set_result(None)
def _get_tasklet(self, todo):
assert todo
# First check memcache.
memkeymap = {}
for fut, key, options in todo:
if self._use_memcache(key, options):
memkeymap[key] = key.urlsafe()
if memkeymap:
results = memcache.get_multi(memkeymap.values(),
key_prefix=self._memcache_prefix)
leftover = []
for fut, key, options in todo:
mkey = memkeymap.get(key)
if mkey is not None and mkey in results:
pb = results[mkey]
ent = self._conn.adapter.pb_to_entity(pb)
fut.set_result(ent)
else:
leftover.append((fut, key, options))
todo = leftover
# Segregate things by ConfigOptions.
by_options = {}
for fut, key, options in todo:
if options in by_options:
futures, keys = by_options[options]
else:
futures, keys = by_options[options] = [], []
futures.append(fut)
keys.append(key)
# Make the RPC calls.
mappings = {} # Maps timeout value to {urlsafe_key: pb} mapping.
for options, (futures, keys) in by_options.iteritems():
datastore_futures = []
datastore_keys = []
for fut, key in zip(futures, keys):
if self._use_datastore(key, options):
datastore_keys.append(key)
datastore_futures.append(fut)
else:
fut.set_result(None)
if datastore_keys:
entities = yield self._conn.async_get(options, datastore_keys)
for ent, fut, key in zip(entities, datastore_futures, datastore_keys):
fut.set_result(ent)
if ent is not None and self._use_memcache(key, options):
pb = self._conn.adapter.entity_to_pb(ent)
timeout = self._get_memcache_timeout(key, options)
mapping = mappings.get(timeout)
if mapping is None:
mapping = mappings[timeout] = {}
mapping[ent._key.urlsafe()] = pb
if mappings:
# If the timeouts are not uniform, make a separate call for each
# distinct timeout value.
for timeout, mapping in mappings.iteritems():
# Use add, not set. This is a no-op within _LOCK_TIME seconds
# of the delete done by the most recent write.
memcache.add_multi(mapping, time=timeout,
key_prefix=self._memcache_prefix)
def _delete_tasklet(self, todo):
assert todo
keys = set(key for (_, key) in todo)
yield self._conn.async_delete(None, keys)
for fut, _ in todo:
fut.set_result(None)
# Now update memcache.
memkeys = [key.urlsafe() for key in keys if self.should_memcache(key)]
if memkeys:
memcache.delete_multi(memkeys)
def _flush_memcache(self, keys):
keys = set(key for key in keys if self.should_memcache(key))
if keys:
memkeys = [key.urlsafe() for key in keys]
memcache.delete_multi(memkeys)
def _clear_memcache(self, keys):
keys = set(key for key in keys if self._use_memcache(key))
if keys:
memkeys = [key.urlsafe() for key in keys]
memcache.delete_multi(memkeys, key_prefix=self._memcache_prefix)