def __get_disk_state(self):
"""Reads the current state of the data on disk.
Returns:
A future whose result is a tuple (<version>, <data tree>).
"""
version_key = ndb.Key(_Version, 1, parent=self.__datasource_key)
values_query = _Value.query(ancestor=self.__datasource_key)
@ndb.tasklet
def txn():
retval = yield version_key.get_async(), values_query.fetch_async()
raise ndb.Return(retval)
version_entity, value_entities = yield ndb.transaction_async(txn)
if version_entity is None:
version = 0
data = {}
else:
version = version_entity.version
data = get_nested_defaultdict()
for valueEntity in value_entities:
if valueEntity.key.id() == '/': path = []
else: path = valueEntity.key.id().split('/')
data_el = data
for path_el in path: data_el = data_el[path_el]
data_el['.'] = json.loads(valueEntity.value)
raise ndb.Return(version, data)
def transaction_async(callback, **ctx_options):
"""Converts all sorts of random exceptions into CommitError.
Arguments:
callback: function to run in the transaction. See
https://cloud.google.com/appengine/docs/python/ndb/functions for more
details.
Sets retries default value to 1 instead 3 (!)
"""
ctx_options.setdefault('retries', 1)
try:
result = yield ndb.transaction_async(callback, **ctx_options)
raise ndb.Return(result)
except (
datastore_errors.InternalError,
datastore_errors.Timeout,
datastore_errors.TransactionFailedError) as e:
# https://cloud.google.com/appengine/docs/python/datastore/transactions
# states the result is ambiguous, it could have succeeded.
logging.info('Transaction likely failed: %s', e)
raise CommitError(e)
except (
apiproxy_errors.CancelledError,
datastore_errors.BadRequestError,
RuntimeError) as e:
logging.info('Transaction failure: %s', e)
raise CommitError(e)
def transaction_async(callback, **ctx_options):
"""Converts all sorts of random exceptions into CommitError.
Arguments:
callback: function to run in the transaction. See
https://cloud.google.com/appengine/docs/python/ndb/functions for more
details.
It is interesting to note that deep down in
google/appengine/ext/ndb/context.py, Context.transaction() looks at the
return value of callback(), and if it is an ndb.Future, will
automatically handle it and return the yielded value.
Sets retries default value to 1 instead 3 (!)
"""
ctx_options.setdefault('retries', 1)
try:
result = yield ndb.transaction_async(callback, **ctx_options)
raise ndb.Return(result)
except (datastore_errors.InternalError, datastore_errors.Timeout,
datastore_errors.TransactionFailedError) as e:
# https://cloud.google.com/appengine/docs/python/datastore/transactions
# states the result is ambiguous, it could have succeeded.
logging.info('Transaction likely failed: %s', e)
raise CommitError(e)
except (apiproxy_errors.CancelledError, datastore_errors.BadRequestError,
RuntimeError) as e:
logging.info('Transaction failure: %s', e.__class__.__name__)
raise CommitError(e)
def transaction_async(callback, **ctx_options):
"""Converts all sorts of random exceptions into CommitError.
Arguments:
callback: function to run in the transaction. See
https://cloud.google.com/appengine/docs/python/ndb/functions for more
details.
Sets retries default value to 1 instead 3 (!)
"""
ctx_options.setdefault('retries', 1)
try:
result = yield ndb.transaction_async(callback, **ctx_options)
raise ndb.Return(result)
except (
datastore_errors.InternalError,
datastore_errors.Timeout,
datastore_errors.TransactionFailedError) as e:
# https://cloud.google.com/appengine/docs/python/datastore/transactions
# states the result is ambiguous, it could have succeeded.
logging.info('Transaction likely failed: %s', e)
raise CommitError(e)
except (
apiproxy_errors.CancelledError,
datastore_errors.BadRequestError,
RuntimeError) as e:
logging.info('Transaction failure: %s', e)
raise CommitError(e)
def _change_async(name,
num_shards,
value,
do_transaction=True,
use_memcache=True):
"""
Asynchronous transaction helper to increment the value for a given sharded counter.
Also takes a number of shards to determine which shard will be used.
Args:
name: The name of the counter.
num_shards: How many shards to use.
"""
logging.info("Shard: change {} by {}".format(name, value))
@ndb.tasklet
def txn():
index = random.randint(0, num_shards - 1)
shard_key_string = SHARD_KEY_TEMPLATE.format(name, index)
counter = yield GeneralCounterShard.get_by_id_async(shard_key_string)
if counter is None:
counter = GeneralCounterShard(id=shard_key_string)
counter.count += value
yield counter.put_async()
if use_memcache:
if value > 0:
memcache.incr(name, delta=value)
elif value < 0:
memcache.decr(name, delta=-value)
else:
memcache.delete(name)
raise ndb.Return(True)
if do_transaction:
return ndb.transaction_async(txn)
else:
return txn()
def _change_async(name, num_shards, value, do_transaction=True, use_memcache=True):
"""
Asynchronous transaction helper to increment the value for a given sharded counter.
Also takes a number of shards to determine which shard will be used.
Args:
name: The name of the counter.
num_shards: How many shards to use.
"""
logging.info("Shard: change {} by {}".format(name, value))
@ndb.tasklet
def txn():
index = random.randint(0, num_shards - 1)
shard_key_string = SHARD_KEY_TEMPLATE.format(name, index)
counter = yield GeneralCounterShard.get_by_id_async(shard_key_string)
if counter is None:
counter = GeneralCounterShard(id=shard_key_string)
counter.count += value
yield counter.put_async()
if use_memcache:
if value > 0:
memcache.incr( name, delta=value )
elif value < 0:
memcache.decr( name, delta=-value )
else:
memcache.delete( name )
raise ndb.Return( True )
if do_transaction:
return ndb.transaction_async( txn )
else:
return txn()
def upsert_async(cls):
@ndb.tasklet
def txn():
logger.info('txn')
key = ndb.Key(cls, 'hoge')
obj = yield key.get_async()
if not obj:
obj = cls(key=key)
yield obj.put_async()
logger.info('call get_parent_async')
parent = yield obj.get_parent_async()
parent.content = 'hogehoge'
logger.info('put parent')
yield parent.put_async()
raise ndb.Return(obj)
res = yield ndb.transaction_async(txn, xg=True)
raise ndb.Return(res)
def upsert_async(cls):
@ndb.tasklet
def txn():
logger.info('txn')
key = ndb.Key(cls, 'hoge')
obj = yield key.get_async()
if not obj:
obj = cls(key=key)
yield obj.put_async()
logger.info('call get_parent_async')
parent = yield obj.get_parent_async()
parent.content = 'hogehoge'
logger.info('put parent')
yield parent.put_async()
raise ndb.Return(obj)
res = yield ndb.transaction_async(txn, xg=True)
raise ndb.Return(res)
def _change_async(name, num_shards, value):
"""
Asynchronous transaction helper to increment the value for a given sharded counter.
Also takes a number of shards to determine which shard will be used.
Args:
name: The name of the counter.
num_shards: How many shards to use.
Returns:
A future whose result is the transaction
"""
@ndb.tasklet
def txn():
if value != 1 and value != -1:
raise Exception("Invalid shard count value %s" % value)
memcache.delete( name )
index = random.randint(0, num_shards - 1)
shard_key_string = SHARD_KEY_TEMPLATE.format(name, index)
counter = yield GeneralCounterShard.get_by_id_async(shard_key_string)
if counter is None:
counter = GeneralCounterShard(id=shard_key_string)
counter.count += value
yield counter.put_async()
memcache.delete( name )
return ndb.transaction_async( txn )
def _LocallyWhitelistForVoter(self, user_key):
"""Creates whitelist rules for a voter."""
host_ids = sorted(list(self._GetHostsToWhitelist(user_key)))
return ndb.transaction_async(
lambda: self._LocallyWhitelist(user_key, host_ids))
def _update(cls, counterName, delta, deadline=3, nbShards=1, sliceId=None):
""" Implements increment() and decrement() """
if delta == 0:
raise ndb.Return(True)
if sliceId is None:
sliceId = datetime.datetime.utcnow().strftime("%Y-%m-%d")
if isinstance(counterName, unicode):
counterName = counterName.encode('utf8')
internalName = cls._PREFIX + counterName
# We use one different memcache key per counter, shard and slice
# but they will all update the same Counter entity.
# Each key has it's own scheduled task to update the Counter entity.
shardId = 1 if nbShards <= 1 else random.randint(1, nbShards)
cacheOpts = {"slice": sliceId, "shard": shardId, "name": internalName}
cacheKey = cls._buildCounterCacheKey(cacheOpts)
ctx = ndb.get_context()
newValueFut = cls._memcacheOffset(cacheKey, delta, deadline)
scheduledFut = ctx.memcache_get(cacheKey + '_scheduled', deadline=deadline)
newValue = yield newValueFut
if newValue is None:
logging.error("Could not update counter %s value in memcache", counterName)
raise ndb.Return(False)
# If memcache was empty, update the underlying entity (unless we are sharding or don't want that feature)
if cls.AVOID_MEMCACHE_AT_LOW_RATES and newValue == delta and nbShards <= 1:
try:
counter = yield ndb.transaction_async(lambda: cls._updateEntity(internalName, delta, sliceId, deadline))
except (apiproxy_errors.DeadlineExceededError, datastore_errors.Timeout, datastore_errors.TransactionFailedError) as e:
# In case of Timeout the Counter is likely being updated and under heavy access
# so do nothing as another request will schedule the write if it's not already scheduled.
logging.warn("Could not persist counter %s in datastore. memcache updated. (%r)", internalName, e)
except datastore_errors.BadRequestError as e:
# In case of BadRequestError, this may be because we execute too many things in //,
# and some transactions are taking too long (the tasklet taking more time to be completed as other operations are on-going).
if 'transaction has expired' in e.message:
logging.warn("Could not persist counter %s in datastore. memcache updated.", internalName, exc_info=1)
else:
raise
else:
# Subtract the value previously added to memcache
yield cls._memcacheOffset(cacheKey, -delta, deadline)
counter._postUpdateHook(delta, sliceId)
# If memcache is not empty this means another update is in progress.
# => Keep the value in memcache and wait for a task to process it.
else:
# schedule the task if needed
isScheduled = yield scheduledFut
if not isScheduled:
# Add an expiration time to plan for next time we should enqueue a task
canSchedule = yield ctx.memcache_add(cacheKey + '_scheduled', True, time=cls._MEMCACHE_LIFE_TIME)
# Do not schedule again if already added(=> memcache.add returns False)
if canSchedule:
# If we use several shards, set countdown at +/- 20% around MEMCACHE_LIFE_TIME
# To avoid having all tasks trying to update the Counter entity at the same time.
countdown = cls._MEMCACHE_LIFE_TIME if nbShards <= 1 else cls._MEMCACHE_LIFE_TIME * (0.8 + 0.4 * random.random())
enqueued = yield _addTask(cls.QUEUES, cls._updateFromMemcache, cacheKey, cacheOpts, _countdown=countdown)
if not enqueued:
yield ctx.memcache_delete(cacheKey + '_scheduled')
else:
logging.debug("Already scheduled: %s", internalName)
# Otherwise, nothing to do
raise ndb.Return(True)
