def __get_disk_state(self): """Reads the current state of the data on disk. Returns: A future whose result is a tuple (<version>, <data tree>). """ version_key = ndb.Key(_Version, 1, parent=self.__datasource_key) values_query = _Value.query(ancestor=self.__datasource_key) @ndb.tasklet def txn(): retval = yield version_key.get_async(), values_query.fetch_async() raise ndb.Return(retval) version_entity, value_entities = yield ndb.transaction_async(txn) if version_entity is None: version = 0 data = {} else: version = version_entity.version data = get_nested_defaultdict() for valueEntity in value_entities: if valueEntity.key.id() == '/': path = [] else: path = valueEntity.key.id().split('/') data_el = data for path_el in path: data_el = data_el[path_el] data_el['.'] = json.loads(valueEntity.value) raise ndb.Return(version, data)
def transaction_async(callback, **ctx_options): """Converts all sorts of random exceptions into CommitError. Arguments: callback: function to run in the transaction. See https://cloud.google.com/appengine/docs/python/ndb/functions for more details. Sets retries default value to 1 instead 3 (!) """ ctx_options.setdefault('retries', 1) try: result = yield ndb.transaction_async(callback, **ctx_options) raise ndb.Return(result) except ( datastore_errors.InternalError, datastore_errors.Timeout, datastore_errors.TransactionFailedError) as e: # https://cloud.google.com/appengine/docs/python/datastore/transactions # states the result is ambiguous, it could have succeeded. logging.info('Transaction likely failed: %s', e) raise CommitError(e) except ( apiproxy_errors.CancelledError, datastore_errors.BadRequestError, RuntimeError) as e: logging.info('Transaction failure: %s', e) raise CommitError(e)
def transaction_async(callback, **ctx_options): """Converts all sorts of random exceptions into CommitError. Arguments: callback: function to run in the transaction. See https://cloud.google.com/appengine/docs/python/ndb/functions for more details. It is interesting to note that deep down in google/appengine/ext/ndb/context.py, Context.transaction() looks at the return value of callback(), and if it is an ndb.Future, will automatically handle it and return the yielded value. Sets retries default value to 1 instead 3 (!) """ ctx_options.setdefault('retries', 1) try: result = yield ndb.transaction_async(callback, **ctx_options) raise ndb.Return(result) except (datastore_errors.InternalError, datastore_errors.Timeout, datastore_errors.TransactionFailedError) as e: # https://cloud.google.com/appengine/docs/python/datastore/transactions # states the result is ambiguous, it could have succeeded. logging.info('Transaction likely failed: %s', e) raise CommitError(e) except (apiproxy_errors.CancelledError, datastore_errors.BadRequestError, RuntimeError) as e: logging.info('Transaction failure: %s', e.__class__.__name__) raise CommitError(e)
def _change_async(name, num_shards, value, do_transaction=True, use_memcache=True): """ Asynchronous transaction helper to increment the value for a given sharded counter. Also takes a number of shards to determine which shard will be used. Args: name: The name of the counter. num_shards: How many shards to use. """ logging.info("Shard: change {} by {}".format(name, value)) @ndb.tasklet def txn(): index = random.randint(0, num_shards - 1) shard_key_string = SHARD_KEY_TEMPLATE.format(name, index) counter = yield GeneralCounterShard.get_by_id_async(shard_key_string) if counter is None: counter = GeneralCounterShard(id=shard_key_string) counter.count += value yield counter.put_async() if use_memcache: if value > 0: memcache.incr(name, delta=value) elif value < 0: memcache.decr(name, delta=-value) else: memcache.delete(name) raise ndb.Return(True) if do_transaction: return ndb.transaction_async(txn) else: return txn()
def _change_async(name, num_shards, value, do_transaction=True, use_memcache=True): """ Asynchronous transaction helper to increment the value for a given sharded counter. Also takes a number of shards to determine which shard will be used. Args: name: The name of the counter. num_shards: How many shards to use. """ logging.info("Shard: change {} by {}".format(name, value)) @ndb.tasklet def txn(): index = random.randint(0, num_shards - 1) shard_key_string = SHARD_KEY_TEMPLATE.format(name, index) counter = yield GeneralCounterShard.get_by_id_async(shard_key_string) if counter is None: counter = GeneralCounterShard(id=shard_key_string) counter.count += value yield counter.put_async() if use_memcache: if value > 0: memcache.incr( name, delta=value ) elif value < 0: memcache.decr( name, delta=-value ) else: memcache.delete( name ) raise ndb.Return( True ) if do_transaction: return ndb.transaction_async( txn ) else: return txn()
def upsert_async(cls): @ndb.tasklet def txn(): logger.info('txn') key = ndb.Key(cls, 'hoge') obj = yield key.get_async() if not obj: obj = cls(key=key) yield obj.put_async() logger.info('call get_parent_async') parent = yield obj.get_parent_async() parent.content = 'hogehoge' logger.info('put parent') yield parent.put_async() raise ndb.Return(obj) res = yield ndb.transaction_async(txn, xg=True) raise ndb.Return(res)
def _change_async(name, num_shards, value): """ Asynchronous transaction helper to increment the value for a given sharded counter. Also takes a number of shards to determine which shard will be used. Args: name: The name of the counter. num_shards: How many shards to use. Returns: A future whose result is the transaction """ @ndb.tasklet def txn(): if value != 1 and value != -1: raise Exception("Invalid shard count value %s" % value) memcache.delete( name ) index = random.randint(0, num_shards - 1) shard_key_string = SHARD_KEY_TEMPLATE.format(name, index) counter = yield GeneralCounterShard.get_by_id_async(shard_key_string) if counter is None: counter = GeneralCounterShard(id=shard_key_string) counter.count += value yield counter.put_async() memcache.delete( name ) return ndb.transaction_async( txn )
def _LocallyWhitelistForVoter(self, user_key): """Creates whitelist rules for a voter.""" host_ids = sorted(list(self._GetHostsToWhitelist(user_key))) return ndb.transaction_async( lambda: self._LocallyWhitelist(user_key, host_ids))
def _update(cls, counterName, delta, deadline=3, nbShards=1, sliceId=None): """ Implements increment() and decrement() """ if delta == 0: raise ndb.Return(True) if sliceId is None: sliceId = datetime.datetime.utcnow().strftime("%Y-%m-%d") if isinstance(counterName, unicode): counterName = counterName.encode('utf8') internalName = cls._PREFIX + counterName # We use one different memcache key per counter, shard and slice # but they will all update the same Counter entity. # Each key has it's own scheduled task to update the Counter entity. shardId = 1 if nbShards <= 1 else random.randint(1, nbShards) cacheOpts = {"slice": sliceId, "shard": shardId, "name": internalName} cacheKey = cls._buildCounterCacheKey(cacheOpts) ctx = ndb.get_context() newValueFut = cls._memcacheOffset(cacheKey, delta, deadline) scheduledFut = ctx.memcache_get(cacheKey + '_scheduled', deadline=deadline) newValue = yield newValueFut if newValue is None: logging.error("Could not update counter %s value in memcache", counterName) raise ndb.Return(False) # If memcache was empty, update the underlying entity (unless we are sharding or don't want that feature) if cls.AVOID_MEMCACHE_AT_LOW_RATES and newValue == delta and nbShards <= 1: try: counter = yield ndb.transaction_async(lambda: cls._updateEntity(internalName, delta, sliceId, deadline)) except (apiproxy_errors.DeadlineExceededError, datastore_errors.Timeout, datastore_errors.TransactionFailedError) as e: # In case of Timeout the Counter is likely being updated and under heavy access # so do nothing as another request will schedule the write if it's not already scheduled. logging.warn("Could not persist counter %s in datastore. memcache updated. (%r)", internalName, e) except datastore_errors.BadRequestError as e: # In case of BadRequestError, this may be because we execute too many things in //, # and some transactions are taking too long (the tasklet taking more time to be completed as other operations are on-going). if 'transaction has expired' in e.message: logging.warn("Could not persist counter %s in datastore. memcache updated.", internalName, exc_info=1) else: raise else: # Subtract the value previously added to memcache yield cls._memcacheOffset(cacheKey, -delta, deadline) counter._postUpdateHook(delta, sliceId) # If memcache is not empty this means another update is in progress. # => Keep the value in memcache and wait for a task to process it. else: # schedule the task if needed isScheduled = yield scheduledFut if not isScheduled: # Add an expiration time to plan for next time we should enqueue a task canSchedule = yield ctx.memcache_add(cacheKey + '_scheduled', True, time=cls._MEMCACHE_LIFE_TIME) # Do not schedule again if already added(=> memcache.add returns False) if canSchedule: # If we use several shards, set countdown at +/- 20% around MEMCACHE_LIFE_TIME # To avoid having all tasks trying to update the Counter entity at the same time. countdown = cls._MEMCACHE_LIFE_TIME if nbShards <= 1 else cls._MEMCACHE_LIFE_TIME * (0.8 + 0.4 * random.random()) enqueued = yield _addTask(cls.QUEUES, cls._updateFromMemcache, cacheKey, cacheOpts, _countdown=countdown) if not enqueued: yield ctx.memcache_delete(cacheKey + '_scheduled') else: logging.debug("Already scheduled: %s", internalName) # Otherwise, nothing to do raise ndb.Return(True)