def increment(cls, platform_id): # platform = cls.get_by_id(platform_id) if platform: value = memcache.get(platform.key.id(), namespace="counters") memcache.decr(platform.key.id(), delta=value, namespace="counters") platform.counter += value platform.put()
def add_or_remove(cls, fingerprint, crash, argv=None, labels=None, is_add=True, delta=1): # use an issue if one already exists issue = CrashReport.most_recent_issue(CrashReport.key_name(fingerprint)) key_name = CrashReport.key_name(fingerprint) config = ShardedCounterConfig.get_sharded_config(key_name) shards = config.shards shard_to_use = random.randint(0, shards-1) shard_key_name = key_name + '_' + str(shard_to_use) if not argv: argv = [] crash_report = CrashReport \ .get_or_insert(shard_key_name, name=key_name, crash=crash, fingerprint=fingerprint, argv=argv, labels=labels, issue=issue) if is_add: crash_report.count += delta crash_report.put() # update caches memcache.incr(CrashReport.count_cache_key(key_name), delta, initial_value=0) else: crash_report.count -= delta crash_report.put() memcache.decr(CrashReport.count_cache_key(key_name), delta) # clear properties cache CrashReport.clear_properties_cache(key_name) return crash_report
def increment(self, incr=1): value = memcache.get(self.key, namespace=self.namespace) if value is None: self.count = incr elif incr > 0: memcache.incr(self.key, incr, namespace=self.namespace) elif incr < 0: memcache.decr(self.key, -incr, namespace=self.namespace)
def UpdateRecord(key, token): item = db.get(key) count = int(memcache.get(token)) item.count += count if count > int(memcache.get(token)): raise db.Rollback() item.put() memcache.decr(token, delta=count)
def increment(self, incr=1): value = memcache.get(self.key) if value is None: self.count = incr elif incr > 0: memcache.incr(self.key, incr) elif incr < 0: memcache.decr(self.key, -incr)
def decr(cls, name, value=1): """Decrements the named counter. Args: name: The name of the counter. value: The value to increment by. """ memcache.decr(name, value, cls._get_kind(), initial_value=0) cls.sched_flush(name)
def decr(self): """Subtract Kudos.""" try: memcache.decr(key = "%s:%s" % (self.set, self.id), initial_value = abs(int(self.stats()))) # Data Store Write Data.incr_kudos(self.set, self.id, score = -1) return True except: return False
def post(self): logging.error('Split Worker') from google.appengine.api import memcache training_set = self.request.get('training_set') dimension = self.request.get('dimension') size = self.request.get('size') _LOG.info('split worker: %s, %s, %s' % (training_set, dimension, size)) from predit.decisiontree import assess_split (best_split, best_value) = assess_split(training_set, dimension, size) memcache.decr('split_counter') memcache.set(str(dimension), (best_split, best_value))
def flushCounter(self, name): # get the counter based on the name, create a new one based on the name if it doesn't exist counter = self.get_or_insert(name) # Get the current value value = memcache.get(name, self.kind()) # Store it to the counter counter.count += value if value else 0 counter.put() # Subtract it from the memcached value if value: memcache.decr(name, value, self.kind())
def releaseWriteLock(self, index): """ Acquires the write lock @param index: an int, the current index """ released = True lockKey = self.lockKey(index) memcache.decr(lockKey) return released
def releaseWriteLock(self, index): """ Acquires the write lock @param index: an int, the current index """ released = True lockKey = self.lockKey(index) memcache.decr(lockKey, namespace=None) return released
def get(self): stathat = StatHat() errors = memcache.get(STATHAT_MEMCACHE_ERROR_COUNT) if errors is not None: stathat.post_count(config.STATHAT_USER_KEY,config.STATHAT_API_ERROR_STAT_KEY,errors,callback=noop) memcache.decr(STATHAT_MEMCACHE_ERROR_COUNT, errors) req_count = memcache.get(STATHAT_MEMCACHE_REQ_COUNT) if req_count is not None: stathat.post_count(config.STATHAT_USER_KEY,config.STATHAT_API_COUNT_STAT_KEY,req_count,callback=noop) memcache.decr(STATHAT_MEMCACHE_REQ_COUNT, req_count)
def decrement(name): config = GeneralCounterShardConfig.get_or_insert(name, name=name) def txn(): index = random.randint(0, config.num_shards - 1) shard_name = name + str(index) counter = GeneralCounterShard.get_by_key_name(shard_name) if counter is None: counter = GeneralCounterShard(key_name=shard_name, name=name) counter.count -= 1 counter.put() db.run_in_transaction(txn) memcache.decr(name)
def incr(name, delta=1, update_interval=10): """Increments a counter. The increment is generally a memcache-only operation, though a task will also be enqueued about once per update_interval. May under-count if memcache contents is lost. Args: name: The name of the counter. delta: Amount to increment counter by, defaulting to 1. update_interval: Approximate interval, in seconds, between updates. Must be greater than zero. """ lock_key = "ctr_lck:" + name delta_key = "ctr_val:" + name # update memcache if delta >= 0: v = memcache.incr(delta_key, delta, initial_value=BASE_VALUE) elif delta < 0: v = memcache.decr(delta_key, -delta, initial_value=BASE_VALUE) if memcache.add(lock_key, None, time=update_interval): # time to enqueue a new task to persist the counter # note: cast to int on next line is due to GAE issue 2012 # (http://code.google.com/p/googleappengine/issues/detail?id=2012) v = int(v) delta_to_persist = v - BASE_VALUE if delta_to_persist == 0: return # nothing to save try: qn = random.randint(0, 4) qname = 'PersistCounter%d' % qn taskqueue.add(url='/task/counter_persist_incr', queue_name=qname, params=dict(name=name, delta=delta_to_persist)) except: # task queue failed but we already put the delta in memcache; # just try to enqueue the task again next interval return # we added the task --> need to decr memcache so we don't double-count failed = False if delta_to_persist > 0: if memcache.decr(delta_key, delta=delta_to_persist) is None: failed = True elif delta_to_persist < 0: if memcache.incr(delta_key, delta=-delta_to_persist) is None: failed = True if failed: logging.warn("counter %s reset failed (will double-count): %d", name, delta_to_persist)
def _sem_lock(self): try: _num=memcache.incr(key=KEY_SEM,initial_value=0,namespace=self.namespace) if _num == 1: #log('_sem_lock: success (namespace=%s)' % (self.namespace)) return True else: memcache.decr(key=KEY_SEM,namespace=self.namespace) log('_sem_lock: failure (number=%s) (namespace=%s)' % (str(_num),self.namespace)) return False except Exception, s: logerr('Error in GAE_Timer()._sem_lock():',s) return False
def calc(): try: hits = memcache.get(key) if hits: article = Article.get_by_id(article_id) if article: article.hits += hits article.put() memcache_client.set_multi_async({'get_article_by_id:%s' % article.key().id(): article, 'get_article_by_url:%s' % hash(article.url): article}, ARTICLE_CACHE_TIME) memcache.decr(key, hits) return True except: return False
def decrement(name): def tnx(): index = random.randint(0, 5) shard_name = name + str(index) counter = Counter.get_by_key_name(shard_name) if counter is None: counter = Counter(key_name=shard_name, name=name) counter.count -= 1 counter.put() db.run_in_transaction(tnx) memcache.decr(name) logging.info("Descrementing %s %s" % (name, memcache.get(name)))
def incr(self, value=1): def update_count(name, incr, error_possible=False): entity = Counter.get_by_key_name(name) if entity: entity.count += incr logging.debug("incr(%s): update_count on retrieved entity by %d to %d", name, incr, entity.count) else: entity = Counter(key_name=name, count=incr) logging.debug("incr(%s): update_count on new entity set to %d", name, incr) if error_possible: entity.error_possible = True entity.put() return entity.count look_ahead_time = 10 + self._update_interval memcache_ops = CapabilitySet('memcache', methods=['add']) memcache_down = not memcache_ops.is_enabled() if memcache_down or memcache.add(self._lock_key, None, time=self._update_interval): # Update the datastore incr = int(memcache.get(self._incr_key) or 0) + value logging.debug("incr(%s): updating datastore with %d", self._name, incr) memcache.set(self._incr_key, 0) try: stored_count = db.run_in_transaction(update_count, self._name, incr) except: memcache.set(self._incr_key, incr) logging.error('Counter(%s): unable to update datastore counter.', self._name) raise memcache.set(self._count_key, stored_count) return stored_count else: incr = memcache.get(self._incr_key) if incr is None: # _incr_key in memcache should be set. If not, two possibilities: # 1) memcache has failed between last datastore update. # 2) this branch has executed before memcache set in update branch (unlikely) stored_count = db.run_in_transaction(update_count, self._name, value, error_possible=True) memcache.set(self._count_key, stored_count) memcache.set(self._incr_key, 0) logging.error('Counter(%s): possible memcache failure in update interval.', self._name) return stored_count else: if value < 0: memcache.decr(self._incr_key, delta=-value) else: memcache.incr(self._incr_key, delta=value) logging.debug("incr(%s): incrementing memcache with %d", self._name, value) return self.count
def calc(key): try: hits = memcache.get(key) if hits: hits = int(hits) if hits: article = Article.get_by_id(int(key.split(':')[1])) if article: article.hits += hits article.put() memcache.decr(key, hits) return True except: return False
def dec(self, namespace, name): """ Decrement a counter. This should not happen often, so double check that it exists, find the first shard that exists and remove one, if the shard drops to 0, we just remove it entirely. """ for scount in self.model.all().filter("namespace = ", namespace).filter("name = ", name): scount.count -= 1 if scount.count == 0: scount.delete() else: scount.put() break memcache.decr("count." + namespace + "." + name)
def decrement(cls,name,num=1): t = num def txn(num): sharecounts = ShardCount.all().filter('name =',name).order('count').fetch(30) for sharecount in sharecounts: if sharecount.count >= num: sharecount.count -= num sharecount.put() return num -= sharecount.count sharecount.count = 0 sharecount.put() txn(num) memcache.decr(name,t)
def decrement(name): shard = Shard.get_or_insert(name) def txn(): index = random.randint(0, shard.num - 1) counter_name = name + str(index) counter = Counter.get_by_key_name(counter_name) if counter is None: counter = Counter(key_name=counter_name, name=name) counter.count -= 1 counter.put() db.run_in_transaction(txn) memcache_name = os.environ["CURRENT_VERSION_ID"] + "." + name memcache.decr(memcache_name)
def reap(): """ Reap the counters from memcache and put them in persistent storage We need the increment/reset instead of get/put in case memcache fails (counter resetted to 0) """ # This might be more efficient by paginating manually and do memcache.get_multi for counter in get_all_counters(): value = memcache.get(counter.key.string_id(), namespace=NAMESPACE) # if value is None: # continue _increment_db(counter, value) memcache.decr(counter.key.string_id(), value, namespace=NAMESPACE) print 'memcache: %s => %s' % (counter.key.string_id(), memcache.get(counter.key.string_id(), namespace=NAMESPACE))
def _close_batch(self): """Release the lock and increment the index to move to next batch.""" # Cuttoff ability to add to the index since it is now processing. # Increment randomly to lower collision of task names when cache evicted. memcache.incr(self.index_name, delta=random.randrange(1, 25)) # The processing has started, stop using index. memcache.decr(self.lock_name, _PIPELINES_SENTINAL_OFFSET_VALUE) # Add a task to cleanup any items that were missed from database delay. eta = (datetime.datetime.utcfromtimestamp(time.time()) + PIPELINES_ETA_BUFFER_CLEANUP) try: deferred.defer(self._process, _name=self.work_index + '-cleanup', _eta=eta, _queue=PIPELINES_QUEUE) except taskqueue.TaskAlreadyExistsError: pass # Expected error to fan-in the tasks.
def decr(self, key, delta=1, version=None): key = self.make_key(key, version=version) val = memcache.decr(key, delta) if val is None: raise ValueError("Key '%s' not found" % key) return val
def flush_views(cls, id): round = cls.get_by_id(id) if not round: round = cls() # Get the current value value = memcache.get('views-' + str(id), cls.kind()) # Subtract it from the memcached value if not value: return memcache.decr('views-' + str(id), int(value), cls.kind()) # Store it to the counter round.views += int(value) round.put()
def _Increment(name, num_shards, delta=1): """Transactional helper to increment the value for a given sharded counter. Also takes a number of shards to determine which shard will be used. Retry count bumped up to work through occasional db latency issues. Args: name: The name of the counter. num_shards: How many shards to use. delta: Non-negative integer to increment key. Returns: The new long integer value or None if a problem arose. """ index = random.randint(0, num_shards - 1) shard_key_string = _SHARD_KEY_TEMPLATE.format(name, index) counter = CounterShardCount.get_by_id(shard_key_string) if counter is None: counter = CounterShardCount(id=shard_key_string) counter.count += delta counter.put() if delta < 0: return memcache.decr(key=name, delta=-delta, initial_value=0) return memcache.incr(key=name, delta=delta, initial_value=0)
def flush_views(cls, id): book_web = cls.get_by_key_name(id) if not book_web: book_web = cls() # Get the current value value = memcache.get('views-' + str(id), cls.kind()) # Subtract it from the memcached value if not value: return memcache.decr('views-' + str(id), int(value), cls.kind()) # Store it to the counter book_web.views += int(value) book_web.put()
def flush_counter(cls, name): """ Flush the value of a counter. :param name: The name of the counter to flush. """ counter = cls.get_by_key_name(name) if not counter: counter = cls(key_name=name) # Get the current value value = memcache.get(name, cls.kind()) # Subtract it from the memcached value memcache.decr(name, int(value), cls.kind()) # Store it to the counter counter.count += int(value) counter.put()
def decrement(self): """Decrement the value of the counter. """ if self.__counter_config is None: self.__counter_config = GeneralCounterShardConfig.get_or_insert(self.__counter_name, name=self.__counter_name) def txn(): index = random.randint(0, self.__counter_config.num_shards - 1) shard_name = self.__counter_name + str(index) counter = GeneralCounterShard.get_by_key_name(shard_name) if counter is None: counter = GeneralCounterShard(key_name=shard_name, name=self.__counter_name) counter.count -= 1 counter.put() db.run_in_transaction(txn) # does nothing if the key does not exist memcache.decr(self.__counter_name)
def increment(cls, name, delta=1, persist_delay=10): """ This function increments the counter in memcache. Args: delta : Amount to be incremented interval : seconds to wait before updating Datastore """ counter_id = cls._get_memcache_id(name) if delta >= 0: val = memcache.incr(counter_id, delta, initial_value=MIDDLE_VALUE) else: print "dec - ", memcache.get(counter_id) - MIDDLE_VALUE, delta val = memcache.decr(counter_id, -delta, initial_value=MIDDLE_VALUE) print "dec2 - ", memcache.get(counter_id) - MIDDLE_VALUE, delta persist_value = val - MIDDLE_VALUE if cls._lock_counter(name, persist_delay): # It's time to persist the value in datastore try: cls._update_datastore(counter_id, persist_value) except datastore_errors.TransactionFailedError: # Just avoid this transaction failure and try again in next iteration pass return persist_value
def incrementCounter(key, delta=1, update_interval=10): """Increments a memcached counter. Args: key: The key of a datastore entity that contains the counter. delta: Non-negative integer value (int or long) to increment key by, defaulting to 1. update_interval: Minimum interval between updates. """ lock_key = "counter_lock:%s" % (key,) count_key = "counter_val:%s" % (key,) if memcache.add(lock_key, None, time=update_interval): # Time to update the DB prev_count = int(memcache.get(count_key) or 0) new_count = prev_count + delta def tx(): entity = CounterModel.get_by_key_name(key) if not entity: entity = CounterModel(key_name=key, counter=0) entity.counter += new_count entity.put() try: db.run_in_transaction(tx) if prev_count>0 and memcache.decr(count_key, delta=prev_count) is None: logging.warn("counter %s could not be decremented (will double-count): %d" % (key, prev_count)) except Exception, e: # db failed to update: we'll try again later; just add delta to memcache like usual for now memcache.incr(count_key, delta, initial_value=0)
def increment(cls, name, delta=1, persist_delay=10): ''' This function increments the counter in memcache. Args: delta : Amount to be incremented interval : seconds to wait before updating Datastore ''' counter_id = cls._get_memcache_id(name) if delta >= 0: val = memcache.incr(counter_id, delta, initial_value=MIDDLE_VALUE) else: print "dec - ", memcache.get(counter_id) - MIDDLE_VALUE, delta val = memcache.decr(counter_id, -delta, initial_value=MIDDLE_VALUE) print "dec2 - ", memcache.get(counter_id) - MIDDLE_VALUE, delta persist_value = val - MIDDLE_VALUE if cls._lock_counter(name, persist_delay): # It's time to persist the value in datastore try: cls._update_datastore(counter_id, persist_value) except datastore_errors.TransactionFailedError: # Just avoid this transaction failure and try again in next iteration pass return persist_value
def flush_views(cls, id): round = cls.get_by_id(id) if not round: round = cls() # Get the current value value = memcache.get("views-" + str(id), cls.kind()) # Subtract it from the memcached value if not value: return memcache.decr("views-" + str(id), int(value), cls.kind()) # Store it to the counter round.views += int(value) round.put()
def decr(key, **kw): result = memcache.decr(key, **kw) if result: action = "hit %s" % result else: action = "miss" _log("cache decr %s %s %s" % (action, key, kw)) return result
def flush_viewcount(cls, name): song = cls.get_by_key_name(name) if not song: song = cls() # Get the current value value = memcache.get('viewcount-' + name, cls.kind()) # Subtract it from the memcached value if not value: return memcache.decr('viewcount-' + name, value, cls.kind()) # Store it to the counter song.viewcount += value song.put()
def _Decr(self, key): """Atomically decrements a key's value. Args: key: Key to decrement. Stored within the local rule context. """ # The logic for _Decr is simpler than _Incr because if a key is not in the # cache during _Decr, then a following call to _Incr will take care of # rebuilding the correct value for that particular key. Rules which call # _Decr to not care about the value of the key after call is complete. Only # subsequent calls to _Incr are of interest. try: if not IsPredictionMode(): memcache.decr(key, namespace=self.namespace) except (TypeError, ValueError), e: # Can happen if key is too long or invalid logging.error(e)
def get(self): stathat = StatHat() errors = memcache.get(STATHAT_MEMCACHE_ERROR_COUNT) if errors is not None: stathat.post_count(config.STATHAT_USER_KEY, config.STATHAT_API_ERROR_STAT_KEY, errors, callback=noop) memcache.decr(STATHAT_MEMCACHE_ERROR_COUNT, errors) req_count = memcache.get(STATHAT_MEMCACHE_REQ_COUNT) if req_count is not None: stathat.post_count(config.STATHAT_USER_KEY, config.STATHAT_API_COUNT_STAT_KEY, req_count, callback=noop) memcache.decr(STATHAT_MEMCACHE_REQ_COUNT, req_count)
def _decrement(name, num_shards): """Transactional helper to decrement the value for a given sharded counter. Also takes a number of shards to determine which shard will be used. Args: name: The name of the counter. num_shards: How many shards to use. """ index = random.randint(0, num_shards - 1) shard_key_string = SHARD_KEY_TEMPLATE.format(name, index) counter = TailboneGeneralCounterShard.get_by_id(shard_key_string) if counter is None: counter = TailboneGeneralCounterShard(id=shard_key_string) counter.count -= 1 counter.put() # Memcache increment does nothing if the name is not a key in memcache memcache.decr(name)
def flush_counter(cls, name, ts): """ Flushes the value of the counter with the specified name and timestamp to the datastore. :param name: The name of the counter to flush. :param ts: The timestamp to get the counter value for. """ ts_name = cls.get_ts_name(name, ts) counter = cls.get_by_key_name(ts_name) if not counter: counter = cls(key_name=ts_name, timestamp=cls.normalize_ts(ts)) # Get the current value value = memcache.get(ts_name, cls.kind()) # Subtract it from the memcached value memcache.decr(ts_name, int(value), cls.kind()) # Store it to the counter counter.count += int(value) counter.put()
def decrement(self): """Decrement the value of the counter. """ if self.__counter_config is None: self.__counter_config = GeneralCounterShardConfig.get_or_insert( self.__counter_name, name=self.__counter_name) def txn(): index = random.randint(0, self.__counter_config.num_shards - 1) shard_name = self.__counter_name + str(index) counter = GeneralCounterShard.get_by_key_name(shard_name) if counter is None: counter = GeneralCounterShard(key_name=shard_name, name=self.__counter_name) counter.count -= 1 counter.put() db.run_in_transaction(txn) # does nothing if the key does not exist memcache.decr(self.__counter_name)
def acquireReadLock(self, index, nextEvent=None, raiseOnFail=False): """ Acquires the read lock @param index: an int, the current index """ acquired = True lockKey = self.lockKey(index) indexKey = self.indexKey() # tell writers to use another index memcache.incr(indexKey) # tell writers they missed the boat memcache.decr(lockKey, 2**15) # busy wait for writers for i in xrange(ReadWriteLock.BUSY_WAIT_ITERS): counter = memcache.get(lockKey) # counter is None --> ejected from memcache, or no writers # int(counter) <= 2**15 --> writers have all called memcache.decr if counter is None or int(counter) <= 2**15: break time.sleep(ReadWriteLock.BUSY_WAIT_ITER_SECS) self.context.logger.debug( "Tried to acquire read lock '%s' %d times...", lockKey, i + 1) # FIXME: is there anything else that can be done? will work packages be lost? maybe queue another task # to sweep up later? if i >= (ReadWriteLock.BUSY_WAIT_ITERS - 1): # pylint: disable-msg=W0631 self.context.logger.critical( "Gave up waiting for all fan-in work items with read lock '%s'.", lockKey) acquired = False if raiseOnFail: raise FanInReadLockFailureRuntimeError( nextEvent, self.context.machineName, self.context.currentState.name, self.context.instanceName) return acquired
def increment(name, by=1): """Increment the value for a given sharded counter. Parameters: name - The name of the counter """ config = GeneralCounterShardConfig.get_or_insert(name, name=name) def txn(): index = random.randint(0, config.num_shards - 1) shard_name = name + str(index) counter = GeneralCounterShard.get_by_key_name(shard_name) if counter is None: counter = GeneralCounterShard(key_name=shard_name, name=name) counter.count += by counter.put() db.run_in_transaction(txn) # incr and decr do nothing if the key does not exist if by > 0: memcache.incr(name, delta=by) elif by < 0: memcache.decr(name, delta=-by)
def testIncrementDecrement(self): """Testing automatically incrementing and decrementing.""" memcache.incr('unknown_key') assert memcache.get('unknown_key') == None memcache.set('counter', 0) assert memcache.get('counter') == 0 memcache.incr('counter') assert memcache.get('counter') == 1 memcache.incr('counter', delta=2) assert memcache.get('counter') == 3 memcache.decr('counter') assert memcache.get('counter') == 2 memcache.decr('counter', 2) assert memcache.get('counter') == 0 memcache.incr('second_counter', initial_value=10) assert memcache.get('second_counter') == 11 memcache.decr('third_counter', initial_value=10) assert memcache.get('third_counter') == 9 # This should cause an error message, because zero deltas are not # allowed. memcache.incr('counter', delta=0) memcache.set('lcounter', long(20)) assert memcache.get('lcounter') == long(20) memcache.incr('lcounter') assert memcache.get('lcounter') == long(21)
def _increment_index(self, last_index): """Moves the work index forward and waits for all writers. Args: last_index: The last index that was used for the reader/writer lock. Returns: True if all writers were definitely finished; False if the reader/writer lock timed out and we are proceeding anyways. """ # Increment the batch index counter so incoming jobs will use a new index. # Don't bother setting an initial value here because next_index() will # do this when it notices no current index is present. Do this *before* # closing the reader/writer lock below to decrease active writers on the # current index. # We do this even in the case that batch_period_ms was zero, just in case # that memcache operation failed for some reason, we'd rather have more # batches then have the work index pipeline stall. memcache.incr(self.index_name) # Prevent new writers by making the counter extremely negative. If the # decrement fails here we can't recover anyways, so just let the worker go. add_counter = self.add_counter_template % last_index memcache.decr(add_counter, self.LOCK_OFFSET) for i in xrange(self.sync_attempts): counter = memcache.get(add_counter) # Less than or equal LOCK_OFFSET here in case a writer decrements twice # due to rerunning failure tasks. if counter is None or int(counter) <= self.LOCK_OFFSET: # Worst-case the counter will be gone due to memcache eviction, which # means the worker can procede with without waiting for writers # and just process whatever it can find. This may drop some work. return True time.sleep(self.sync_timeout) else: logging.critical('Worker for %s gave up waiting for writers', self.name) return False
def txn(): index = random.randint(0, num_shards - 1) shard_key_string = SHARD_KEY_TEMPLATE.format(name, index) counter = yield GeneralCounterShard.get_by_id_async(shard_key_string) if counter is None: counter = GeneralCounterShard(id=shard_key_string) counter.count += value yield counter.put_async() if use_memcache: if value > 0: memcache.incr( name, delta=value ) elif value < 0: memcache.decr( name, delta=-value ) else: memcache.delete( name ) raise ndb.Return( True )
def add_or_subtract(name, value=0): """Decrement the value for a given sharded counter. Parameters: name - The name of the counter """ config = GeneralCounterShardConfig.get_or_insert(name, name=name) def txn(): index = random.randint(0, config.num_shards - 1) shard_name = "%s-%s" % (name, index) counter = GeneralCounterShard.get_by_key_name(shard_name) if counter is None: counter = GeneralCounterShard(key_name=shard_name, name=name) counter.count += value counter.put() db.run_in_transaction(txn) if value > 0: memcache.incr(name, value) elif value < 0: memcache.decr(name, -value)
def _open_batch(self): """Determine which batch to use for the pipeline.""" self.index_name = '{}-index'.format(self.batch_name) while True: # Find the latest batch index. index = memcache.get(self.index_name) if index is None: # Random to lower collision of task names when cache evicted. memcache.add(self.index_name, random.randrange(10, 10000)) index = memcache.get(self.index_name) # Create a non-monotonically increasing name for the work index. self.md5_hash = hashlib.md5(str(index)).hexdigest() self.work_index = '{}-{}'.format(self.batch_name, self.md5_hash) self.lock_name = '{}-lock'.format(self.work_index) # Determine if the batch is still valid and available. writers = memcache.incr(self.lock_name, initial_value=_PIPELINES_SENTINAL_VALUE) if writers < _PIPELINES_SENTINAL_VALUE: memcache.decr(self.lock_name) continue return