def increment(cls, platform_id): #
platform = cls.get_by_id(platform_id)
if platform:
value = memcache.get(platform.key.id(), namespace="counters")
memcache.decr(platform.key.id(), delta=value, namespace="counters")
platform.counter += value
platform.put()
def add_or_remove(cls, fingerprint, crash, argv=None, labels=None, is_add=True, delta=1):
# use an issue if one already exists
issue = CrashReport.most_recent_issue(CrashReport.key_name(fingerprint))
key_name = CrashReport.key_name(fingerprint)
config = ShardedCounterConfig.get_sharded_config(key_name)
shards = config.shards
shard_to_use = random.randint(0, shards-1)
shard_key_name = key_name + '_' + str(shard_to_use)
if not argv:
argv = []
crash_report = CrashReport \
.get_or_insert(shard_key_name,
name=key_name,
crash=crash,
fingerprint=fingerprint,
argv=argv,
labels=labels,
issue=issue)
if is_add:
crash_report.count += delta
crash_report.put()
# update caches
memcache.incr(CrashReport.count_cache_key(key_name), delta, initial_value=0)
else:
crash_report.count -= delta
crash_report.put()
memcache.decr(CrashReport.count_cache_key(key_name), delta)
# clear properties cache
CrashReport.clear_properties_cache(key_name)
return crash_report
def increment(self, incr=1):
value = memcache.get(self.key, namespace=self.namespace)
if value is None:
self.count = incr
elif incr > 0:
memcache.incr(self.key, incr, namespace=self.namespace)
elif incr < 0:
memcache.decr(self.key, -incr, namespace=self.namespace)
def UpdateRecord(key, token):
item = db.get(key)
count = int(memcache.get(token))
item.count += count
if count > int(memcache.get(token)):
raise db.Rollback()
item.put()
memcache.decr(token, delta=count)
def UpdateRecord(key, token):
item = db.get(key)
count = int(memcache.get(token))
item.count += count
if count > int(memcache.get(token)):
raise db.Rollback()
item.put()
memcache.decr(token, delta=count)
def increment(self, incr=1): value = memcache.get(self.key, namespace=self.namespace) if value is None: self.count = incr elif incr > 0: memcache.incr(self.key, incr, namespace=self.namespace) elif incr < 0: memcache.decr(self.key, -incr, namespace=self.namespace)
def increment(self, incr=1): value = memcache.get(self.key) if value is None: self.count = incr elif incr > 0: memcache.incr(self.key, incr) elif incr < 0: memcache.decr(self.key, -incr)
def decr(cls, name, value=1):
"""Decrements the named counter.
Args:
name: The name of the counter.
value: The value to increment by.
"""
memcache.decr(name, value, cls._get_kind(), initial_value=0)
cls.sched_flush(name)
def decr(self):
"""Subtract Kudos."""
try:
memcache.decr(key = "%s:%s" % (self.set, self.id), initial_value = abs(int(self.stats())))
# Data Store Write
Data.incr_kudos(self.set, self.id, score = -1)
return True
except:
return False
def post(self):
logging.error('Split Worker')
from google.appengine.api import memcache
training_set = self.request.get('training_set')
dimension = self.request.get('dimension')
size = self.request.get('size')
_LOG.info('split worker: %s, %s, %s' % (training_set, dimension, size))
from predit.decisiontree import assess_split
(best_split, best_value) = assess_split(training_set, dimension, size)
memcache.decr('split_counter')
memcache.set(str(dimension), (best_split, best_value))
def flushCounter(self, name):
# get the counter based on the name, create a new one based on the name if it doesn't exist
counter = self.get_or_insert(name)
# Get the current value
value = memcache.get(name, self.kind())
# Store it to the counter
counter.count += value if value else 0
counter.put()
# Subtract it from the memcached value
if value:
memcache.decr(name, value, self.kind())
def releaseWriteLock(self, index):
""" Acquires the write lock
@param index: an int, the current index
"""
released = True
lockKey = self.lockKey(index)
memcache.decr(lockKey)
return released
def releaseWriteLock(self, index):
""" Acquires the write lock
@param index: an int, the current index
"""
released = True
lockKey = self.lockKey(index)
memcache.decr(lockKey, namespace=None)
return released
def get(self):
stathat = StatHat()
errors = memcache.get(STATHAT_MEMCACHE_ERROR_COUNT)
if errors is not None:
stathat.post_count(config.STATHAT_USER_KEY,config.STATHAT_API_ERROR_STAT_KEY,errors,callback=noop)
memcache.decr(STATHAT_MEMCACHE_ERROR_COUNT, errors)
req_count = memcache.get(STATHAT_MEMCACHE_REQ_COUNT)
if req_count is not None:
stathat.post_count(config.STATHAT_USER_KEY,config.STATHAT_API_COUNT_STAT_KEY,req_count,callback=noop)
memcache.decr(STATHAT_MEMCACHE_REQ_COUNT, req_count)
def decrement(name):
config = GeneralCounterShardConfig.get_or_insert(name, name=name)
def txn():
index = random.randint(0, config.num_shards - 1)
shard_name = name + str(index)
counter = GeneralCounterShard.get_by_key_name(shard_name)
if counter is None:
counter = GeneralCounterShard(key_name=shard_name, name=name)
counter.count -= 1
counter.put()
db.run_in_transaction(txn)
memcache.decr(name)
def incr(name, delta=1, update_interval=10):
"""Increments a counter. The increment is generally a memcache-only
operation, though a task will also be enqueued about once per
update_interval. May under-count if memcache contents is lost.
Args:
name: The name of the counter.
delta: Amount to increment counter by, defaulting to 1.
update_interval: Approximate interval, in seconds, between updates. Must
be greater than zero.
"""
lock_key = "ctr_lck:" + name
delta_key = "ctr_val:" + name
# update memcache
if delta >= 0:
v = memcache.incr(delta_key, delta, initial_value=BASE_VALUE)
elif delta < 0:
v = memcache.decr(delta_key, -delta, initial_value=BASE_VALUE)
if memcache.add(lock_key, None, time=update_interval):
# time to enqueue a new task to persist the counter
# note: cast to int on next line is due to GAE issue 2012
# (http://code.google.com/p/googleappengine/issues/detail?id=2012)
v = int(v)
delta_to_persist = v - BASE_VALUE
if delta_to_persist == 0:
return # nothing to save
try:
qn = random.randint(0, 4)
qname = 'PersistCounter%d' % qn
taskqueue.add(url='/task/counter_persist_incr',
queue_name=qname,
params=dict(name=name,
delta=delta_to_persist))
except:
# task queue failed but we already put the delta in memcache;
# just try to enqueue the task again next interval
return
# we added the task --> need to decr memcache so we don't double-count
failed = False
if delta_to_persist > 0:
if memcache.decr(delta_key, delta=delta_to_persist) is None:
failed = True
elif delta_to_persist < 0:
if memcache.incr(delta_key, delta=-delta_to_persist) is None:
failed = True
if failed:
logging.warn("counter %s reset failed (will double-count): %d",
name, delta_to_persist)
def _sem_lock(self):
try:
_num=memcache.incr(key=KEY_SEM,initial_value=0,namespace=self.namespace)
if _num == 1:
#log('_sem_lock: success (namespace=%s)' % (self.namespace))
return True
else:
memcache.decr(key=KEY_SEM,namespace=self.namespace)
log('_sem_lock: failure (number=%s) (namespace=%s)' % (str(_num),self.namespace))
return False
except Exception, s:
logerr('Error in GAE_Timer()._sem_lock():',s)
return False
def calc():
try:
hits = memcache.get(key)
if hits:
article = Article.get_by_id(article_id)
if article:
article.hits += hits
article.put()
memcache_client.set_multi_async({'get_article_by_id:%s' % article.key().id(): article, 'get_article_by_url:%s' % hash(article.url): article}, ARTICLE_CACHE_TIME)
memcache.decr(key, hits)
return True
except:
return False
def decrement(name):
def tnx():
index = random.randint(0, 5)
shard_name = name + str(index)
counter = Counter.get_by_key_name(shard_name)
if counter is None:
counter = Counter(key_name=shard_name, name=name)
counter.count -= 1
counter.put()
db.run_in_transaction(tnx)
memcache.decr(name)
logging.info("Descrementing %s %s" % (name, memcache.get(name)))
def incr(self, value=1):
def update_count(name, incr, error_possible=False):
entity = Counter.get_by_key_name(name)
if entity:
entity.count += incr
logging.debug("incr(%s): update_count on retrieved entity by %d to %d", name, incr, entity.count)
else:
entity = Counter(key_name=name, count=incr)
logging.debug("incr(%s): update_count on new entity set to %d", name, incr)
if error_possible:
entity.error_possible = True
entity.put()
return entity.count
look_ahead_time = 10 + self._update_interval
memcache_ops = CapabilitySet('memcache', methods=['add'])
memcache_down = not memcache_ops.is_enabled()
if memcache_down or memcache.add(self._lock_key, None, time=self._update_interval):
# Update the datastore
incr = int(memcache.get(self._incr_key) or 0) + value
logging.debug("incr(%s): updating datastore with %d", self._name, incr)
memcache.set(self._incr_key, 0)
try:
stored_count = db.run_in_transaction(update_count, self._name, incr)
except:
memcache.set(self._incr_key, incr)
logging.error('Counter(%s): unable to update datastore counter.', self._name)
raise
memcache.set(self._count_key, stored_count)
return stored_count
else:
incr = memcache.get(self._incr_key)
if incr is None:
# _incr_key in memcache should be set. If not, two possibilities:
# 1) memcache has failed between last datastore update.
# 2) this branch has executed before memcache set in update branch (unlikely)
stored_count = db.run_in_transaction(update_count,
self._name, value, error_possible=True)
memcache.set(self._count_key, stored_count)
memcache.set(self._incr_key, 0)
logging.error('Counter(%s): possible memcache failure in update interval.',
self._name)
return stored_count
else:
if value < 0:
memcache.decr(self._incr_key, delta=-value)
else:
memcache.incr(self._incr_key, delta=value)
logging.debug("incr(%s): incrementing memcache with %d", self._name, value)
return self.count
def incr(name, delta=1, update_interval=10):
"""Increments a counter. The increment is generally a memcache-only
operation, though a task will also be enqueued about once per
update_interval. May under-count if memcache contents is lost.
Args:
name: The name of the counter.
delta: Amount to increment counter by, defaulting to 1.
update_interval: Approximate interval, in seconds, between updates. Must
be greater than zero.
"""
lock_key = "ctr_lck:" + name
delta_key = "ctr_val:" + name
# update memcache
if delta >= 0:
v = memcache.incr(delta_key, delta, initial_value=BASE_VALUE)
elif delta < 0:
v = memcache.decr(delta_key, -delta, initial_value=BASE_VALUE)
if memcache.add(lock_key, None, time=update_interval):
# time to enqueue a new task to persist the counter
# note: cast to int on next line is due to GAE issue 2012
# (http://code.google.com/p/googleappengine/issues/detail?id=2012)
v = int(v)
delta_to_persist = v - BASE_VALUE
if delta_to_persist == 0:
return # nothing to save
try:
qn = random.randint(0, 4)
qname = 'PersistCounter%d' % qn
taskqueue.add(url='/task/counter_persist_incr',
queue_name=qname,
params=dict(name=name, delta=delta_to_persist))
except:
# task queue failed but we already put the delta in memcache;
# just try to enqueue the task again next interval
return
# we added the task --> need to decr memcache so we don't double-count
failed = False
if delta_to_persist > 0:
if memcache.decr(delta_key, delta=delta_to_persist) is None:
failed = True
elif delta_to_persist < 0:
if memcache.incr(delta_key, delta=-delta_to_persist) is None:
failed = True
if failed:
logging.warn("counter %s reset failed (will double-count): %d",
name, delta_to_persist)
def calc(key):
try:
hits = memcache.get(key)
if hits:
hits = int(hits)
if hits:
article = Article.get_by_id(int(key.split(':')[1]))
if article:
article.hits += hits
article.put()
memcache.decr(key, hits)
return True
except:
return False
def calc(key):
try:
hits = memcache.get(key)
if hits:
hits = int(hits)
if hits:
article = Article.get_by_id(int(key.split(':')[1]))
if article:
article.hits += hits
article.put()
memcache.decr(key, hits)
return True
except:
return False
def dec(self, namespace, name):
"""
Decrement a counter. This should not happen often, so double check that it exists, find the
first shard that exists and remove one, if the shard drops to 0, we just remove it entirely.
"""
for scount in self.model.all().filter("namespace = ", namespace).filter("name = ", name):
scount.count -= 1
if scount.count == 0:
scount.delete()
else:
scount.put()
break
memcache.decr("count." + namespace + "." + name)
def decrement(cls,name,num=1):
t = num
def txn(num):
sharecounts = ShardCount.all().filter('name =',name).order('count').fetch(30)
for sharecount in sharecounts:
if sharecount.count >= num:
sharecount.count -= num
sharecount.put()
return
num -= sharecount.count
sharecount.count = 0
sharecount.put()
txn(num)
memcache.decr(name,t)
def decrement(name): shard = Shard.get_or_insert(name) def txn(): index = random.randint(0, shard.num - 1) counter_name = name + str(index) counter = Counter.get_by_key_name(counter_name) if counter is None: counter = Counter(key_name=counter_name, name=name) counter.count -= 1 counter.put() db.run_in_transaction(txn) memcache_name = os.environ["CURRENT_VERSION_ID"] + "." + name memcache.decr(memcache_name)
def decrement(name):
config = GeneralCounterShardConfig.get_or_insert(name, name=name)
def txn():
index = random.randint(0, config.num_shards - 1)
shard_name = name + str(index)
counter = GeneralCounterShard.get_by_key_name(shard_name)
if counter is None:
counter = GeneralCounterShard(key_name=shard_name, name=name)
counter.count -= 1
counter.put()
db.run_in_transaction(txn)
memcache.decr(name)
def reap():
"""
Reap the counters from memcache and put them in persistent storage
We need the increment/reset instead of get/put in case memcache fails (counter resetted to 0)
"""
# This might be more efficient by paginating manually and do memcache.get_multi
for counter in get_all_counters():
value = memcache.get(counter.key.string_id(), namespace=NAMESPACE)
# if value is None:
# continue
_increment_db(counter, value)
memcache.decr(counter.key.string_id(), value, namespace=NAMESPACE)
print 'memcache: %s => %s' % (counter.key.string_id(), memcache.get(counter.key.string_id(), namespace=NAMESPACE))
def _close_batch(self):
"""Release the lock and increment the index to move to next batch."""
# Cuttoff ability to add to the index since it is now processing.
# Increment randomly to lower collision of task names when cache evicted.
memcache.incr(self.index_name, delta=random.randrange(1, 25))
# The processing has started, stop using index.
memcache.decr(self.lock_name, _PIPELINES_SENTINAL_OFFSET_VALUE)
# Add a task to cleanup any items that were missed from database delay.
eta = (datetime.datetime.utcfromtimestamp(time.time()) +
PIPELINES_ETA_BUFFER_CLEANUP)
try:
deferred.defer(self._process, _name=self.work_index + '-cleanup',
_eta=eta, _queue=PIPELINES_QUEUE)
except taskqueue.TaskAlreadyExistsError:
pass # Expected error to fan-in the tasks.
def decr(self, key, delta=1, version=None):
key = self.make_key(key, version=version)
val = memcache.decr(key, delta)
if val is None:
raise ValueError("Key '%s' not found" % key)
return val
def flush_views(cls, id):
round = cls.get_by_id(id)
if not round:
round = cls()
# Get the current value
value = memcache.get('views-' + str(id), cls.kind())
# Subtract it from the memcached value
if not value:
return
memcache.decr('views-' + str(id), int(value), cls.kind())
# Store it to the counter
round.views += int(value)
round.put()
def _Increment(name, num_shards, delta=1):
"""Transactional helper to increment the value for a given sharded counter.
Also takes a number of shards to determine which shard will be used.
Retry count bumped up to work through occasional db latency issues.
Args:
name: The name of the counter.
num_shards: How many shards to use.
delta: Non-negative integer to increment key.
Returns:
The new long integer value or None if a problem arose.
"""
index = random.randint(0, num_shards - 1)
shard_key_string = _SHARD_KEY_TEMPLATE.format(name, index)
counter = CounterShardCount.get_by_id(shard_key_string)
if counter is None:
counter = CounterShardCount(id=shard_key_string)
counter.count += delta
counter.put()
if delta < 0:
return memcache.decr(key=name, delta=-delta, initial_value=0)
return memcache.incr(key=name, delta=delta, initial_value=0)
def flush_views(cls, id):
book_web = cls.get_by_key_name(id)
if not book_web:
book_web = cls()
# Get the current value
value = memcache.get('views-' + str(id), cls.kind())
# Subtract it from the memcached value
if not value:
return
memcache.decr('views-' + str(id), int(value), cls.kind())
# Store it to the counter
book_web.views += int(value)
book_web.put()
def flush_counter(cls, name):
"""
Flush the value of a counter.
:param name: The name of the counter to flush.
"""
counter = cls.get_by_key_name(name)
if not counter:
counter = cls(key_name=name)
# Get the current value
value = memcache.get(name, cls.kind())
# Subtract it from the memcached value
memcache.decr(name, int(value), cls.kind())
# Store it to the counter
counter.count += int(value)
counter.put()
def decr(self, key, delta=1, version=None):
key = self.make_key(key, version=version)
val = memcache.decr(key, delta)
if val is None:
raise ValueError("Key '%s' not found" % key)
return val
def decrement(self):
"""Decrement the value of the counter.
"""
if self.__counter_config is None:
self.__counter_config = GeneralCounterShardConfig.get_or_insert(self.__counter_name, name=self.__counter_name)
def txn():
index = random.randint(0, self.__counter_config.num_shards - 1)
shard_name = self.__counter_name + str(index)
counter = GeneralCounterShard.get_by_key_name(shard_name)
if counter is None:
counter = GeneralCounterShard(key_name=shard_name, name=self.__counter_name)
counter.count -= 1
counter.put()
db.run_in_transaction(txn)
# does nothing if the key does not exist
memcache.decr(self.__counter_name)
def increment(cls, name, delta=1, persist_delay=10):
"""
This function increments the counter in memcache.
Args:
delta : Amount to be incremented
interval : seconds to wait before updating Datastore
"""
counter_id = cls._get_memcache_id(name)
if delta >= 0:
val = memcache.incr(counter_id, delta, initial_value=MIDDLE_VALUE)
else:
print "dec - ", memcache.get(counter_id) - MIDDLE_VALUE, delta
val = memcache.decr(counter_id, -delta, initial_value=MIDDLE_VALUE)
print "dec2 - ", memcache.get(counter_id) - MIDDLE_VALUE, delta
persist_value = val - MIDDLE_VALUE
if cls._lock_counter(name, persist_delay):
# It's time to persist the value in datastore
try:
cls._update_datastore(counter_id, persist_value)
except datastore_errors.TransactionFailedError:
# Just avoid this transaction failure and try again in next iteration
pass
return persist_value
def incrementCounter(key, delta=1, update_interval=10):
"""Increments a memcached counter.
Args:
key: The key of a datastore entity that contains the counter.
delta: Non-negative integer value (int or long) to increment key by, defaulting to 1.
update_interval: Minimum interval between updates.
"""
lock_key = "counter_lock:%s" % (key,)
count_key = "counter_val:%s" % (key,)
if memcache.add(lock_key, None, time=update_interval):
# Time to update the DB
prev_count = int(memcache.get(count_key) or 0)
new_count = prev_count + delta
def tx():
entity = CounterModel.get_by_key_name(key)
if not entity:
entity = CounterModel(key_name=key, counter=0)
entity.counter += new_count
entity.put()
try:
db.run_in_transaction(tx)
if prev_count>0 and memcache.decr(count_key, delta=prev_count) is None:
logging.warn("counter %s could not be decremented (will double-count): %d" % (key, prev_count))
except Exception, e:
# db failed to update: we'll try again later; just add delta to memcache like usual for now
memcache.incr(count_key, delta, initial_value=0)
def increment(cls, name, delta=1, persist_delay=10):
'''
This function increments the counter in memcache.
Args:
delta : Amount to be incremented
interval : seconds to wait before updating Datastore
'''
counter_id = cls._get_memcache_id(name)
if delta >= 0:
val = memcache.incr(counter_id, delta, initial_value=MIDDLE_VALUE)
else:
print "dec - ", memcache.get(counter_id) - MIDDLE_VALUE, delta
val = memcache.decr(counter_id, -delta, initial_value=MIDDLE_VALUE)
print "dec2 - ", memcache.get(counter_id) - MIDDLE_VALUE, delta
persist_value = val - MIDDLE_VALUE
if cls._lock_counter(name, persist_delay):
# It's time to persist the value in datastore
try:
cls._update_datastore(counter_id, persist_value)
except datastore_errors.TransactionFailedError:
# Just avoid this transaction failure and try again in next iteration
pass
return persist_value
def flush_views(cls, id):
round = cls.get_by_id(id)
if not round:
round = cls()
# Get the current value
value = memcache.get("views-" + str(id), cls.kind())
# Subtract it from the memcached value
if not value:
return
memcache.decr("views-" + str(id), int(value), cls.kind())
# Store it to the counter
round.views += int(value)
round.put()
def flush_counter(cls, name):
"""
Flush the value of a counter.
:param name: The name of the counter to flush.
"""
counter = cls.get_by_key_name(name)
if not counter:
counter = cls(key_name=name)
# Get the current value
value = memcache.get(name, cls.kind())
# Subtract it from the memcached value
memcache.decr(name, int(value), cls.kind())
# Store it to the counter
counter.count += int(value)
counter.put()
def decr(key, **kw):
result = memcache.decr(key, **kw)
if result:
action = "hit %s" % result
else:
action = "miss"
_log("cache decr %s %s %s" % (action, key, kw))
return result
def flush_viewcount(cls, name):
song = cls.get_by_key_name(name)
if not song:
song = cls()
# Get the current value
value = memcache.get('viewcount-' + name, cls.kind())
# Subtract it from the memcached value
if not value:
return
memcache.decr('viewcount-' + name, value, cls.kind())
# Store it to the counter
song.viewcount += value
song.put()
def _Decr(self, key):
"""Atomically decrements a key's value.
Args:
key: Key to decrement. Stored within the local rule context.
"""
# The logic for _Decr is simpler than _Incr because if a key is not in the
# cache during _Decr, then a following call to _Incr will take care of
# rebuilding the correct value for that particular key. Rules which call
# _Decr to not care about the value of the key after call is complete. Only
# subsequent calls to _Incr are of interest.
try:
if not IsPredictionMode():
memcache.decr(key, namespace=self.namespace)
except (TypeError, ValueError), e:
# Can happen if key is too long or invalid
logging.error(e)
def _close_batch(self):
"""Release the lock and increment the index to move to next batch."""
# Cuttoff ability to add to the index since it is now processing.
# Increment randomly to lower collision of task names when cache evicted.
memcache.incr(self.index_name, delta=random.randrange(1, 25))
# The processing has started, stop using index.
memcache.decr(self.lock_name, _PIPELINES_SENTINAL_OFFSET_VALUE)
# Add a task to cleanup any items that were missed from database delay.
eta = (datetime.datetime.utcfromtimestamp(time.time()) +
PIPELINES_ETA_BUFFER_CLEANUP)
try:
deferred.defer(self._process,
_name=self.work_index + '-cleanup',
_eta=eta,
_queue=PIPELINES_QUEUE)
except taskqueue.TaskAlreadyExistsError:
pass # Expected error to fan-in the tasks.
def get(self):
stathat = StatHat()
errors = memcache.get(STATHAT_MEMCACHE_ERROR_COUNT)
if errors is not None:
stathat.post_count(config.STATHAT_USER_KEY,
config.STATHAT_API_ERROR_STAT_KEY,
errors,
callback=noop)
memcache.decr(STATHAT_MEMCACHE_ERROR_COUNT, errors)
req_count = memcache.get(STATHAT_MEMCACHE_REQ_COUNT)
if req_count is not None:
stathat.post_count(config.STATHAT_USER_KEY,
config.STATHAT_API_COUNT_STAT_KEY,
req_count,
callback=noop)
memcache.decr(STATHAT_MEMCACHE_REQ_COUNT, req_count)
def _decrement(name, num_shards):
"""Transactional helper to decrement the value for a given sharded counter.
Also takes a number of shards to determine which shard will be used.
Args:
name: The name of the counter.
num_shards: How many shards to use.
"""
index = random.randint(0, num_shards - 1)
shard_key_string = SHARD_KEY_TEMPLATE.format(name, index)
counter = TailboneGeneralCounterShard.get_by_id(shard_key_string)
if counter is None:
counter = TailboneGeneralCounterShard(id=shard_key_string)
counter.count -= 1
counter.put()
# Memcache increment does nothing if the name is not a key in memcache
memcache.decr(name)
def flush_counter(cls, name, ts):
"""
Flushes the value of the counter with the specified name and timestamp to the datastore.
:param name: The name of the counter to flush.
:param ts: The timestamp to get the counter value for.
"""
ts_name = cls.get_ts_name(name, ts)
counter = cls.get_by_key_name(ts_name)
if not counter:
counter = cls(key_name=ts_name, timestamp=cls.normalize_ts(ts))
# Get the current value
value = memcache.get(ts_name, cls.kind())
# Subtract it from the memcached value
memcache.decr(ts_name, int(value), cls.kind())
# Store it to the counter
counter.count += int(value)
counter.put()
def decrement(self):
"""Decrement the value of the counter.
"""
if self.__counter_config is None:
self.__counter_config = GeneralCounterShardConfig.get_or_insert(
self.__counter_name, name=self.__counter_name)
def txn():
index = random.randint(0, self.__counter_config.num_shards - 1)
shard_name = self.__counter_name + str(index)
counter = GeneralCounterShard.get_by_key_name(shard_name)
if counter is None:
counter = GeneralCounterShard(key_name=shard_name,
name=self.__counter_name)
counter.count -= 1
counter.put()
db.run_in_transaction(txn)
# does nothing if the key does not exist
memcache.decr(self.__counter_name)
def acquireReadLock(self, index, nextEvent=None, raiseOnFail=False):
""" Acquires the read lock
@param index: an int, the current index
"""
acquired = True
lockKey = self.lockKey(index)
indexKey = self.indexKey()
# tell writers to use another index
memcache.incr(indexKey)
# tell writers they missed the boat
memcache.decr(lockKey, 2**15)
# busy wait for writers
for i in xrange(ReadWriteLock.BUSY_WAIT_ITERS):
counter = memcache.get(lockKey)
# counter is None --> ejected from memcache, or no writers
# int(counter) <= 2**15 --> writers have all called memcache.decr
if counter is None or int(counter) <= 2**15:
break
time.sleep(ReadWriteLock.BUSY_WAIT_ITER_SECS)
self.context.logger.debug(
"Tried to acquire read lock '%s' %d times...", lockKey, i + 1)
# FIXME: is there anything else that can be done? will work packages be lost? maybe queue another task
# to sweep up later?
if i >= (ReadWriteLock.BUSY_WAIT_ITERS -
1): # pylint: disable-msg=W0631
self.context.logger.critical(
"Gave up waiting for all fan-in work items with read lock '%s'.",
lockKey)
acquired = False
if raiseOnFail:
raise FanInReadLockFailureRuntimeError(
nextEvent, self.context.machineName,
self.context.currentState.name, self.context.instanceName)
return acquired
def increment(name, by=1):
"""Increment the value for a given sharded counter.
Parameters:
name - The name of the counter
"""
config = GeneralCounterShardConfig.get_or_insert(name, name=name)
def txn():
index = random.randint(0, config.num_shards - 1)
shard_name = name + str(index)
counter = GeneralCounterShard.get_by_key_name(shard_name)
if counter is None:
counter = GeneralCounterShard(key_name=shard_name, name=name)
counter.count += by
counter.put()
db.run_in_transaction(txn)
# incr and decr do nothing if the key does not exist
if by > 0:
memcache.incr(name, delta=by)
elif by < 0:
memcache.decr(name, delta=-by)
def testIncrementDecrement(self):
"""Testing automatically incrementing and decrementing."""
memcache.incr('unknown_key')
assert memcache.get('unknown_key') == None
memcache.set('counter', 0)
assert memcache.get('counter') == 0
memcache.incr('counter')
assert memcache.get('counter') == 1
memcache.incr('counter', delta=2)
assert memcache.get('counter') == 3
memcache.decr('counter')
assert memcache.get('counter') == 2
memcache.decr('counter', 2)
assert memcache.get('counter') == 0
memcache.incr('second_counter', initial_value=10)
assert memcache.get('second_counter') == 11
memcache.decr('third_counter', initial_value=10)
assert memcache.get('third_counter') == 9
# This should cause an error message, because zero deltas are not
# allowed.
memcache.incr('counter', delta=0)
memcache.set('lcounter', long(20))
assert memcache.get('lcounter') == long(20)
memcache.incr('lcounter')
assert memcache.get('lcounter') == long(21)
def _increment_index(self, last_index):
"""Moves the work index forward and waits for all writers.
Args:
last_index: The last index that was used for the reader/writer lock.
Returns:
True if all writers were definitely finished; False if the reader/writer
lock timed out and we are proceeding anyways.
"""
# Increment the batch index counter so incoming jobs will use a new index.
# Don't bother setting an initial value here because next_index() will
# do this when it notices no current index is present. Do this *before*
# closing the reader/writer lock below to decrease active writers on the
# current index.
# We do this even in the case that batch_period_ms was zero, just in case
# that memcache operation failed for some reason, we'd rather have more
# batches then have the work index pipeline stall.
memcache.incr(self.index_name)
# Prevent new writers by making the counter extremely negative. If the
# decrement fails here we can't recover anyways, so just let the worker go.
add_counter = self.add_counter_template % last_index
memcache.decr(add_counter, self.LOCK_OFFSET)
for i in xrange(self.sync_attempts):
counter = memcache.get(add_counter)
# Less than or equal LOCK_OFFSET here in case a writer decrements twice
# due to rerunning failure tasks.
if counter is None or int(counter) <= self.LOCK_OFFSET:
# Worst-case the counter will be gone due to memcache eviction, which
# means the worker can procede with without waiting for writers
# and just process whatever it can find. This may drop some work.
return True
time.sleep(self.sync_timeout)
else:
logging.critical('Worker for %s gave up waiting for writers',
self.name)
return False
def txn():
index = random.randint(0, num_shards - 1)
shard_key_string = SHARD_KEY_TEMPLATE.format(name, index)
counter = yield GeneralCounterShard.get_by_id_async(shard_key_string)
if counter is None:
counter = GeneralCounterShard(id=shard_key_string)
counter.count += value
yield counter.put_async()
if use_memcache:
if value > 0:
memcache.incr( name, delta=value )
elif value < 0:
memcache.decr( name, delta=-value )
else:
memcache.delete( name )
raise ndb.Return( True )
def add_or_subtract(name, value=0):
"""Decrement the value for a given sharded counter.
Parameters:
name - The name of the counter
"""
config = GeneralCounterShardConfig.get_or_insert(name, name=name)
def txn():
index = random.randint(0, config.num_shards - 1)
shard_name = "%s-%s" % (name, index)
counter = GeneralCounterShard.get_by_key_name(shard_name)
if counter is None:
counter = GeneralCounterShard(key_name=shard_name, name=name)
counter.count += value
counter.put()
db.run_in_transaction(txn)
if value > 0:
memcache.incr(name, value)
elif value < 0:
memcache.decr(name, -value)
def _open_batch(self):
"""Determine which batch to use for the pipeline."""
self.index_name = '{}-index'.format(self.batch_name)
while True:
# Find the latest batch index.
index = memcache.get(self.index_name)
if index is None:
# Random to lower collision of task names when cache evicted.
memcache.add(self.index_name, random.randrange(10, 10000))
index = memcache.get(self.index_name)
# Create a non-monotonically increasing name for the work index.
self.md5_hash = hashlib.md5(str(index)).hexdigest()
self.work_index = '{}-{}'.format(self.batch_name, self.md5_hash)
self.lock_name = '{}-lock'.format(self.work_index)
# Determine if the batch is still valid and available.
writers = memcache.incr(self.lock_name,
initial_value=_PIPELINES_SENTINAL_VALUE)
if writers < _PIPELINES_SENTINAL_VALUE:
memcache.decr(self.lock_name)
continue
return
