def push_email():
"""Receives Google API push notification and relays to frontend.
GAE supports HTTPS automatically, so we use it as a proxy."""
# https://developers.google.com/gmail/api/guides/push
# Test: curl -X POST http://www.darkzero.net/webhook/google/push/email
with requests.Session() as session:
# noinspection PyTypeChecker
session.mount('http://', HTTPAdapter(max_retries=Retry(total=5, backoff_factor=.5)))
headers = {
'Content-Type': 'application/json'
}
data = flask.request.data
# Make request.
r = session.post(PUSH_URL, headers=headers, data=data)
if r.status_code == requests.codes.ok:
memcache.incr(KEY_SENT)
LOG.info('Pushed: %s:%s' % (PUSH_URL, flask.request.data))
else:
# Count error but ACK to Google so retries don't stack up.
memcache.incr(KEY_ERROR)
return flask.make_response()
def post(self):
name = self.request.get('name', '')
comment = self.request.get('comment', '')
try:
message = model.Message(name=name, comment=comment)
message.put()
counter = model.Counter.get_by_key_name('message')
if counter is None:
counter = model.Counter(key_name='message')
counter.value = counter.value + 1
counter.put()
if memcache.get('total_msgs'):
memcache.incr('total_msgs')
else:
total_msgs = counter.value
memcache.set('total_msgs', total_msgs)
total_pages = counter.value / MSGS_PER_PAGE
if counter.value % MSGS_PER_PAGE:
total_pages += 1
if memcache.get('msgs_%d' % total_pages):
memcache.delete('msgs_%d' % total_pages)
self.redirect('/')
except BadValueError:
self.response.out.write('You have not filled the name or comment field. <a href="/">Back »</a>')
def post(self):
teamName=self.request.get("teamName")
#Lock Mechanism to prevent multi-thread incidence
#if lock then wait
lock=memcache.get("lock"+teamName)
count=0
while lock and count<=10:
memcache.set("sleep"+teamName,0)
sleep=0
while sleep<=100:
memcache.incr("sleep"+teamName)
sleep=memcache.get("sleep"+teamName)
lock=memcache.get("lock"+teamName)
#set lock
memcache.set("lock"+teamName,True)
team=Team.getByTeamName(teamName)
if team:
try:
team.updateAdvancedPayment()
team.save()
self.render("backend.html",success=True)
except:
self.render("backend.html",error=True)
else:
self.render("backend.html",error=True)
#release lock
memcache.delete("lock"+teamName)
def get(self):
guestbook_name = self.request.get('guestbook_name',
DEFAULT_GUESTBOOK_NAME)
greetings_query = Greeting.query(
ancestor=guestbook_key(guestbook_name)).order(-Greeting.date)
greetings = greetings_query.fetch(10)
user = users.get_current_user()
if user:
data = memcache.get('counter')
if data==None:
memcache.set(key="counter", value=0)
memcache.incr("counter")
url = users.create_logout_url(self.request.uri)
url_linktext = 'Logout'
else:
url = users.create_login_url(self.request.uri)
url_linktext = 'Login'
count = memcache.get('counter')
template_values = {
'user': user,
'greetings': greetings,
'guestbook_name': urllib.quote_plus(guestbook_name),
'url': url,
'count':count,
'url_linktext': url_linktext,
}
template = JINJA_ENVIRONMENT.get_template('templates/index.html')
self.response.write(template.render(template_values))
def post(self):
t = memcache.get("t")
if t == None:
t = 0
memcache.set("t", t)
else:
memcache.incr("t")
def incrementCounter(key, delta=1, update_interval=10):
"""Increments a memcached counter.
Args:
key: The key of a datastore entity that contains the counter.
delta: Non-negative integer value (int or long) to increment key by, defaulting to 1.
update_interval: Minimum interval between updates.
"""
lock_key = "counter_lock:%s" % (key,)
count_key = "counter_val:%s" % (key,)
if memcache.add(lock_key, None, time=update_interval):
# Time to update the DB
prev_count = int(memcache.get(count_key) or 0)
new_count = prev_count + delta
def tx():
entity = CounterModel.get_by_key_name(key)
if not entity:
entity = CounterModel(key_name=key, counter=0)
entity.counter += new_count
entity.put()
try:
db.run_in_transaction(tx)
if prev_count>0 and memcache.decr(count_key, delta=prev_count) is None:
logging.warn("counter %s could not be decremented (will double-count): %d" % (key, prev_count))
except Exception, e:
# db failed to update: we'll try again later; just add delta to memcache like usual for now
memcache.incr(count_key, delta, initial_value=0)
def testAddMultipleDifferent(self):
"""Tests adding multiple different tasks to different indexes."""
t1 = TestModel(number=1)
t2 = TestModel(number=2)
t3 = TestModel(number=3)
tasks1 = [t1, t2, t3]
work_index = TEST_QUEUE.next_index()
for t in tasks1:
t.work_index = work_index
db.put(tasks1)
TEST_QUEUE.add(work_index, gettime=self.gettime1)
memcache.incr('fjq-TestModel-index')
t4 = TestModel(number=4)
t5 = TestModel(number=5)
t6 = TestModel(number=6)
tasks2 = [t4, t5, t6]
work_index2 = TEST_QUEUE.next_index()
for t in tasks2:
t.work_index = work_index2
db.put(tasks2)
TEST_QUEUE.add(work_index2, gettime=self.gettime2)
self.assertNotEqual(work_index, work_index2)
self.assertTasksEqual(
[self.expect_task(work_index),
self.expect_task(work_index2, now_time=self.now2)],
testutil.get_tasks('default', usec_eta=True))
def incr(cls, key, delta, namespace=None):
"""Incr an item in memcache if memcache is enabled."""
#logging.debug("MemcacheManager:incr(%s, %s, ns=%s)", key, str(delta), namespace)
if CAN_USE_MEMCACHE.value:
if not namespace:
namespace = appengine_config.DEFAULT_NAMESPACE_NAME
memcache.incr(key, delta, namespace=namespace, initial_value=0)
def loggingreq(self, response_status, response_content_length, if_use_cache):
header_referer = ''
header_user_agent = ''
for name, address in self.request.headers.items():
if name.lower() == "referer":
header_referer = address
elif name.lower() == "user-agent":
header_user_agent = address
request_get = {'ip_addr' : self.request.remote_addr,
'time' : datetime.datetime.now(),
'req_method' : self.request.environ["REQUEST_METHOD"],
'req_url' : self.request.url,
'req_PROTOCOL' : self.request.environ["SERVER_PROTOCOL"],
'resp_status' : response_status,
'resp_length' : response_content_length,
'referer' : header_referer,
'user_agent' : header_user_agent,
'if_cache' : if_use_cache
}
count = memcache.get('AccessLogNo')
if count is not None:
memcache.incr('AccessLogNo')
count = count + 1
else:
memcache.set('AccessLogNo', 1)
count = 1
memcache.set("AccessLogNo" + repr(count), request_get)
def decorated(myself, *args, **kwargs):
method = myself.request.method
parts = [method, myself.request.path]
whitelisted = False
if DISABLE_FOR_TESTING:
return func(myself, *args, **kwargs)
if param:
value = myself.request.get(param)
if value:
parts.append('%s=%s' % (param, value))
if value in param_whitelist:
whitelisted = True
if header:
value = os.environ.get(header)
if value:
parts.append('%s=%s' % (header, value))
if value in header_whitelist:
whitelisted = True
key = ' '.join(parts)
result = None
if len(parts) != required_parts:
logging.critical('Incomplete rate-limit key = "%s" for param = "%s", '
'header = "%s" on "%s" where count = %s, period = %s, '
'limit = %.3f/sec', key, param, header, method,
count, period, limit)
else:
result = memcache.incr(key)
if result is None:
# Rate limit not yet in memcache.
result = 1
if not memcache.add(key, result, time=period):
# Possible race for who adds to the cache first.
result = memcache.incr(key)
if result is None:
# Memcache definitely down.
skip_enforcement = True
logging.error('Memcache failed for rate limit on "%s" by "%s" '
'where count = %s, period = %s, limit = %.3f/s',
method, key, count, period, limit)
if not whitelisted and result > count:
rate = float(result) / period
if (result - count) == 1:
log_level = logging.error
else:
log_level = logging.debug
log_level('Hit rate limit on "%s" by "%s" where '
'count = %s, period = %s, rate = %.3f/s, limit = %.3f/s',
method, key, count, period, rate, limit)
myself.response.set_status(error_code)
myself.response.headers['Content-Type'] = 'text/plain'
if retry_after is not None:
myself.response.headers['Retry-After'] = str(retry_after)
values = {'key': key, 'rate': rate, 'limit': limit}
myself.response.out.write(message % values)
else:
return func(myself, *args, **kwargs)
def post(self, base_url):
fetched_content = self.fetch_content(urlfetch.POST, self.request.path_qs, self.request.headers)
if fetched_content is not None:
self.content_response(fetched_content, 1)
if (fetched_content["code"] is 200) and IF_USE_MEMCACHE:
for name, address in self.request.headers.items():
if name.lower() == "referer":
item = re.sub(r'(^http://)' + PROXY_SER_URL + '/' , '/', address)
#refresh the memcache of referer page
#often do not work very well due to the structure of site.
result = self.fetch_content(urlfetch.GET, item, {})
if result is not None:
self.cache_content(item ,result)
# self.loggingreq(fetched_content["code"], len(fetched_content["main_content"]), False)
else:
count = memcache.get("pending_post_no")
if count is not None:
memcache.incr('pending_post_no')
count = count + 1
else:
#Store pending post(incomplete fetch due to network problems) in memcache
memcache.set('pending_post_no', 1)
count = 1
pending_post = {'ip_addr' : self.request.remote_addr,
'time' : datetime.datetime.now(),
'req_url' : self.request.path_qs,
'content' : self.request.body
}
memcache.set("pending_post_no" + repr(count) + "info", pending_post)
pending_post_headers = {}
for name, address in self.request.headers.items():
pending_post_headers[name] = address
memcache.set("pending_post_no" + repr(count) + "headers", \
pending_post_headers)
def get(self):
memcache.incr("pv_userlist", initial_value=0)
user = users.get_current_user()
prefs = InternalUser.from_user(user)
q = decode(self.request.get('q'))
p = decode(self.request.get('p'))
_users = InternalUser.all()
if q == "1" or not q:
_users.order("-points")
else:
#_users.filter("date_lastactivity !=", None)
_users.order("-date_lastactivity")
page = int(p) if p else 1
items_per_page = 40
n = items_per_page / 2
offset = (page - 1) * items_per_page
_users1 = _users.fetch(n, offset)
_users2 = _users.fetch(n, (offset + (items_per_page / 2)))
values = {'prefs': prefs, 'users1': _users1, 'users2': _users2,
'page': page, 'pages': range(1, page), 'prefix': q}
self.response.out.write( \
template.render(tdir + "users.html", values))
def post(self):
# Gets the parameters.
user_id = self.request.get('user_id')
timestamp = self.request.get('timestamp')
index = int(self.request.get('resource_index'))
memcache.set(get_key(self, 'counter'), index)
# Moves to the next phase.
if get_phase(self) == 3:
memcache.set(get_key(self, 'counter'), 0)
memcache.incr(get_key(self, 'phase'))
# Terminates the process if canceled.
if get_phase(self) != 4:
logging.info('This request was canceled.')
return
if index < int(memcache.get(get_key(self, 'resource_num')) or 0):
# Fetches a resource file.
resource = fetch_content(memcache.get(get_resources_key(self, index)))
# Updates download size.
bytes = int(memcache.get(get_key(self, 'resources_bytes')) or 0)
bytes += len(resource)
memcache.set(get_key(self, 'resources_bytes'), bytes)
taskqueue.add(url='/smartfm/memento/tasks/fetch', params={'user_id': user_id, 'timestamp': timestamp, 'resource_index': index + 1})
else:
# Executes the next task to move to the next phase.
taskqueue.add(url='/smartfm/memento/tasks/divide', params={'user_id': user_id, 'timestamp': timestamp})
def post(self):
if not self.current_user: return self.redirect('/login') # login check
# rate limit to 5 sends every hour
memcache_key = self.current_user.id + "_s"
if not memcache.get(memcache_key):
# set hourly limit
memcache.add(memcache_key, 1, 3600)
elif memcache.get(memcache_key) < 5:
# increase hourly limit
memcache.incr(memcache_key)
else:
# hit limit
return self.redirect('/home?limit=1')
message = Message()
if self.request.get("hide"):
message.hide_user = True
else:
message.hide_user = False
message.user = self.current_user
message.content = self.request.get('content')[0:499]
if message.content.strip() != '': message.put()
return self.redirect(self.request.get('redirect'))
def _increase_cache(self, key): """ Increases a cache value, creates the key on demand. """ added = memcache.add(key, 1, time=self.expire_after) if not added: memcache.incr(key)
def get(self, category=None):
if not category or category == "/":
category = "/active"
memcache.incr("pv_main", initial_value=0)
user = users.get_current_user()
prefs = InternalUser.from_user(user)
p = decode(self.request.get('p'))
page = int(p) if p else 1
# Get snippets list from cache
#mc.cache.snippet_list(None, clear=True)
snippets = mc.cache.snippet_list(category, page)
# Set title
if category == "/new":
title = "New Snippets"
elif category == "/active":
title = "Active Snippets"
elif category == "/popular":
title = "Popular Snippets"
elif category == "/comments":
title = "Recently Commented Snippets"
elif category == "/edits":
title = "Snippets with Edits"
# Build template
values = {'prefs': prefs, 'snippets': snippets, 'title': title, \
'page': page, 'pages': range(1, page)}
self.response.out.write(template.render(tdir + \
"snippet_list.html", values))
def fan_in(self, work_item, eta_delta=PIPELINES_ETA_DEFAULT_DELTA):
"""Creates a fan in task for batch processing the work item."""
counter = 0
eta = datetime.datetime.utcfromtimestamp(time.time()) + eta_delta
# Keep trying to find a valid work index when the taskname is tombstoned.
while True:
try:
try:
# Use a named queue to perform the fan-in.
deferred.defer(self._fan_in, _name=self.work_index, _eta=eta,
_queue=PIPELINES_QUEUE)
except taskqueue.TaskAlreadyExistsError:
pass # Expected error to fan-in the tasks.
break # End the loop since the task was created or fanned-in.
except taskqueue.TombstonedTaskError:
# Keep trying until we get to a non-tombstoned task name.
# Increment randomly to find another non-tomestoned task name.
memcache.incr(self.index_name, delta=random.randrange(1, 100))
self._open_batch() # Reattempt to reopen a new batch.
counter += 1
logging.info('Tombstoned error, retrying: {}'.format(counter))
# Add processor to the Datastore if successfully fanned in.
work_item.work_index = self.work_index
work_item.put_async()
def _reload_version(self, key):
"""Reload a single package version from its tarball."""
version = PackageVersion.get(key)
with closing(cloud_storage.read(version.storage_path)) as f:
new_version = PackageVersion.from_archive(
f, uploader=version.uploader)
with models.transaction():
# Reload the old version in case anything (e.g. sort order) changed.
version = PackageVersion.get(key)
package = version.package
# We don't load new_version.package.latest_version here for two
# reasons. One is to avoid a needless data store lookup; the other
# is because it's possible that that version is being reloaded in
# another transaction and thus in a weird transitional state.
latest_version_key = Package.latest_version.get_value_for_datastore(
package)
if latest_version_key == key:
package.latest_version = new_version
new_version.created = version.created
new_version.downloads = version.downloads
new_version.sort_order = version.sort_order
version.delete()
new_version.put()
package.put()
memcache.incr('versions_reloaded')
def get_keys(keys):
"""
Fetch and save the given keys into memcache.
"""
# Perform the first load from memcache.
urlsafes = [key.urlsafe() for key in keys]
cache = memcache.get_multi(urlsafes, key_prefix=RTC)
results = {}
missing = []
for urlsafe, key in zip(urlsafes, keys):
if urlsafe in cache:
results[key] = protobuf_to_ndb_entity(cache[urlsafe])
else:
missing.append(key)
# Record cache stats.
memcache.incr(MISSES, delta=len(missing), initial_value=0)
memcache.incr(HITS, delta=len(results), initial_value=0)
# Fill anything not yet in cache.
if missing:
writeback = {}
for key, entity in zip(missing, ndb.get_multi(missing)):
results[key] = entity
writeback[key.urlsafe()] = ndb_entity_to_protobuf(entity)
memcache.set_multi(writeback, key_prefix=RTC)
return [results[key] for key in keys]
def invalidate_keys(keys):
"""
Removes the given entities from memcache.
"""
memcache.incr(INVALIDATIONS, delta=len(keys), initial_value=0)
memcache.delete_multi([key.urlsafe() for key in keys], key_prefix=RTC)
def post(self):
teamName=self.request.get("teamName")
#Lock Mechanism to prevent multi-thread incidence
#if lock then wait
lock=memcache.get("lock"+teamName)
count=0
while lock and count<=10:
memcache.set("sleep"+teamName,0)
sleep=0
while sleep<=100:
memcache.incr("sleep"+teamName)
sleep=memcache.get("sleep"+teamName)
lock=memcache.get("lock"+teamName)
#set lock
memcache.set("lock"+teamName,True)
team=Team.getByTeamName(teamName)
if team:
try:
startTime=team.appointmentSlot
team.appointmentSlot=None
team.save()
timeSlot=TimeSlot.getByStartTime(startTime)
timeSlot.cancelTeamParticipant(teamName)
timeSlot.save()
self.render("suspend.html",success=True)
except:
self.render("suspend.html",error=True)
else:
self.render("suspend.html",error=True)
def post(self):
if not self.current_user: return self.redirect('/login') # login check
# rate limit to 60 replies every hour
memcache_key = self.current_user.id + "_r"
if not memcache.get(memcache_key):
# set hourly limit
memcache.add(memcache_key, 1, 3600)
elif memcache.get(memcache_key) < 60:
# increase hourly limit
memcache.incr(memcache_key)
else:
# hit limit
return self.redirect('/home?limit=2')
reply = Reply()
if self.request.get("hide"):
reply.hide_user = True
else:
reply.hide_user = False
reply.user = self.current_user
reply.message = db.get(self.request.get('message_id'))
reply.content = self.request.get('content')[0:499]
if reply.content.strip() != '': reply.put()
return self.redirect(self.request.get('redirect'))
def get(self, campaign_id, platform_name):
"""
Handles incoming clicks for given campaign_id and platform_name.
If click is valid then user is redirected to url defined in the campaign
and statistic about this click is saved. All invalid clicks (e.g. for non
existing campaigns, platforms) users are redirected to http://outfit7.com.
"""
# cast campaign_id, type checking is done through route definition
try:
campaign_id = int(campaign_id)
except ValueError:
return webapp2.redirect("http://outfit7.com", permanent=True)
platform_id = "%d-%s" % (campaign_id, platform_name)
platform = Platform.get_by_id(platform_id)
if platform:
memcache.incr(platform_id, 1, namespace="counters", initial_value=0)
try:
deferred.defer(Platform.increment, platform_id, _countdown=TRACKER_COUNTER_UPDATE_INTERVAL_LENGTH,
_name="%s-%d" % (platform_id, get_interval_index()))
except (taskqueue.TaskAlreadyExistsError, taskqueue.TombstonedTaskError), e:
pass
# TODO: optimize with async operations
campaign = Campaign.get_by_id(campaign_id)
return webapp2.redirect(campaign.link.encode("utf8"))
def report(request, uid=None):
if not uid:
items = { 'success' : False, 'num_magnets' : 0, 'uids' : None }
if request.method == 'POST':
items['success'] = True
items['uids'] = re.findall(r"[0-9a-f]{40}", request.POST.get('shit', ''))
for uid in items['uids']:
ref = uid_append(uid)
if ref:
ref.put()
memcache.incr('numt', initial_value=1)
items['num_magnets'] += 1
items['num_torrents'] = memcache.get('numt')
return render_to_response('report.html', items)
ref = uid_append(uid)
if ref:
ref.put()
memcache.incr('numt', initial_value=1)
return HttpResponse("uid added")
else:
return HttpResponse("duplicate uid")
def add_or_remove(cls, fingerprint, crash, argv=None, labels=None, is_add=True, delta=1):
# use an issue if one already exists
issue = CrashReport.most_recent_issue(CrashReport.key_name(fingerprint))
key_name = CrashReport.key_name(fingerprint)
config = ShardedCounterConfig.get_sharded_config(key_name)
shards = config.shards
shard_to_use = random.randint(0, shards-1)
shard_key_name = key_name + '_' + str(shard_to_use)
if not argv:
argv = []
crash_report = CrashReport \
.get_or_insert(shard_key_name,
name=key_name,
crash=crash,
fingerprint=fingerprint,
argv=argv,
labels=labels,
issue=issue)
if is_add:
crash_report.count += delta
crash_report.put()
# update caches
memcache.incr(CrashReport.count_cache_key(key_name), delta, initial_value=0)
else:
crash_report.count -= delta
crash_report.put()
memcache.decr(CrashReport.count_cache_key(key_name), delta)
# clear properties cache
CrashReport.clear_properties_cache(key_name)
return crash_report
def increment(self, incr=1): value = memcache.get(self.key, namespace=self.namespace) if value is None: self.count = incr elif incr > 0: memcache.incr(self.key, incr, namespace=self.namespace) elif incr < 0: memcache.decr(self.key, -incr, namespace=self.namespace)
def incr_counter():
data = memcache.get("counter")
if data is not None:
memcache.incr("counter")
return data + 1
else:
memcache.set(key="counter", value=0, time=10) # 10 sec
return 0
def increment_message_counter(owner):
"""Increment the value for a owner.
Parameters:
owner - The owner
"""
keyname = owner.nickname()
memcache.incr(keyname)
def post(self):
get_or_create_pings()
memcache.incr('pings');
current_user = users.get_current_user()
if current_user == None:
self.response.out.write('logged_out')
else:
self.response.out.write('ok')
def increment(self, incr=1): value = memcache.get(self.key) if value is None: self.count = incr elif incr > 0: memcache.incr(self.key, incr) elif incr < 0: memcache.decr(self.key, -incr)
def testIncrementAfterDeleteWithTimeout(self):
"""Tests that increment after a delete with timeout will work properly."""
self.assertTrue(memcache.set(self.key1, 10))
self.assertEqual(10, memcache.get(self.key1))
self.assertEqual(memcache.DELETE_SUCCESSFUL,
memcache.delete(self.key1, seconds=5))
self.assertEqual(None, memcache.get(self.key1))
self.assertEqual(1, memcache.incr(self.key1, delta=1, initial_value=0))
def bump_batch(work_group):
"""Return the incremented batch id for the work group
:param work_group: :class: `str`
:return: :class: `int` current batch id.
"""
key = "%s-%s" % (MESSAGE_BATCH_NAME, work_group)
return memcache.incr(key)
def GetNextSuccessMsg(username=None, cheeky=False):
"""Return the next success message."""
key = 'next_success_msg_index'
if cheeky:
messages = _CHEEKY_MSGS
else:
messages = _SUCCESS_MSGS
try:
index = int(memcache.get(key, namespace=username))
except TypeError:
index = 0
if index + 1 >= len(messages):
memcache.set(key, 0, namespace=username) # Reached the end, reset to 0.
else:
memcache.incr(key, initial_value=0, namespace=username) # Increment by 1.
return messages[index]
def getPeopleCount(increment=False):
count = memcache.get("people_counter")
if count is None:
count = Person.query().count()
memcache.add('people_counter', count)
if increment:
count = memcache.incr("people_counter")
return count
def incr(cls, name, interval=5, value=1):
"""Increments a counter.
:param name: The name of the counter to increment.
:param interval: How frequently (in seconds) to call :py:meth:`flush_counter`.
:param value: The value to increment the counter by.
"""
memcache.incr(name, value, cls.kind(), initial_value=0)
interval_num = get_interval_number(datetime.datetime.now(), interval)
task_name = '-'.join(
[cls.kind(), name,
str(interval), str(interval_num)])
try:
deferred.defer(cls.flush_counter, name, _name=task_name)
except (taskqueue.TaskAlreadyExistsError,
taskqueue.TombstonedTaskError):
pass
def pingpong_incr(key_name):
# Changing slot every 10 minutes
slot = gmtime().tm_min / 10 % 2
# A bug with initial value: http://code.google.com/p/googleappengine/issues/detail?id=2012
if memcache.incr('%s_%s' % (key_name, slot), namespace='ia') is None:
# Can set no such key existed
memcache.set('%s_%s' % (key_name, slot), 1, namespace='ia')
def _Increment(name, num_shards):
"""Transactional helper to increment the value for a given sharded counter.
Also takes a number of shards to determine which shard will be used.
Args:
name: The name of the counter.
num_shards: How many shards to use.
"""
index = random.randint(0, num_shards - 1)
shard_key_string = SHARD_KEY_TEMPLATE.format(name, index)
counter = GeneralCounterShard.get_by_id(shard_key_string)
if counter is None:
counter = GeneralCounterShard(id=shard_key_string)
counter.count += 1
counter.put()
# Memcache increment does nothing if the name is not a key in memcache
memcache.incr(name)
def _close_batch(self):
"""Release the lock and increment the index to move to next batch."""
# Cuttoff ability to add to the index since it is now processing.
# Increment randomly to lower collision of task names when cache evicted.
memcache.incr(self.index_name, delta=random.randrange(1, 25))
# The processing has started, stop using index.
memcache.decr(self.lock_name, _PIPELINES_SENTINAL_OFFSET_VALUE)
# Add a task to cleanup any items that were missed from database delay.
eta = (datetime.datetime.utcfromtimestamp(time.time()) +
PIPELINES_ETA_BUFFER_CLEANUP)
try:
deferred.defer(self._process,
_name=self.work_index + '-cleanup',
_eta=eta,
_queue=PIPELINES_QUEUE)
except taskqueue.TaskAlreadyExistsError:
pass # Expected error to fan-in the tasks.
def post(self):
key = self.request.get('key')
if memcache.incr(key) is None:
memcache.add(key, 1)
if memcache.add(key + '_dirty', 1):
taskqueue.add(
url='/worker/write_behind',
params={'key': key})
self.redirect('/counters')
def increase_counter(cls, instance):
'''
Increment the counter of given key
'''
instance = str(instance)
def increase():
index = random.randint(0, SHARDS -
1) #select a random shard to increases
shard_key = instance + str(index) #creates key_name
counter = RouteCounter.get_by_key_name(shard_key)
if not counter: #if counter doesn't exist, create a new one
counter = RouteCounter(key_name=shard_key, instance=instance)
counter.count += 1
counter.put()
db.run_in_transaction(increase)
memcache.incr(instance, initial_value=0)
def increment(name):
"""Increment the value for a given sharded counter.
Parameters:
name - The name of the counter
"""
config = GeneralCounterShardConfig.get_or_insert(name, name=name)
def txn():
index = random.randint(0, config.num_shards - 1)
shard_name = name + str(index)
counter = GeneralCounterShard.get_by_key_name(shard_name)
if counter is None:
counter = GeneralCounterShard(key_name=shard_name, name=name)
counter.count += 1
counter.put()
db.run_in_transaction(txn)
# does nothing if the key does not exist
memcache.incr(name)
def txn():
calls = memcache.incr('calls')
result = ResultModel.get_by_key_name('test')
if not result:
result = ResultModel(key_name='test', total=0)
result.total += len(keys)
result.put()
if memcache.get('raise') and (calls % 2):
raise db.Error()
def get(self):
"""Iterate and clean useless page data and it's related data."""
data_to_delete = []
count = 0
# Let's check if there is cursor provided, then let's use it to iterate.
last_cursor = self.GetOptionalParameter(parameter_name='cursor')
# Limit is used to restrict the amount of data being scanned each time.
limit = self.GetOptionalParameter(parameter_name='limit',
default_value=10)
q = page_data.PageData.all()
if last_cursor:
q = q.with_cursor(last_cursor)
pd = q.fetch(int(limit))
# Let's iterate through page data model and find values which has no
# corresponding test suite.
for data in pd:
try:
db.get(data.test_suite.key())
except db.Error:
data.DeleteData()
data_to_delete.append(data)
count += 1
memcache.incr(key=MEMCACHE_KEY_CLEANUP_DATA_COUNT,
initial_value=1)
db.delete(data_to_delete)
cursor = q.cursor()
# Let's prepate next task info and add that task to queue.
task_params = {'cursor': cursor, 'limit': limit}
if pd:
logging.info('%d data removed.', count)
taskqueue.add(url=CLEANUP_URL, params=task_params, method='GET')
logging.info('Cleanup Data Task Added.')
self.response.out.write('%d data removed. Cleanup task is added.' %
count)
else:
# Looks like there is no more PageData values to scan. So we are done !
final_count = memcache.get(key=MEMCACHE_KEY_CLEANUP_DATA_COUNT)
if not final_count:
logging.info('No Data removed.')
else:
logging.info('Total %s data removed.', final_count)
memcache.delete(key=MEMCACHE_KEY_CLEANUP_DATA_COUNT)
logging.info('Clean up data task done!')
self.response.out.write('Cleanup task is Done!')
def detect_spam(ip, interval=600, count=600):
if not memcache.get(ip, namespace='spam') is None:
return True
key = ip + ' '
value = memcache.incr(key, namespace='spam')
if value is None:
memcache.set(key, 0, time=interval, namespace='spam')
elif int(value) >= count:
memcache.set(ip, '', time=24 * 60 * 60, namespace='spam')
return False
def add(self, index, gettime=time.time):
"""Adds a task for a work index, decrementing the writer lock."""
now_stamp = gettime()
# Nearest gap used to kickstart the queues when a task is dropped or
# memcache is evicted. This prevents new task names from overlapping with
# old ones.
nearest_gap = int(now_stamp / self.stall_timeout)
# Include major version in the task name to ensure that test tasks
# enqueued from a non-default major version will run in the new context
# instead of the default major version.
major_version, minor_version = os.environ['CURRENT_VERSION_ID'].split(
'.')
task_name = '%s-%s-%d-%d-%d' % (self.name, major_version, nearest_gap,
index, 0)
# When the batch_period_ms is zero, then there should be no ETA, the task
# should run immediately and the reader will busy wait for all writers.
if self.batch_delta is None:
eta = None
else:
eta = datetime_from_stamp(now_stamp) + self.batch_delta
try:
taskqueue.Task(method='POST',
name=task_name,
url=self.task_path,
eta=eta).add(self.get_queue_name(index))
if self.batch_delta is None:
# When the batch_period_ms is zero, we want to immediately move the
# index to the next position as soon as the current batch finishes
# writing its task. This will only run for the first successful task
# inserter.
memcache.incr(self.index_name)
except taskqueue.TaskAlreadyExistsError:
# This is okay. It means the task has already been inserted by another
# add() call for this same batch. We're holding the lock at this point
# so we know that job won't start yet.
pass
except taskqueue.TombstonedTaskError, e:
# This is bad. This means 1) the lock we held expired and the task already
# ran, 2) this task name somehow overlaps with an old task. Return the
# error to the caller so they can try again.
raise TaskConflictError('Task named tombstoned: %s' % e)
def get(self):
# [START sharing]
self.response.headers['Content-Type'] = 'text/plain'
who = memcache.get('who')
self.response.write('Previously incremented by %s\n' % who)
memcache.set('who', 'Python')
count = memcache.incr('count', 1, initial_value=0)
self.response.write('Count incremented by Python = %s\n' % count)
def increment(name):
"""Increment the value for a given sharded counter.
Parameters:
name - The name of the counter
"""
config = GeneralCounterShardConfig.get_or_insert(name, name=name)
def txn():
index = random.randint(0, config.num_shards - 1)
shard_name = name + str(index)
counter = GeneralCounterShard.get_by_key_name(shard_name)
if counter is None:
counter = GeneralCounterShard(key_name=shard_name, name=name)
counter.count += 1
counter.put()
db.run_in_transaction_custom_retries(100, txn)
memcache.incr(name, initial_value=0)
def post(self):
# Gets the parameters.
user_id = self.request.get('user_id')
timestamp = self.request.get('timestamp')
index = int(self.request.get('resource_index'))
memcache.set(get_key(self, 'counter'), index)
# Moves to the next phase.
if get_phase(self) == 3:
memcache.set(get_key(self, 'counter'), 0)
memcache.incr(get_key(self, 'phase'))
# Terminates the process if canceled.
if get_phase(self) != 4:
logging.info('This request was canceled.')
return
if index < int(memcache.get(get_key(self, 'resource_num')) or 0):
# Fetches a resource file.
resource = fetch_content(
memcache.get(get_resources_key(self, index)))
# Updates download size.
bytes = int(memcache.get(get_key(self, 'resources_bytes')) or 0)
bytes += len(resource)
memcache.set(get_key(self, 'resources_bytes'), bytes)
taskqueue.add(url='/smartfm/memento/tasks/fetch',
params={
'user_id': user_id,
'timestamp': timestamp,
'resource_index': index + 1
})
else:
# Executes the next task to move to the next phase.
taskqueue.add(url='/smartfm/memento/tasks/divide',
params={
'user_id': user_id,
'timestamp': timestamp
})
def filter_the_abusers(caller):
# filter the troublemakers
if caller in config.ABUSERS:
counter = memcache.get(caller)
if counter is None:
memcache.set(caller, 1)
elif int(counter) <= 3:
memcache.incr(caller, 1)
else:
# create an event to log the quota problem
task = Task(url='/loggingtask',
params={
'phone': self.request.get('From'),
'inboundBody': self.request.get('Body'),
'sid': self.request.get('SmsSid'),
'outboundBody': 'exceeded quota',
})
task.add('eventlogger')
return True
return False
def post(self, post_id):
data = article.get_by_id(int(post_id))
if memcache.get(post_id+"total") == None:
if data.total == 0:
memcache.add(key=post_id+"total", value=0, time = 3600)
memcache.add(key=post_id+"up", value=0, time = 3600)
else:
memcache.add(key=post_id+"total", value=data.total, time = 4000)
memcache.add(key=post_id+"up", value=data.up, time = 4000)
memcache.incr(post_id+"total")
value = int(self.request.get("value"))
if value == 1:
memcache.incr(post_id+"up")
d = datetime.datetime.now()
d = d + datetime.timedelta(30)
total = memcache.get(post_id+"total")
up = memcache.get(post_id+"up")
ans = "%s,%s" % (up,(total-up))
self.response.set_cookie('vote',"success",path='/article/'+ post_id,expires=d)
self.response.out.write(ans)
def simulation_queue(request):
#Getting the parameters
betType = request.POST.get('betType', 'JP')
nDeck = int(request.POST.get('nDeck', '2'))
maxBet = int(request.POST.get('maxBet', '0'))
iRun = int(request.POST.get('iRun', '0'))
NRun = int(request.POST.get('NRun', '0'))
#Simulating
sim = simulation.simulation(betType, nDeck, 1000, maxBet=maxBet)
sim.run()
#Saving results to cache
memcache.incr("counter")
count = memcache.get("counter")
memcache.set("run" + str(iRun), sim)
memcache.set("progress", count * 100 / NRun)
return HttpResponse()
def update_event_counts(event_key):
"""Fetch work for this event and update the event counts."""
event_key = event_key.urlsafe()
base_task_name = "counts-%s" % (event_key, )
memcache.incr(base_task_name, initial_value=0)
queue = taskqueue.Queue(name=EVENT_UPDATE_QUEUE)
work = queue.lease_tasks_by_tag(lease_seconds=20,
max_tasks=250,
tag=event_key,
deadline=3)
counts = _get_counts_from_work(work)
_apply_count_updates(event_key, counts)
queue.delete_tasks(work)
def acquireReadLock(self, index, nextEvent=None, raiseOnFail=False):
""" Acquires the read lock
@param index: an int, the current index
"""
acquired = True
lockKey = self.lockKey(index)
indexKey = self.indexKey()
# tell writers to use another index
memcache.incr(indexKey)
# tell writers they missed the boat
memcache.decr(lockKey, 2**15)
# busy wait for writers
for i in xrange(ReadWriteLock.BUSY_WAIT_ITERS):
counter = memcache.get(lockKey)
# counter is None --> ejected from memcache, or no writers
# int(counter) <= 2**15 --> writers have all called memcache.decr
if counter is None or int(counter) <= 2**15:
break
time.sleep(ReadWriteLock.BUSY_WAIT_ITER_SECS)
self.context.logger.debug(
"Tried to acquire read lock '%s' %d times...", lockKey, i + 1)
# FIXME: is there anything else that can be done? will work packages be lost? maybe queue another task
# to sweep up later?
if i >= (ReadWriteLock.BUSY_WAIT_ITERS -
1): # pylint: disable-msg=W0631
self.context.logger.critical(
"Gave up waiting for all fan-in work items with read lock '%s'.",
lockKey)
acquired = False
if raiseOnFail:
raise FanInReadLockFailureRuntimeError(
nextEvent, self.context.machineName,
self.context.currentState.name, self.context.instanceName)
return acquired
def run_analysis(org_key=None):
"""
Task that deletes all existing CombinedUsers and regenerates them from the
underlying IntakeUsers. This task can take a long time to run, so it should
be run on a dedicated instance.
"""
DELETE_BATCH = 500
ANALYZE_BATCH = 50
# Clear out the existing combined users
cu_query = CombinedUser.query()
if org_key:
org_key = ndb.Key(urlsafe=org_key)
cu_query = cu_query.filter(CombinedUser.orgs == org_key)
while True:
results, cursor, more = cu_query.fetch_page(DELETE_BATCH,
keys_only=True)
ndb.delete_multi(results)
if not more:
break
# Analyze all the intake users, on a per-organization basis
if org_key:
org_keys = [org_key]
else:
org_keys = Organization.query().iter(keys_only=True)
for org_key in org_keys:
counter_key = "analysis::run_count::%s" % org_key.urlsafe()
memcache.set(key=counter_key, value=0)
iu_query = IntakeUser.query(IntakeUser.org == org_key)
for iu_key in iu_query.iter(keys_only=True):
memcache.incr(counter_key)
#deferred.defer(analyze_user, intake_user_key=iu_key)
analyze_user(iu_key)
generate_csv(org_key.get().name)
return "Great Success"
def incr_counter(key, url, delta=1, namespace=None):
'''
用memcache缓存计数
@type key: str
@param key: 用作memcache的key
@type url: str
@param url: 用作taskqueue的URL
@type delta: int
@param delta: 用作memcache.incr()的delta
@type namespace: str
@param namespace: 用作memcache的namespace
'''
try:
memcache.incr(key, delta, namespace, 0)
taskqueue.add(queue_name='counter', url=url, params={'key': key}, countdown=COUNTER_TASK_DELAY)
except:
pass
def current_rate(entity, limit, duration):
key = "ratelimit:{}:{}".format(int(time.time() / duration), entity)
value = memcache.incr(key, initial_value=0)
if value > limit:
logging.info(
"RateLimitDenied({!r}, value={!r}, limit={!r}, duration={!r})"
.format(entity, value, limit, duration))
else:
logging.info(
"RateLimitAllowed({!r}, value={!r}, limit={!r}, duration={!r})"
.format(entity, value, limit, duration))
return value
def process_days(self, today):
logging.info('Processing day %s', today.isoformat())
starttimestamp = calendar.timegm(
(today.year, today.month, today.day, 0, 0, 0))
endtimestamp = starttimestamp + 24 * 60 * 60
base_key = today.isoformat()
count = 0
for item in dbmodel.ReportItem.all().filter('counted =', None).filter(
'eventtype =',
'Information').filter('timestamp <', endtimestamp).filter(
'timestamp >=', starttimestamp).order('timestamp'):
self.update_day_record(base_key, starttimestamp, item)
count += 1
if count > 0:
memcache.incr('aggregate-day', initial_value=0)
logging.info('Processed %s day items for %s', count, today.isoformat())
return count
def incrCount(counter_name, num_shards=20, amount=1):
"""Increments the given counter by one.
See getCount() for example of the counter_names."""
def _tx():
if num_shards == 1:
index = 0
else:
index = random.randint(0, num_shards - 1)
shard_name = counter_name+":"+str(index)
counter = model.forum.ForumShardedCounter.get_by_key_name(shard_name)
if not counter:
counter = model.forum.ForumShardedCounter(key_name=shard_name,
name=counter_name)
counter.count += amount
counter.put()
db.run_in_transaction(_tx)
keyname = "counter:%s" % (counter_name)
memcache.incr(keyname)
def _increment_index(self, last_index):
"""Moves the work index forward and waits for all writers.
Args:
last_index: The last index that was used for the reader/writer lock.
Returns:
True if all writers were definitely finished; False if the reader/writer
lock timed out and we are proceeding anyways.
"""
# Increment the batch index counter so incoming jobs will use a new index.
# Don't bother setting an initial value here because next_index() will
# do this when it notices no current index is present. Do this *before*
# closing the reader/writer lock below to decrease active writers on the
# current index.
# We do this even in the case that batch_period_ms was zero, just in case
# that memcache operation failed for some reason, we'd rather have more
# batches then have the work index pipeline stall.
memcache.incr(self.index_name)
# Prevent new writers by making the counter extremely negative. If the
# decrement fails here we can't recover anyways, so just let the worker go.
add_counter = self.add_counter_template % last_index
memcache.decr(add_counter, self.LOCK_OFFSET)
for i in xrange(self.sync_attempts):
counter = memcache.get(add_counter)
# Less than or equal LOCK_OFFSET here in case a writer decrements twice
# due to rerunning failure tasks.
if counter is None or int(counter) <= self.LOCK_OFFSET:
# Worst-case the counter will be gone due to memcache eviction, which
# means the worker can procede with without waiting for writers
# and just process whatever it can find. This may drop some work.
return True
time.sleep(self.sync_timeout)
else:
logging.critical('Worker for %s gave up waiting for writers',
self.name)
return False
def counter():
if 'reset' in request.args:
memcache.flush_all()
# set up names if it doesn't exist
memcache.add('names','')
name = "Franciso"
if request.method == 'GET' and 'person_name' in request.args:
name = request.args['person_name']
# Check to see if that name is in names
names = memcache.get('names')
if name in names.split('/'):
# that person is already there
memcache.incr(name)
else:
memcache.set('names', names + '/' + name)
memcache.set(name,1)
names = memcache.get('names').split('/')
counts = []
for n in names:
counts.append( (n, memcache.get(n) ) )
#counts = [("bob",5),("mary",6)]
return render_template('counter.html', counts=counts, name=name)
