def bruteforce(ident):
# hash the identifier to prevent control character problems
prefix = '%s_%s' % (BRUTEFORCE_PREFIX, hashlib.md5(ident).hexdigest())
# create cache keys for each of the last n minutes
cache_keys = []
dt = datetime.datetime.now()
count = 0
while count < BRUTEFORCE_MINUTES:
cache_keys.append('%s_%s:%s' % (prefix, dt.hour, dt.minute))
dt -= datetime.timedelta(seconds=60)
count +=1
# increment count for this minute
try:
cache.incr(cache_keys[0])
except:
cache.set(cache_keys[0], 1, (60 * (BRUTEFORCE_MINUTES + 1)))
# collect count from cache
attempt_count = 0
attempt_dict = cache.get_many(cache_keys)
for v in attempt_dict.itervalues():
attempt_count += v
# if we detect multiple attempts, sleep
sleep(min(
BRUTEFORCE_MAX_PAUSE,
max(0, attempt_count - BRUTEFORCE_ALLOWED_ATTEMPTS)
))
def _update_submission_count_for_today(
form_id: int, incr: bool = True, date_created=None):
# Track submissions made today
current_timzone_name = timezone.get_current_timezone_name()
current_timezone = pytz.timezone(current_timzone_name)
today = datetime.today()
current_date = current_timezone.localize(
datetime(today.year, today.month, today.day)).isoformat()
date_cache_key = (f"{XFORM_SUBMISSION_COUNT_FOR_DAY_DATE}" f"{form_id}")
count_cache_key = (f"{XFORM_SUBMISSION_COUNT_FOR_DAY}{form_id}")
if not cache.get(date_cache_key) == current_date:
cache.set(date_cache_key, current_date, 86400)
if date_created:
date_created = current_timezone.localize(
datetime(date_created.year, date_created.month, date_created.day)
).isoformat()
current_count = cache.get(count_cache_key)
if not current_count and incr:
cache.set(count_cache_key, 1, 86400)
elif incr:
cache.incr(count_cache_key)
elif current_count and current_count > 0 and date_created == current_date:
cache.decr(count_cache_key)
def set_results_status(self, test_result):
"""
Sets params for the results data for this workshop
:param test_result:
"""
workshop_id = test_result.submission.workshop_id
problem_id = test_result.test.problem_id
test_id = test_result.test_id
status = test_result.result_data["status"]["id"]
if status == Judge0Status.ACCEPTED.value:
cache.incr(
f'workshop_{workshop_id}_problem_{problem_id}_test_{test_id}_users_passed',
ignore_key_check=True),
elif status == Judge0Status.WRONG_ANSWER.value:
cache.incr(
f'workshop_{workshop_id}_problem_{problem_id}_test_{test_id}_users_wrong_count',
ignore_key_check=True),
elif status == Judge0Status.TIMEOUT.value:
cache.incr(
f'workshop_{workshop_id}_problem_{problem_id}_test_{test_id}_users_time_count',
ignore_key_check=True),
elif status == Judge0Status.COMPILATION_ERR.value:
cache.incr(
f'workshop_{workshop_id}_problem_{problem_id}_test_{test_id}_users_compilation_count',
ignore_key_check=True),
elif status in (Judge0Status.RUNTIME_NZEC.value,
Judge0Status.RUNTIME_OTHER.value,
Judge0Status.RUNTIME_SIGABRT.value,
Judge0Status.RUNTIME_SIGFPE.value,
Judge0Status.RUNTIME_SIGSEGV.value,
Judge0Status.RUNTIME_SIGXFSZ.value):
cache.incr(
f'workshop_{workshop_id}_problem_{problem_id}_test_{test_id}_users_runtime_count',
ignore_key_check=True),
def middleware(request):
user = request.user
if settings.DEBUG:
return get_response(request)
if user.is_anonymous:
oip = get_ip(request)
ips = oip.split(".")[:-1]
ip = ".".join(ips)
if ip in settings.IP_WHITELIST:
return get_response(request)
if ip not in cache:
cache.set(ip, 0, settings.TIME_PERIOD)
value = cache.get(ip)
if value >= settings.MAX_VISITS:
# Raise redirect exception
if domain_is_whitelisted(oip):
cache.set(ip, 0)
else:
now = util.now()
message = f"{now}\tbanned\t{ip}\t{oip}\n"
logger.error(message)
fp = open(settings.BANNED_IPS, "a")
fp.write(message)
fp.close()
return redirect('/static/message.txt')
else:
cache.incr(ip)
return get_response(request)
def inner(*args, **kwargs):
t0 = time.time()
result = method(*args, **kwargs)
if not (isinstance(result, tuple) and len(result) == 2):
# happens when fetch() is used recursively
return result
result, hit_or_miss = result
if not getattr(settings, 'ANALYZE_MODEL_FETCHES', False):
return result
t1 = time.time()
self = args[0]
msecs = int((t1 - t0) * 1000)
hit_or_miss = 'HIT' if hit_or_miss else 'MISS'
try:
value = self.__class__.__name__
key = 'all_classes'
all_ = cache.get(key) or []
if value not in all_:
all_.append(value)
cache.set(key, all_, 60 * 60 * 24)
valuekey = hashlib.md5(value.encode('utf-8')).hexdigest()
for prefix, incr in (('times', msecs), ('uses', 1)):
key = '%s_%s_%s' % (prefix, hit_or_miss, valuekey)
try:
cache.incr(key, incr)
except ValueError:
cache.set(key, incr, 60 * 60 * 24)
except Exception:
logger.error('Unable to collect model fetches data', exc_info=True)
finally:
return result
def decorated_login(request, *args, **kwargs):
""" Decorated login """
response = func(request, *args, **kwargs)
if request.method == 'POST':
login_unsuccessful = (
response and
not response.has_header('location') and
response.status_code != 302
)
if request.POST['username']:
username = request.POST['username']
key = 'police_' + username
attemps = cache.get(key, 0)
if attemps >= POLICE_FAILURE_ATTEMPTS:
logout(request)
return render_to_response(
'police/blocked.html',
{'username': username},
context_instance=RequestContext(request)
)
if attemps == 0 and login_unsuccessful:
cache.set(key, 1, POLICE_BLOCK_TIME)
elif attemps < POLICE_FAILURE_ATTEMPTS and login_unsuccessful:
cache.incr(key)
return response
def incr_login_failed_attempts(username=None, ip=None):
"""Increase login failed attempts by 1 for both username and ip.
Arguments:
- `username`:
- `ip`:
Returns new value of failed attempts.
"""
timeout = settings.LOGIN_ATTEMPT_TIMEOUT
username_attempts = 1
ip_attempts = 1
if username:
cache_key = normalize_cache_key(username, prefix=LOGIN_ATTEMPT_PREFIX)
try:
username_attempts = cache.incr(cache_key)
except ValueError:
cache.set(cache_key, 1, timeout)
if ip:
cache_key = normalize_cache_key(ip, prefix=LOGIN_ATTEMPT_PREFIX)
try:
ip_attempts = cache.incr(cache_key)
except ValueError:
cache.set(cache_key, 1, timeout)
return max(username_attempts, ip_attempts)
def receive_data_chunk(self, raw_data, start):
'''
Receives a "chunk" of data from the file upload.
``raw_data is`` a byte string containing the uploaded data.
``start`` is the position in the file where this raw_data chunk begins.
The data you return will get fed into the subsequent upload handlers'
receive_data_chunk methods. In this way, one handler can be a "filter"
for other handlers.
Return None from receive_data_chunk to sort-circuit remaining upload
handlers from getting this chunk.. This is useful if you're storing
the uploaded data yourself and don't want future handlers to store
a copy of the data.
If you raise a StopUpload or a SkipFile exception, the upload will
abort or the file will be completely skipped.
'''
self.progress += self.chunk_size
if self.cach_key:
try:
percent = min(100, int(100 * self.progress / self.length))
cache.incr(self.cach_key, percent)
if settings.DEBUG:
logging.debug('uploaded proceeded for %s and filename %s @ %s'
% (self.cach_key, self.name, ctime.ctime()))
except ValueError, e:
logging.error('Tried to increment a non-existing cache-key;\
%s %s' % (self.cach_key, e))
def set(self, data):
if self.count():
cache.incr(self.counter_key)
else:
# First setting of the task, write a celery task to post
# this in DELAYED_MAILER_WAIT seconds.
#
# We force a timeout so that if the original post fails and
# never goes out, eventually the cache will clear again and
# we've lost a few errors.
time = getattr(settings, 'DELAYED_MAILER_WAIT', 60)
cache.set_many({
self.data_key: data,
self.counter_key: 1
},
timeout=time * 2)
# If memcached didn't get that, we can't do the async, it would
# be nice if this didn't just fail silently.
if not self.count():
self.mail(1, data)
try:
delayed_send.apply_async([self.hash], countdown=time)
except socket.error:
# If the celery backend is down, we can't queue so just
# send.
self.send()
def clear_permission_cache():
version = get_cache_version()
if version > 1:
cache.incr(get_cache_version_key())
else:
cache.set(get_cache_version_key(), 2,
get_cms_setting('CACHE_DURATIONS')['permissions'])
def login(request):
if request.method == "POST":
if cache.get('loginAttempts') != None and cache.get(
'loginAttempts') > 3:
cache.set('loginAttempts', cache.get('loginAttempts'), 600)
context = {
"message":
'Your account has been temporarily locked out because of too many failed login attempts.'
}
return render(request, "login.html", context)
username = request.POST['username'].replace(" ", "")
password = request.POST['password']
if not request.POST.get('remember_me', None):
request.session.set_expiry(0)
user = authenticate(username=username, password=password)
if user:
auth_login(request, user)
return redirect('book:mainpage')
else:
if cache.get('loginAttempts') == None:
cache.set('loginAttempts', 1)
else:
cache.incr('loginAttempts', 1)
context = {
"message": "Username or Password did not match!",
"username": username
}
return render(request, "login.html", context)
return render(request, "login.html", {})
def fetch(self, request, key=None):
cross_origin = request.META.get('HTTP_ORIGIN')
if cross_origin is not None:
try:
if request.META['HTTP_HOST'] == urlparse(
cross_origin).hostname:
cross_origin = None
except ValueError:
pass
counter_key = 'api_updates_fetch_requests%s' % (
'_cross_origin' if cross_origin is not None else '')
try:
cache.incr(counter_key)
except ValueError:
cache.set(counter_key, 0, None)
from c3nav.site.models import SiteUpdate
result = {
'last_site_update': SiteUpdate.last_update(),
'last_map_update': MapUpdate.current_processed_cache_key(),
}
if cross_origin is None:
result.update({
'user': get_user_data(request),
})
response = Response(result)
if cross_origin is not None:
response['Access-Control-Allow-Origin'] = cross_origin
response['Access-Control-Allow-Credentials'] = 'true'
set_tile_access_cookie(request, response)
return response
def cheungssh_login(request):
info = {"msgtype": "ERR", "content": "", "auth": "no"}
logintime = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time()))
client_ip = request.META['REMOTE_ADDR']
limit_ip = 'fail.limit.%s' % (client_ip)
ip_threshold_r = cache.get('ip.threshold')
ip_threshold = lambda x: x if x is not None else 4
ip_threshold = ip_threshold(ip_threshold_r)
if cache.has_key(limit_ip):
if cache.get(limit_ip) > ip_threshold:
info['content'] = "无效登陆"
cache.incr(limit_ip)
cache.expire(limit_ip, 8640000)
info = json.dumps(info)
return HttpResponse(info)
if request.method == "POST":
username = request.POST.get("username", '非法用户名')
password = request.POST.get("password", False)
print username, password, request.POST
user = authenticate(username=username, password=password)
if user is not None:
if user.is_active:
print "成功登陆"
login(request, user)
request.session["username"] = username
info["msgtype"] = "OK"
info['auth'] = "yes"
info['content'] = "成功登录"
request.session.set_expiry(0)
if cache.has_key(limit_ip): cache.delete(limit_ip)
print request.COOKIES, request.session.keys(
), request.session['_auth_user_id']
info['sid'] = str(request.session.session_key)
else:
info["content"] = "用户状态无效"
print info["content"]
else:
if cache.has_key(limit_ip):
cache.incr(limit_ip)
else:
cache.set(limit_ip, 1, 3600)
info["content"] = "用户名或密码错误"
info["IP"] = client_ip
info["IP-Locate"] = IP.find(client_ip)
info["username"] = username
info["logintime"] = logintime
redis_to_redis.set_redis_data(
'sign.record',
json.dumps(info, encoding='utf-8', ensure_ascii=False))
else:
info["content"] = "No Get"
info = json.dumps(info, encoding="utf-8", ensure_ascii=False)
response = HttpResponse(info)
response["Access-Control-Allow-Origin"] = "*"
response["Access-Control-Allow-Methods"] = "POST"
response["Access-Control-Allow-Credentials"] = "true"
return response
def _incr_login_failed_attempts(username=None, ip=None):
"""Increase login failed attempts by 1 for both username and ip.
Arguments:
- `username`:
- `ip`:
Returns new value of failed attempts.
"""
timeout = settings.LOGIN_ATTEMPT_TIMEOUT
username_attempts = 1
ip_attempts = 1
if username:
try:
username_attempts = cache.incr(LOGIN_ATTEMPT_PREFIX + username)
except ValueError:
cache.set(LOGIN_ATTEMPT_PREFIX + username, 1, timeout)
if ip:
try:
ip_attempts = cache.incr(LOGIN_ATTEMPT_PREFIX + ip)
except ValueError:
cache.set(LOGIN_ATTEMPT_PREFIX + ip, 1, timeout)
return max(username_attempts, ip_attempts)
def filter(self, record):
from django.conf import settings
from django.core.cache import cache
# Rate is specified as 1 messages logged per N seconds. (aka cache timeout)
rate = getattr(settings, 'RATE_LIMITER_FILTER_RATE', 10)
prefix = getattr(settings, 'RATE_LIMITER_FILTER_PREFIX',
'ratelimiterfilter')
subject = record.getMessage()
cache_key = "%s:%s" % (prefix, md5(subject).hexdigest())
cache_count_key = "%s:count" % cache_key
result = cache.get_many([cache_key, cache_count_key])
value = result.get(cache_key)
cntr = result.get(cache_count_key)
if not cntr:
cntr = 1
cache.set(cache_count_key, cntr, rate + 60)
if value:
cache.incr(cache_count_key)
return False
record.msg = "[%sx] %s" % (cntr, record.msg)
cache.set(cache_key, time.time(), rate)
return True
def submit(request):
"""Accept submission of a demo"""
if not request.user.is_authenticated():
return jingo.render(request, 'demos/submit_noauth.html', {})
if request.method != "POST":
form = SubmissionNewForm()
else:
form = SubmissionNewForm(request.POST, request.FILES)
if form.is_valid():
new_sub = form.save(commit=False)
if request.user.is_authenticated():
new_sub.creator = request.user
new_sub.save()
ns_key = cache.get(DEMOS_CACHE_NS_KEY)
if ns_key is None:
ns_key = random.randint(1,10000)
cache.set(DEMOS_CACHE_NS_KEY, ns_key)
else:
cache.incr(DEMOS_CACHE_NS_KEY)
# TODO: Process in a cronjob?
new_sub.process_demo_package()
return HttpResponseRedirect(reverse(
'demos.views.detail', args=(new_sub.slug,)))
return jingo.render(request, 'demos/submit.html', {'form': form})
def test_incr(self):
cache.set("num", 1)
cache.incr("num")
res = cache.get("num")
assert res == 2
cache.incr("num", 10)
res = cache.get("num")
assert res == 12
# max 64 bit signed int
cache.set("num", 9223372036854775807)
cache.incr("num")
res = cache.get("num")
assert res == 9223372036854775808
cache.incr("num", 2)
res = cache.get("num")
assert res == 9223372036854775810
cache.set("num", 3)
cache.incr("num", 2)
res = cache.get("num")
assert res == 5
def ticket_service_view(request):
if request.method == 'POST':
method, data = 'post', request.POST
else:
method, data = 'get', request.GET
client_ip = request.META.get('REMOTE_ADDR')
logger.info("Request from {}".format(client_ip))
matched_ips = cache.keys('{}_status'.format(client_ip))
if len(matched_ips) > 0:
if cache.get('{}_status'.format(client_ip)) == 'w':
return call_third_party(method, data)
else:
return JsonResponse({'detail': 'Permission Denied.'})
else:
g_recaptcha_response = request.POST.get('g-recaptcha-response', [])
if g_recaptcha_response != '' and check_captcha(g_recaptcha_response, client_ip):
cache.set('{}_status'.format(client_ip), 'w')
return call_third_party(method, data)
else:
unathorised_request_num = cache.get_or_set('{}_unathorised_request_num'.format(client_ip), 0)
if unathorised_request_num < settings.MAX_UNATHORISED_REQUEST_NUM:
cache.incr('{}_unathorised_request_num'.format(client_ip))
template = loader.get_template('ticket_template.html')
context = {
'site_key': settings.RECAPTCHA_SITEKEY,
'data': data
}
return HttpResponse(template.render(context, request))
return HttpResponse('You are not allowed to access this service anymore.')
def release(self, value: int=1):
"""
Increment semaphore value.
:param value: Number of values to release.
"""
cache.incr(self._cache_key, value)
def view_proxy(request, promo_id, hash):
"""Track a view of a promotion and redirect to the image."""
promo = get_object_or_404(SupporterPromo, pk=promo_id)
if not promo.image:
raise Http404('No image defined for this promo.')
count = cache.get(promo.cache_key(type=VIEWS, hash=hash), None)
if count is None:
log.warning('Old or nonexistent hash tried on View.')
elif count == 0:
promo.incr(VIEWS)
cache.incr(promo.cache_key(type=VIEWS, hash=hash))
project_slug = cache.get(
promo.cache_key(type='project', hash=hash),
None
)
if project_slug:
project = Project.objects.get(slug=project_slug)
promo.incr(VIEWS, project=project)
else:
agent = request.META.get('HTTP_USER_AGENT', 'Unknown')
log.warning(
'Duplicate view logged. {count} total views tried. User Agent: [{agent}]'.format(
count=count, agent=agent
)
)
cache.incr(promo.cache_key(type=VIEWS, hash=hash))
raise Http404('Invalid click. This has been logged.')
return redirect(promo.image)
def log_symbol_cache_hit(self, cache_key):
try:
cache.incr(cache_key)
except ValueError:
# If it wasn't in cache we can't increment this
# hit, so we have to start from 1.
cache.set(cache_key, 1, timeout=self.log_cache_timeout)
def save_messages(self, msgs):
msg_ids = ['.'.join(msg['id']) for msg in msgs]
cache.set('validation.job_id:%s' % self.job_id, msg_ids)
for msg, key in zip(msgs, msg_ids):
if isinstance(msg['description'], list):
des = []
for _m in msg['description']:
if isinstance(_m, list):
for x in _m:
des.append(x)
else:
des.append(_m)
des = '; '.join(des)
else:
des = msg['description']
cache.set(
'validation.msg_key:' + key, {
'long_message': des,
'message': msg['message'],
'type': msg.get('compatibility_type', msg.get('type'))
})
aa = ('validation.job_id:%s.msg_key:%s:addons_affected' %
(self.job_id, key))
try:
cache.incr(aa)
except ValueError:
cache.set(aa, 1)
def inner(*args, **kwargs):
t0 = time.time()
result = method(*args, **kwargs)
if not (isinstance(result, tuple) and len(result) == 2):
# happens when fetch() is used recursively
return result
result, hit_or_miss = result
if not getattr(settings, 'ANALYZE_MODEL_FETCHES', False):
return result
t1 = time.time()
self = args[0]
url = args[1]
msecs = int((t1 - t0) * 1000)
hit_or_miss = 'HIT' if hit_or_miss else 'MISS'
try:
groups = (('classes', self.__class__.__name__), ('urls', url))
for value_type, value in groups:
key = 'all_%s' % value_type
all = cache.get(key) or []
if value not in all:
all.append(value)
cache.set(key, all, 60 * 60 * 24)
for prefix, incr in (('times', msecs), ('uses', 1)):
key = '%s_%s_%s' % (prefix, hit_or_miss, value)
# memcache is max 250 but raises warnings >240
key = key[:240]
try:
cache.incr(key, incr)
except ValueError:
cache.set(key, incr, 60 * 60 * 24)
except Exception:
logger.error('Unable to collect model fetches data', exc_info=True)
finally:
return result
def update_article_hits(article):
"""更新文章点击次数"""
# 指定key不存在则从数据库hits字段添加，并设置永不过期
cache.add("article:{}:views".format(article.id), article.hits, timeout=None)
# 指定key的值自增1
cache.incr("article:{}:views".format(article.id))
def get(self, request, *args, **kwargs):
"""
Rewrite of the get function to add the similar posts
to the render function.
Similar functions by Tag and ordered by Count and Date
"""
# Check to see if the search query is being called
form = self.form_class(self.request.GET)
if form.is_valid():
form_sub = '&'.join(map(lambda x: '='.join(x), self.request.GET.items()))
redirect_url = reverse('post_list') + '?' + form_sub
return redirect(redirect_url)
self.object = self.get_object()
# Get the views and increment, if not available create it
post_cache_key = 'blog-post-{}'.format(self.object.id)
post_views_count = cache.get(post_cache_key)
if post_views_count:
cache.incr(post_cache_key, 1)
else:
post_views_count = 1
cache.add(post_cache_key, post_views_count)
# get post tags
post_tags = self.object.tags.values_list('id', flat=True)
# get similar posts by tag
similar_posts = (Post.published.filter(tags__in=post_tags)
.exclude(id=self.object.id))
similar_posts = (similar_posts.annotate(same_tags=Count('tags'))
.order_by('-same_tags', '-publish'))[:4]
post_comments = Comment.objects.filter(post=self.object)
context = self.get_context_data(object=self.object)
context['post_comments'] = post_comments
context['similar_posts'] = similar_posts
context['post_views_count'] = post_views_count
return self.render_to_response(context)
def acquire_lock(lock_name):
"""
A contextmanager to wait until an exclusive lock is available,
hold the lock and then release it when the code under context
is complete.
TODO: This code doesn't work like it should. It doesn't
wait indefinitely for the lock and in fact cycles through
very quickly.
"""
for _ in range(10):
try:
value = cache.incr(lock_name)
except ValueError:
cache.set(lock_name, 0)
value = cache.incr(lock_name)
if value == 1:
break
else:
cache.decr(lock_name)
else:
yield
cache.set(lock_name, 0)
return
yield
cache.decr(lock_name)
def detect_troll_pre_request_creation(request, **kwargs):
user = kwargs['user']
if user.trusted():
return kwargs
ip_address = request.META['REMOTE_ADDR']
cache_key = 'froide:foirequest:request_per_ip:%s' % ip_address
count = cache.get(cache_key, 0)
if count == 0:
cache.set(cache_key, 1)
else:
try:
cache.incr(cache_key)
except ValueError:
pass
count += 1
if user.is_blocked:
kwargs['blocked'] = True
return kwargs
now = timezone.now()
diff = now - user.date_joined
if (diff < timedelta(days=1) and count > 5) or count > 15:
user.is_blocked = True
user.save()
mail_managers(_('User blocked'), user.pk)
kwargs['blocked'] = True
return kwargs
def monitor_abuse(self, ip):
"""
Track the number of hits per second for a given IP.
If the count is over ABUSE_THRESHOLD block user
"""
cache_key = self.ABUSE_PREFIX + ip
abuse_count = cache.get(cache_key)
if self.DEBUG:
print >> sys.stderr, "BANISH ABUSE COUNT: ", abuse_count
print >> sys.stderr, "BANISH CACHE KEY: ", cache_key
over_abuse_limit = False
if not abuse_count:
cache.set(cache_key, 1, 60)
else:
if abuse_count >= self.ABUSE_THRESHOLD:
over_abuse_limit = True
# Store IP Abuse in memcache and database
ban = Banishment(
ban_reason="IP Abuse limit exceeded",
type="ip-address",
condition=ip,
)
ban.save()
cache.set(self.BANISH_PREFIX + ip, "1")
cache.incr(cache_key)
return over_abuse_limit
def detect_troll_pre_request_creation(request, **kwargs):
user = kwargs['user']
if user.trusted():
return kwargs
ip_address = request.META['REMOTE_ADDR']
cache_key = 'froide:foirequest:request_per_ip:%s' % ip_address
count = cache.get(cache_key, 0)
if count == 0:
cache.set(cache_key, 1)
else:
try:
cache.incr(cache_key)
except ValueError:
pass
count += 1
if user.is_blocked:
kwargs['blocked'] = True
return kwargs
now = timezone.now()
diff = now - user.date_joined
if (diff < timedelta(days=1) and count > 10):
user.is_blocked = True
user.save()
mail_managers(_('User auto blocked'), str(user.pk))
kwargs['blocked'] = True
return kwargs
def __call__(self, *args, **kwargs):
# influenced by http://charlesleifer.com/blog/django-patterns-view-decorators/
request = args[0]
remote_addr = request.META.get('REMOTE_ADDR')
key = 'throttled_count_%s.%s' % (remote_addr, request.get_full_path())
key_date = 'throttled_date_%s.%s' % (remote_addr,
request.get_full_path())
key_date_value = cache.get(key_date)
if key_date_value and key_date_value < datetime.datetime.now():
cache.delete_many([key, key_date])
allow = True
view_count = cache.get(key, '')
if view_count:
if view_count >= settings.DEFAULT_MAX_REQUESTS_PER_INTERVAL:
allow = False
else:
allow = True
cache.incr(key)
else:
cache.set(key, 1, settings.DEFAULT_CACHE_RATE_LIMIT)
allow = True
if not allow:
future = datetime.datetime.now() + datetime.timedelta(
seconds=settings.DEFAULT_CACHE_RATE_LIMIT)
cache.set(key_date, future)
response = HttpResponse("throttled",
status=403,
content_type="text/plain")
response['Content-Disposition'] = 'inline; filename=test.txt'
return response
return self.orig_func(*args, **kwargs)
def delete(self, *args, **kwargs):
old_id = self.id
super(XHTML, self).delete(*args, **kwargs)
try:
cache.incr('_widget_xhtml_version/%s' % old_id)
except ValueError:
pass
def newtopic(request):
grouptitle = request.session.get('group', False)
group = Group.objects.get(title = grouptitle)
# group = False
if request.method == 'POST':
form = TopicForm(request.POST)
if form.is_valid():
content = form.cleaned_data['content']
title = form.cleaned_data['title']
new_topic = Topic()
new_topic.content = content
new_topic.title = title
new_topic.writer = request.user
new_topic.group = group
new_topic.save()
group.topicount += 1
group.save()
cachekey = "group_topic_count_" + str(group.id)
if cache.get(cachekey) != None:
cache.incr(cachekey)
else:
group = Group.objects.get(id=group.id)
cache.set(cachekey, group.topicount)
return redirect(request.session['lastpage'])
else:
messages.error(request, '您输入的话题内容有误,请改正！')
return redirect("newtopic")
else:
print request.user
context = {
'myform': TopicForm,
'group': group,
}
return render(request, 'newtopic.html', context)
def inner(*args, **kwargs):
t0 = time.time()
result = method(*args, **kwargs)
if not (isinstance(result, tuple) and len(result) == 2):
# happens when fetch() is used recursively
return result
result, hit_or_miss = result
if not getattr(settings, 'ANALYZE_MODEL_FETCHES', False):
return result
t1 = time.time()
self = args[0]
msecs = int((t1 - t0) * 1000)
hit_or_miss = 'HIT' if hit_or_miss else 'MISS'
try:
value = self.__class__.__name__
key = 'all_classes'
all_ = cache.get(key) or []
if value not in all_:
all_.append(value)
cache.set(key, all_, 60 * 60 * 24)
valuekey = hashlib.md5(value.encode('utf-8')).hexdigest()
for prefix, incr in (('times', msecs), ('uses', 1)):
key = '%s_%s_%s' % (prefix, hit_or_miss, valuekey)
try:
cache.incr(key, incr)
except ValueError:
cache.set(key, incr, 60 * 60 * 24)
except Exception:
logger.error('Unable to collect model fetches data', exc_info=True)
finally:
return result
def collectioninvestment(request):
try:
investmentid = request.POST.get('investmentid')
investment = Investment.objects.get(pk=investmentid)
user = request.user
except investment.DoesNotExist:
raise Http404("investment does not exist")
collection = CollectionInvestment.objects.filter(investment=investment, user=user)
cachekey = "investment_collection_" + str(investmentid)
if collection:
collection.delete()
collecicon = '收藏'
collection_icon = 'glyphicon-star-empty'
if cache.get(cachekey) != None:
cache.decr(cachekey)
else:
cache.set(cachekey, investment.collectioninvestment_set.count(), 1209600)
else:
c = CollectionInvestment(user=user, investment=investment)
c.save()
collecicon = '已收藏'
collection_icon = 'glyphicon-star'
if cache.get(cachekey) != None:
cache.incr(cachekey)
else:
cache.set(cachekey, investment.collectioninvestment_set.count(), 1209600)
data = {
'collecicon': collecicon,
'collection_icon': collection_icon,
'collectioncount': cache.get(cachekey),
}
json_data = json.dumps(data)
return HttpResponse(json_data, content_type='application/json')
def __call__(self, *args, **kwargs):
# influenced by http://charlesleifer.com/blog/django-patterns-view-decorators/
request = args[0]
remote_addr = request.META.get('REMOTE_ADDR')
key = 'throttled_count_%s.%s' % (remote_addr, request.get_full_path())
key_date = 'throttled_date_%s.%s' % (remote_addr, request.get_full_path())
key_date_value = cache.get(key_date)
if key_date_value and key_date_value < datetime.datetime.now():
cache.delete_many([key, key_date])
allow = True
view_count = cache.get(key, '')
if view_count:
if view_count >= settings.DEFAULT_MAX_REQUESTS_PER_INTERVAL:
allow = False
else:
allow = True
cache.incr(key)
else:
cache.set(key, 1, settings.DEFAULT_CACHE_RATE_LIMIT)
allow = True
if not allow:
future = datetime.datetime.now() + datetime.timedelta(seconds=settings.DEFAULT_CACHE_RATE_LIMIT)
cache.set(key_date, future)
response = HttpResponse("throttled", status=403, content_type="text/plain")
response['Content-Disposition'] = 'inline; filename=test.txt'
return response
return self.orig_func(*args, **kwargs)
def collectioncompany(request):
try:
companyid = request.POST.get('companyid')
company = Company.objects.get(pk=companyid)
user = request.user
except Company.DoesNotExist:
raise Http404("Company does not exist")
collection = CollectionCompany.objects.filter(company=company, user=user)
cachekey = "company_collection_" + str(companyid)
if collection:
collection.delete()
collecicon = '收藏'
collection_icon = 'glyphicon-star-empty'
if cache.get(cachekey) != None:
cache.decr(cachekey)
else:
cache.set(cachekey, company.collectioncompany_set.count(), 1209600)
else:
c = CollectionCompany(user=user, company=company)
c.save()
collecicon = '已收藏'
collection_icon = 'glyphicon-star'
if cache.get(cachekey) != None:
cache.incr(cachekey)
else:
cache.set(cachekey, company.collectioncompany_set.count(), 1209600)
data = {
'collecicon': collecicon,
'collection_icon': collection_icon,
'collectioncount': cache.get(cachekey),
}
json_data = json.dumps(data)
return HttpResponse(json_data, content_type='application/json')
def login_failed(self, request, name, via, now, user=None):
"""
this function is called if a login failed
"""
if settings.THROTTLE_IP:
if request.META['REMOTE_ADDR'] not in settings.INTERNAL_IPS:
try:
cache.incr(self.key('ip', request.META['REMOTE_ADDR'], now), 1)
except ValueError:
cache.set(self.key('ip', request.META['REMOTE_ADDR'], now), 1, (settings.THROTTLE_IP_INTERVAL + 1) * 60)
if user:
logger.warning('Login failed for "%s" (#%s) via %s', name, user.pk, via)
if settings.THROTTLE_USER:
try:
cache.incr(self.key('user', user.pk, now), 1)
except ValueError:
cache.set(self.key('user', user.pk, now), 1, (settings.THROTTLE_USER_INTERVAL + 1) * 60)
# emit a signal if the login failed
cache_key = '%s-%s-%s' % (
settings.CACHE_PREFIX,
'loginfail',
user.pk,
)
if not settings.THROTTLE_SIGNAL_TIMEOUT or cache.get(cache_key):
login_failed.send(sender=self.__class__, user=user, seen=bool(settings.THROTTLE_SIGNAL_TIMEOUT))
else:
cache.set(cache_key, 1, settings.THROTTLE_SIGNAL_TIMEOUT * 60)
login_failed.send(sender=self.__class__, user=user, seen=False)
else:
logger.warning('Login failed for "%s" via %s', name, via)
def delete(self, *args, **kwargs):
from wirecloud.catalogue.utils import wgt_deployer
# Delete the related wiring information for that resource
WidgetWiring.objects.filter(idResource=self.id).delete()
if hasattr(self, 'widget'):
from wirecloud.platform.models import Widget
try:
self.widget.delete()
except Widget.DoesNotExist:
pass
# Delete media resources if needed
if not self.template_uri.startswith(('http', 'https')):
wgt_deployer.undeploy(self.vendor, self.short_name, self.version)
old_id = self.id
super(CatalogueResource, self).delete(*args, **kwargs)
# Remove cache for this resource
try:
cache.incr('_catalogue_resource_version/' + str(old_id))
except ValueError:
pass
def incr_login_failed_attempts(username=None, ip=None):
"""Increase login failed attempts by 1 for both username and ip.
Arguments:
- `username`:
- `ip`:
Returns new value of failed attempts.
"""
timeout = settings.LOGIN_ATTEMPT_TIMEOUT
username_attempts = 1
ip_attempts = 1
if username:
cache_key = normalize_cache_key(username, prefix=LOGIN_ATTEMPT_PREFIX)
try:
username_attempts = cache.incr(cache_key)
except ValueError:
cache.set(cache_key, 1, timeout)
if ip:
cache_key = normalize_cache_key(ip, prefix=LOGIN_ATTEMPT_PREFIX)
try:
ip_attempts = cache.incr(cache_key)
except ValueError:
cache.set(cache_key, 1, timeout)
return max(username_attempts, ip_attempts)
def click_proxy(request, promo_id, hash):
"""Track a click on a promotion and redirect to the link."""
promo = get_object_or_404(SupporterPromo, pk=promo_id)
count = cache.get(promo.cache_key(type=CLICKS, hash=hash), None)
if count is None:
log.warning('Old or nonexistent hash tried on Click.')
elif count == 0:
promo.incr(CLICKS)
cache.incr(promo.cache_key(type=CLICKS, hash=hash))
project_slug = cache.get(
promo.cache_key(type='project', hash=hash),
None
)
if project_slug:
project = Project.objects.get(slug=project_slug)
promo.incr(CLICKS, project=project)
else:
agent = request.META.get('HTTP_USER_AGENT', 'Unknown')
log.warning(
'Duplicate click logged. {count} total clicks tried. User Agent: [{agent}]'.format(
count=count, agent=agent
)
)
cache.incr(promo.cache_key(type=CLICKS, hash=hash))
raise Http404('Invalid click. This has been logged.')
return redirect(promo.link)
def test_incr(self):
""""Test the incr cache operation"""
cache.set("num", 1)
cache.incr("num")
res = cache.get("num")
self.assertEqual(res, 2)
cache.incr("num", 10)
res = cache.get("num")
self.assertEqual(res, 12)
# max 64 bit signed int
cache.set("num", 9223372036854775807)
cache.incr("num")
res = cache.get("num")
self.assertEqual(res, 9223372036854775808)
cache.incr("num", 2)
res = cache.get("num")
self.assertEqual(res, 9223372036854775810)
cache.set("num", 3)
cache.incr("num", 2)
res = cache.get("num")
self.assertEqual(res, 5)
def increment_counter(sender, comment, **kwargs):
if getattr(comment, "is_public", True):
key = get_cache_key_from_comment(comment)
try:
cache.incr(key)
except ValueError:
pass
def filter(self, record):
from django.conf import settings
from django.core.cache import cache
# Rate is specified as 1 messages logged per N seconds. (aka cache timeout)
rate = getattr(settings, 'RATE_LIMITER_FILTER_RATE', 10)
prefix = getattr(settings, 'RATE_LIMITER_FILTER_PREFIX', 'ratelimiterfilter')
subject = record.getMessage()
cache_key = "%s:%s" % (prefix, md5(subject).hexdigest())
cache_count_key = "%s:count" % cache_key
result = cache.get_many([cache_key, cache_count_key])
value = result.get(cache_key)
cntr = result.get(cache_count_key)
if not cntr:
cntr = 1
cache.set(cache_count_key, cntr, rate + 60)
if value:
cache.incr(cache_count_key)
return False
record.msg = "[%sx] %s" % (cntr, record.msg)
cache.set(cache_key, time.time(), rate)
return True
def pid_incr(self):
"""Increments section last post PID cache by 1."""
try:
return cache.incr(self.key)
except ValueError:
self.rebuild_cache()
return cache.incr(self.key)
def _incr_login_faied_attempts(username=None, ip=None):
"""Increase login failed attempts by 1 for both username and ip.
Arguments:
- `username`:
- `ip`:
Returns new value of failed attempts.
"""
timeout = settings.LOGIN_ATTEMPT_TIMEOUT
username_attempts = 1
ip_attempts = 1
if username:
try:
username_attempts = cache.incr(LOGIN_ATTEMPT_PREFIX + username)
except ValueError:
cache.set(LOGIN_ATTEMPT_PREFIX + username, 1, timeout)
if ip:
try:
ip_attempts = cache.incr(LOGIN_ATTEMPT_PREFIX + ip)
except ValueError:
cache.set(LOGIN_ATTEMPT_PREFIX + ip, 1, timeout)
return max(username_attempts, ip_attempts)
def save_messages(self, msgs):
msg_ids = ['.'.join(msg['id']) for msg in msgs]
cache.set('validation.job_id:%s' % self.job_id, msg_ids)
for msg, key in zip(msgs, msg_ids):
if isinstance(msg['description'], list):
des = []
for _m in msg['description']:
if isinstance(_m, list):
for x in _m:
des.append(x)
else:
des.append(_m)
des = '; '.join(des)
else:
des = msg['description']
cache.set('validation.msg_key:' + key,
{'long_message': des,
'message': msg['message'],
'type': msg.get('compatibility_type',
msg.get('type'))
})
aa = ('validation.job_id:%s.msg_key:%s:addons_affected'
% (self.job_id, key))
try:
cache.incr(aa)
except ValueError:
cache.set(aa, 1)
def inner(*args, **kwargs):
t0 = time.time()
result = method(*args, **kwargs)
if not (isinstance(result, tuple) and len(result) == 2):
# happens when fetch() is used recursively
return result
result, hit_or_miss = result
if not getattr(settings, 'ANALYZE_MODEL_FETCHES', False):
return result
t1 = time.time()
self = args[0]
url = args[1]
msecs = int((t1 - t0) * 1000)
hit_or_miss = 'HIT' if hit_or_miss else 'MISS'
try:
groups = (('classes', self.__class__.__name__), ('urls', url))
for value_type, value in groups:
key = 'all_%s' % value_type
all = cache.get(key) or []
if value not in all:
all.append(value)
cache.set(key, all, 60 * 60 * 24)
for prefix, incr in (('times', msecs), ('uses', 1)):
key = '%s_%s_%s' % (prefix, hit_or_miss, value)
# memcache is max 250 but raises warnings >240
key = key[:240]
try:
cache.incr(key, incr)
except ValueError:
cache.set(key, incr, 60 * 60 * 24)
except Exception:
logger.error('Unable to collect model fetches data', exc_info=True)
finally:
return result
def touch_and_return_is_pass(key):
"""
限制每秒 RATE_LIMIT_COUNT_PER_SECOND 次, Redis中key, value, TTL的變化
Key的組成 = 前綴:Key:目前秒數
key value TTL
rate_limit:xxx:0 1 1
rate_limit:xxx:0 2 0.8
rate_limit:xxx:0 3 0.7
rate_limit:xxx:0 4 0.6
rate_limit:xxx:0 5 0.2
rate_limit:xxx:1 1 1
"""
now_second = int(time.time() % 10)
key = f'rate_limit:{key}:{now_second}' # prefix:key:now_second
value = cache.get(key)
if value is not None:
# cache is set
if value >= RateLimitService.RATE_LIMIT_COUNT_PER_SECOND:
# exceed rate limit
return False
else:
# not exceed, increase count
cache.incr(key)
else:
# cache not set, touch and set expire 1 seconds
cache.set(key, 1, 1)
return True
def monitor_abuse(self, ip):
"""
Track the number of hits per second for a given IP.
If the count is over ABUSE_THRESHOLD block user
"""
cache_key = self.ABUSE_PREFIX + ip
abuse_count = cache.get(cache_key)
if self.DEBUG:
print >> sys.stderr, "BANISH ABUSE COUNT: ", abuse_count
print >> sys.stderr, "BANISH CACHE KEY: ", cache_key
over_abuse_limit = False
if not abuse_count:
cache.set(cache_key, 1, 60)
else:
if abuse_count >= self.ABUSE_THRESHOLD:
over_abuse_limit = True
# Store IP Abuse in memcache and database
ban = Banishment(
ban_reason="IP Abuse limit exceeded",
type="ip-address",
condition=ip,
)
ban.save()
cache.set(self.BANISH_PREFIX + ip, "1")
cache.incr(cache_key)
return over_abuse_limit
def stat_update_count(key, incr=1):
"""Increment a key on the Cache, for stats monitoring"""
if getattr(settings, 'EVE_PROXY_STATS', False) and len(getattr(settings, 'CACHES', {})):
try:
cache.incr(key, incr)
except ValueError:
cache.set(key, incr, 2592000)
def delete(self, *args, **kwargs):
old_id = self.id
super(XHTML, self).delete(*args, **kwargs)
try:
cache.incr('_widget_xhtml_version/' + str(old_id))
except ValueError:
pass
def save_messages(self, msgs):
msg_ids = [".".join(msg["id"]) for msg in msgs]
cache.set("validation.job_id:%s" % self.job_id, msg_ids)
for msg, key in zip(msgs, msg_ids):
if isinstance(msg["description"], list):
des = []
for _m in msg["description"]:
if isinstance(_m, list):
for x in _m:
des.append(x)
else:
des.append(_m)
des = "; ".join(des)
else:
des = msg["description"]
cache.set(
"validation.msg_key:" + key,
{
"long_message": des,
"message": msg["message"],
"type": msg.get("compatibility_type", msg.get("type")),
},
)
aa = "validation.job_id:%s.msg_key:%s:addons_affected" % (self.job_id, key)
try:
cache.incr(aa)
except ValueError:
cache.set(aa, 1)
def acquire_lock(lock_name):
"""
A contextmanager to wait until an exclusive lock is available,
hold the lock and then release it when the code under context
is complete.
TODO: This code doesn't work like it should. It doesn't
wait indefinitely for the lock and in fact cycles through
very quickly.
"""
for _ in range(10):
try:
value = cache.incr(lock_name)
except ValueError:
cache.set(lock_name, 0)
value = cache.incr(lock_name)
if value == 1:
break
else:
cache.decr(lock_name)
else:
yield
cache.set(lock_name, 0)
return
yield
cache.decr(lock_name)
def clear_permission_cache():
version = get_cache_version()
if version > 1:
cache.incr(get_cache_version_key())
else:
cache.set(get_cache_version_key(), 2,
get_cms_setting('CACHE_DURATIONS')['permissions'])
def atwho(text, sender, targetcomment, targetarticle, targetopic , targetproducts):
commmentdecode = text.decode("utf8")
pattern = re.compile(u'@([\u4e00-\u9fa5\w\-]+)')
results = pattern.findall(commmentdecode) #用正则把评论中有@的字符串分割开
userlist = []
for item in results:
try:
user = MyUser.objects.get(username = item.encode('utf8'))
except:
user = None
if user:
user = MyUser.objects.get(username = item.encode('utf8'))
notify.send(sender=sender, target_object=targetcomment
, recipient = user, verb="@"
, text=text, target_article = targetarticle
, target_products = targetproducts
, target_topic = targetopic)
cachekey = "user_unread_count" + str(user.id)
if cache.get(cachekey) != None:
cache.incr(cachekey)
else:
unread = Notification.objects.filter(recipient = user).filter(read = False).count()
cache.set(cachekey, unread, settings.CACHE_EXPIRETIME)
userlist.append(item.encode('utf8'))
return userlist
def invalidate_cache_group_id(group_code):
""" Invalidation of group is in fact only incrementation of group_id
"""
cache_group_key = '%s-%s-GROUP' % (settings.CACHE_MIDDLEWARE_KEY_PREFIX, group_code)
try:
cache.incr(cache_group_key)
except ValueError, e:
pass
def clear_permission_cache():
from django.core.cache import cache
version = get_cache_permission_version()
if version > 1:
cache.incr(get_cache_permission_version_key())
else:
cache.set(get_cache_permission_version_key(), 2,
get_cms_setting('CACHE_DURATIONS')['permissions'])
def set_auth(sessionid, message):
data = {
'sessionid': sessionid,
}
cache.incr(sessionid)
response = generate_response(constants.FIELD_ERROR, msg=message, data=data)
response.set_cookie(key='sessionid', value=sessionid)
return response
def cache_incr(self, key):
# memcache is only backend that can increment atomically
try:
# add first, to ensure the key exists
cache.add(key, 0, self.expire_after())
cache.incr(key)
except (AttributeError, ValueError):
cache.set(key, cache.get(key, 0) + 1, self.expire_after())
def incr_message_count(cls, channel_id, timeout):
bucket = int(time.time() // cls.BUCKET_SIZE)
key = cls.get_key(channel_id, bucket)
# Add the bucket size to the expiry time so messages that start at
# the end of the bucket still complete
if not cache.add(key, 1, timeout + cls.BUCKET_SIZE):
cache.incr(key)
