def middleware(request): # TODO: DISCLAIMER!!! THIS IS A TEMPORARY HACK TO ESCAPE FROM CURRENT "DDOS ATTACK" # WE MUST IMPLEMENT RATE LIMIT CONTROL IN NGINX OR CLOUDFARE SO WE DON'T HAVE TO RELY ON DJANGO TO DO THAT agent = request.META.get("HTTP_USER_AGENT", "").lower().strip() if not agent or agent in settings.BLOCKED_WEB_AGENTS: raise Ratelimited() path = request.path if settings.APPEND_SLASH and not path.endswith("/"): path += "/" if is_valid_path(path): match = resolve(path) if getattr(match.func, RATELIMITED_VIEW_ATTR, None): # based in ratelimit decorator # https://github.com/jsocol/django-ratelimit/blob/main/ratelimit/decorators.py#L13 if settings.RATELIMIT_ENABLE and is_ratelimited( request=request, group=None, fn=match.func, key=ratelimit_key, rate=settings.RATELIMIT_RATE, method=ALL, increment=True, ): raise Ratelimited() return get_response(request)
def _wrapped(*args, **kwargs): request = args[0] if isinstance(args[0], HttpRequest) else args[1] request.limited = getattr(request, 'limited', False) if not getattr(settings, 'RATELIMIT_ENABLE', True): request.limited = False return fn(*args, **kwargs) rate_local = self.rate if callable(rate_local): rate_local = rate_local(self.group, request) if rate_local is None: """ This has been taken from the original ratelimit function. Ideally it should raise ImproperlyConfigured. """ return fn(*args, **kwargs) ban_duration = self._extract_ban_duration(self.ban, request) group_local = self.get_group(fn) key_value = self.get_key_value(group=group_local, request=request) ban_cache_key = self._make_ban_cache_key(group_local, rate_local, key_value, self.method, ban_duration) banned = self.is_banned(ban_cache_key) if banned and self.block: raise Ratelimited # Checks if the user is ratelimited. ratelimited = is_ratelimited(request=request, group=self.group, fn=fn, key=self.key, rate=self.rate, method=self.method, increment=True) if ratelimited: self.cache.add(ban_cache_key, ban_duration, ban_duration + self.EXPIRATION_FUDGE) if self.block: exception = Ratelimited() exception.banlimit_data = { "key": self.key, "key_value": key_value, "ban_duration": ban_duration } # Raise Ratelimited exception with details about banned entity. raise exception response = fn(*args, **kwargs) return response
def confirm(request, token): if is_ratelimited(request, group='basket.news.views.confirm', key=lambda x, y: token, rate=EMAIL_SUBSCRIBE_RATE_LIMIT, increment=True): raise Ratelimited() confirm_user.delay(token, start_time=time()) return HttpResponseJSON({'status': 'ok'})
def middleware(request): # Code to be executed for each request before the view (and later middleware) # are called. # We need to provide a group because we don't have the view func available here # to do it automatically, plus, this is a good opportunity to split admin users # from non-admins, so we can offer different throttling rates. _group = _get_group(request) # We need to pick an appropriate rate so that Wagtail Admin users aren't # adversely affected in some places, such as the Wagtail Image Chooser _rate = _get_appropriate_rate(_group) old_limited = getattr(request, "limited", False) ratelimited = is_ratelimited( request=request, group=_group, key="ip", rate=_rate, increment=True, method=ALL, # ie include GET, not just ratelimit.UNSAFE methods ) request.limited = ratelimited or old_limited if ratelimited: raise Ratelimited() response = get_response(request) return response
def _wrapped(root, info, **kw): request = info.context old_limited = getattr(request, "limited", False) if key and key.startswith("gql:"): _key = key.split("gql:")[1] value = kw.get(_key, None) if not value: raise ValueError(f"Cannot get key: {key}") request.gql_rl_field = value new_key = GQLRatelimitKey else: new_key = key ratelimited = is_ratelimited( request=request, group=group, fn=fn, key=new_key, rate=rate, method=method, increment=True, ) request.limited = ratelimited or old_limited if ratelimited and block: logger.warn("url:<%s> is denied for <%s> in Ratelimit" % (request.path, request.META["REMOTE_ADDR"])) raise Ratelimited("rate_limited") return fn(root, info, **kw)
def test_home_when_rate_limited(mock_render, client, db): """ Cloudfront CDN's don't cache 429's, but let's test this anyway. """ mock_render.side_effect = Ratelimited() response = client.get(reverse('home')) assert response.status_code == 429 assert_no_cache_header(response)
def subscribe_sms(request): mobile = request.POST.get("mobile_number") if not mobile: return HttpResponseJSON( { "status": "error", "desc": "mobile_number is missing", "code": errors.BASKET_USAGE_ERROR, }, 400, ) country = request.POST.get("country", "us") language = request.POST.get("lang", "en-US") msg_name = request.POST.get("msg_name", "SMS_Android") vendor_id = get_sms_vendor_id(msg_name, country, language) if not vendor_id: if language != "en-US": # if not available in the requested language, try the default language = "en-US" vendor_id = get_sms_vendor_id(msg_name, country, language) if not vendor_id: return HttpResponseJSON( { "status": "error", "desc": "Invalid msg_name + country + language", "code": errors.BASKET_USAGE_ERROR, }, 400, ) mobile = parse_phone_number(mobile, country) if not mobile: return HttpResponseJSON( { "status": "error", "desc": "mobile_number is invalid", "code": errors.BASKET_USAGE_ERROR, }, 400, ) # only rate limit numbers here so we don't rate limit errors. if is_ratelimited( request, group="basket.news.views.subscribe_sms", key=lambda x, y: "%s-%s" % (msg_name, mobile), rate=PHONE_NUMBER_RATE_LIMIT, increment=True, ): raise Ratelimited() optin = request.POST.get("optin", "N") == "Y" add_sms_user.delay(msg_name, mobile, optin, vendor_id=vendor_id) return HttpResponseJSON({"status": "ok"})
def dispatch(self, *args, **kwargs): ratelimited = is_ratelimited(request=self.request, group=self.ratelimit_group, key=self.ratelimit_key, rate=self.ratelimit_rate, increment=False) if ratelimited and self.ratelimit_block: raise Ratelimited() return super(RateLimitedFormView, self).dispatch(*args, **kwargs)
def _wrapped(request, *args, **kw): old_limited = getattr(request, 'limited', False) ratelimited = is_ratelimited(request=request, group=group, fn=fn, key=key, rate=rate, method=method, increment=True) request.limited = ratelimited or old_limited if ratelimited and block: raise Ratelimited() return fn(request, *args, **kw)
def test_ratelimit_429(client, db): '''Custom 429 view is used for Ratelimited exception.''' url = reverse('home') with mock.patch('kuma.landing.views.render') as render: render.side_effect = Ratelimited() response = client.get(url) assert response.status_code == 429 assert '429.html' in [t.name for t in response.templates] assert response['Retry-After'] == '60' assert_no_cache_header(response)
def test_ratelimit_429(client, db): """Custom 429 view is used for Ratelimited exception.""" url = reverse("home") with mock.patch("kuma.landing.views.render") as render: render.side_effect = Ratelimited() response = client.get(url) assert response.status_code == 429 assert "429.html" in [t.name for t in response.templates] assert response["Retry-After"] == "60" assert_no_cache_header(response)
def test_ratelimit_429(client, db): '''Custom 429 view is used for Ratelimited exception.''' url = reverse('home', locale='en-US') with mock.patch('kuma.landing.views.render') as render: render.side_effect = Ratelimited() response = client.get(url) assert response.status_code == 429 assert '429.html' in [t.name for t in response.templates] assert response['Retry-After'] == '60' assert response['Cache-Control'] == 'no-cache, no-store, must-revalidate'
def test_home_when_rate_limited(mock_render, client, db): """ Cloudfront CDN's don't cache 429's, but let's test this anyway. """ mock_render.side_effect = Ratelimited() response = client.get(reverse('home', locale='en-US')) assert response.status_code == 429 assert 'max-age=0' in response['Cache-Control'] assert 'no-cache' in response['Cache-Control'] assert 'no-store' in response['Cache-Control'] assert 'must-revalidate' in response['Cache-Control']
def confirm(request, token): token = str(token) if is_ratelimited( request, group="basket.news.views.confirm", key=lambda x, y: token, rate=EMAIL_SUBSCRIBE_RATE_LIMIT, increment=True, ): raise Ratelimited() confirm_user.delay(token, start_time=time()) return HttpResponseJSON({"status": "ok"})
def wrapper(request, *args, **kwargs): if not settings.DEBUG: block_info = get_usage(request, key="ip", fn=func, rate="2/10s", increment=True) print("Block_info: {}".format(block_info)) if block_info['should_limit']: raise Ratelimited() tacking_info(request) return func(request, *args, **kwargs)
def _wrapped(request, *args, **kw): request.limited = getattr(request, 'limited', False) if skip_if is None or not skip_if(request): ratelimited = is_ratelimited(request=request, increment=True, ip=ip, method=method, field=field, rate=rate, keys=keys) if ratelimited and block: raise Ratelimited() return fn(request, *args, **kw)
def _wrapped(*args, **kw): # Work as a CBV method decorator. if isinstance(args[0], HttpRequest): request = args[0] else: request = args[1] request.limited = getattr(request, 'limited', False) ratelimited = is_ratelimited(request=request, group=group, fn=fn, key=key, rate=rate, method=method, increment=True) if ratelimited and block: raise Ratelimited() return fn(*args, **kw)
def _wrapped(*args, **kw): # Work as a CBV method decorator. if isinstance(args[0], HttpRequest): request = args[0] else: request = args[1] request.limited = getattr(request, 'limited', False) rate_limit_logging(request, *args, **kw) # rule-out exempt if check_bypassed_rules(request): #print("ruled-out") return fn(*args, **kw) # whitelist if check_bypassed_ip(request): #print("whitelisted") return fn(*args, **kw) if is_authenticated(request.user) is False \ and is_ratelimited(request=request, group=nlgroup, fn=fn, key="ip", rate=nlrate, method=method, increment=True): if redirecturl is None: # return redirect("/accounts/login/?next="+request.path) raise Ratelimited() else: return redirect("/accounts/login/?next=" + redirecturl) if is_authenticated(request.user) is True \ and is_ratelimited(request=request, group=group, fn=fn, key="user", rate=rate, method=method, increment=True): raise Ratelimited() return fn(*args, **kw)
def subscribe_sms(request): mobile = request.POST.get('mobile_number') if not mobile: return HttpResponseJSON( { 'status': 'error', 'desc': 'mobile_number is missing', 'code': errors.BASKET_USAGE_ERROR, }, 400) country = request.POST.get('country', 'us') language = request.POST.get('lang', 'en-US') msg_name = request.POST.get('msg_name', 'SMS_Android') vendor_id = get_sms_vendor_id(msg_name, country, language) if not vendor_id: if language != 'en-US': # if not available in the requested language, try the default language = 'en-US' vendor_id = get_sms_vendor_id(msg_name, country, language) if not vendor_id: return HttpResponseJSON( { 'status': 'error', 'desc': 'Invalid msg_name + country + language', 'code': errors.BASKET_USAGE_ERROR, }, 400) mobile = parse_phone_number(mobile, country) if not mobile: return HttpResponseJSON( { 'status': 'error', 'desc': 'mobile_number is invalid', 'code': errors.BASKET_USAGE_ERROR, }, 400) # only rate limit numbers here so we don't rate limit errors. if is_ratelimited(request, group='basket.news.views.subscribe_sms', key=lambda x, y: '%s-%s' % (msg_name, mobile), rate=PHONE_NUMBER_RATE_LIMIT, increment=True): raise Ratelimited() optin = request.POST.get('optin', 'N') == 'Y' add_sms_user.delay(msg_name, mobile, optin, vendor_id=vendor_id) return HttpResponseJSON({'status': 'ok'})
def _wrapped(request, *args, **kw): cache = getattr(settings, 'RATELIMIT_USE_CACHE', 'default') cache = get_cache(cache) request.limited = False if (RATELIMIT_ENABLE and _method_match(request, method) and (skip_if is None or not skip_if(request))): _keys = _get_keys(request, ip, field, keys) counts = _incr(cache, _keys, period) if any([c > count for c in counts.values()]): request.limited = True if block: raise Ratelimited() return fn(request, *args, **kw)
def wrapper(request, *args, **kwargs): if request.limited: if getattr(settings, 'BLACKLIST_ENABLE', True) \ and getattr(settings, 'BLACKLIST_RATELIMITED_ENABLE', True): if user_duration and request.user.is_authenticated: _create_user_rule(request, user_duration) elif ip_duration: _create_ip_rule(request, ip_duration) else: logger.warning( 'Unable to blacklist ratelimited client.') if block: raise Ratelimited() return fn(request, *args, **kwargs)
def login(self, request, user, data): if user.is_authenticated: return self._ok_response(user, request.META['CSRF_COOKIE']) if is_ratelimited(request, **self.ratelimit_config, increment=False): raise Ratelimited() username = input_to_username(data.get('username')) password = data.get('password') user = authenticate(request, username=username, password=password) if not user: is_ratelimited(request, **self.ratelimit_config, increment=True) logger.info({'event': 'login_failed', 'username': username}) raise AuthenticationFailed() login(request, user) logger.info({'event': 'login_success', 'username': user.username}) return self._ok_response(user, request.META['CSRF_COOKIE'])
def _wrapped(request, *args, **kw): _block = getattr(settings, 'RATE_LIMITER_BLOCK', block) _rate = getattr(settings, 'RATE_LIMITER_RATE', rate) _lockout = getattr(settings, 'RATE_LIMITER_ACCOUNT_LOCKOUT', False) old_limited = getattr(request, 'limited', False) ratelimited = is_ratelimited(request=request, fn=fn, key=key, rate=_rate, method=method, increment=True) request.limited = ratelimited or old_limited if ratelimited and _block: if _lockout: username = request.POST.get('username', None) if username: dojo_user = Dojo_User.objects.filter( username=username).first() if dojo_user: Dojo_User.enable_force_password_rest(dojo_user) raise Ratelimited() return fn(request, *args, **kw)
def subscribe_sms(request): if 'mobile_number' not in request.POST: return HttpResponseJSON({ 'status': 'error', 'desc': 'mobile_number is missing', 'code': errors.BASKET_USAGE_ERROR, }, 400) messages = get_sms_messages() msg_name = request.POST.get('msg_name', 'SMS_Android') if msg_name not in messages: return HttpResponseJSON({ 'status': 'error', 'desc': 'Invalid msg_name', 'code': errors.BASKET_USAGE_ERROR, }, 400) mobile = request.POST['mobile_number'] mobile = re.sub(r'\D+', '', mobile) if len(mobile) == 10: mobile = '1' + mobile elif len(mobile) != 11 or mobile[0] != '1': return HttpResponseJSON({ 'status': 'error', 'desc': 'mobile_number must be a US number', 'code': errors.BASKET_USAGE_ERROR, }, 400) # only rate limit numbers here so we don't rate limit errors. if is_ratelimited(request, group='news.views.subscribe_sms', key=lambda x, y: '%s-%s' % (msg_name, mobile), rate=PHONE_NUMBER_RATE_LIMIT, increment=True): raise Ratelimited() optin = request.POST.get('optin', 'N') == 'Y' add_sms_user.delay(msg_name, mobile, optin) return HttpResponseJSON({'status': 'ok'})
def middleware(request): # Code to be executed for each request before # the view (and later middleware) are called. # Set an arbitrary catch-all group name, because we need to provide # a group because we don't have the view func available here. group = "all_requests" old_limited = getattr(request, "limited", False) ratelimited = is_ratelimited( request=request, group=group, key="ip", rate=settings.DEVPORTAL_RATELIMIT_DEFAULT_LIMIT, increment=True, method=ALL, # ie include GET, not just ratelimit.UNSAFE methods ) request.limited = ratelimited or old_limited if ratelimited: raise Ratelimited() response = get_response(request) return response
def _raise_exception(request): setattr(request, BLOCKED_REQUEST_ATTR, True) raise Ratelimited()
from django.conf.urls import url from django.test import TestCase, override_settings from django.urls import reverse from developerportal.urls import urlpatterns as real_urlpatterns from ratelimit.exceptions import Ratelimited from ..views import rate_limited # Make the rate_limited view available to this test only without needing DEBUG=True. # We prepend our test url(), else wagtail's catch-all at the bottom of # real_urlpatterns will stop it being reached urlpatterns = [ url( r"^test-rate-limited/$", rate_limited, {"exception": Ratelimited()}, name="test-rl-view", ) ] + real_urlpatterns @override_settings(ROOT_URLCONF=__name__) class RateLimitViewTests(TestCase): def test_rate_limited__does_not_get_cached_at_cdn(self): resp = self.client.get(reverse("test-rl-view")) self.assertEqual( resp["Cache-Control"], "max-age=0, no-cache, no-store, must-revalidate" )
TemplateView.as_view(template_name="robots.txt", content_type="text/plain"), ), ] # Custom 403 handling, sending either a rate-limited response or a regular Forbidden def handler403(request, exception=None): if isinstance(exception, Ratelimited): return rate_limited(request, exception) return permission_denied(request, exception) if settings.DEBUG: urlpatterns += [ url(r"^429/$", rate_limited, {"exception": Ratelimited()}), url(r"^403/$", permission_denied, {"exception": HttpResponseForbidden()}), url(r"^404/$", page_not_found, {"exception": Http404()}), url(r"^500/$", server_error), ] from django.conf.urls.static import static from django.contrib.staticfiles.urls import staticfiles_urlpatterns # Serve static and media files from development server urlpatterns += staticfiles_urlpatterns() urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) else: from django.views.static import serve
def update_user_task(request, api_call_type, data=None, optin=False, sync=False): """Call the update_user task async with the right parameters. If sync==True, be sure to include the token in the response. Otherwise, basket can just do everything in the background. """ data = data or request.POST.dict() newsletters = parse_newsletters_csv(data.get('newsletters')) if newsletters: if api_call_type == SUBSCRIBE: all_newsletters = newsletter_and_group_slugs() + get_transactional_message_ids() else: all_newsletters = newsletter_slugs() private_newsletters = newsletter_private_slugs() for nl in newsletters: if nl not in all_newsletters: return HttpResponseJSON({ 'status': 'error', 'desc': 'invalid newsletter', 'code': errors.BASKET_INVALID_NEWSLETTER, }, 400) if api_call_type != UNSUBSCRIBE and nl in private_newsletters: if not is_authorized(request, data.get('email')): return HttpResponseJSON({ 'status': 'error', 'desc': 'private newsletter subscription requires a valid API key or OAuth', 'code': errors.BASKET_AUTH_ERROR, }, 401) if 'lang' in data: if not language_code_is_valid(data['lang']): data['lang'] = 'en' elif 'accept_lang' in data: lang = get_best_language(get_accept_languages(data['accept_lang'])) if lang: data['lang'] = lang del data['accept_lang'] # if lang not provided get the best one from the accept-language header else: lang = get_best_request_lang(request) if lang: data['lang'] = lang email = data.get('email') token = data.get('token') if not (email or token): return HttpResponseJSON({ 'status': 'error', 'desc': MSG_EMAIL_OR_TOKEN_REQUIRED, 'code': errors.BASKET_USAGE_ERROR, }, 400) if optin: data['optin'] = True if api_call_type == SUBSCRIBE and email and data.get('newsletters'): # only rate limit here so we don't rate limit errors. if is_ratelimited(request, group='basket.news.views.update_user_task.subscribe', key=lambda x, y: '%s-%s' % (data['newsletters'], email), rate=EMAIL_SUBSCRIBE_RATE_LIMIT, increment=True): raise Ratelimited() if api_call_type == SET and token and data.get('newsletters'): # only rate limit here so we don't rate limit errors. if is_ratelimited(request, group='basket.news.views.update_user_task.set', key=lambda x, y: '%s-%s' % (data['newsletters'], token), rate=EMAIL_SUBSCRIBE_RATE_LIMIT, increment=True): raise Ratelimited() if sync: statsd.incr('news.views.subscribe.sync') if settings.MAINTENANCE_MODE and not settings.MAINTENANCE_READ_ONLY: # save what we can upsert_user.delay(api_call_type, data, start_time=time()) # have to error since we can't return a token return HttpResponseJSON({ 'status': 'error', 'desc': 'sync is not available in maintenance mode', 'code': errors.BASKET_NETWORK_FAILURE, }, 400) try: user_data = get_user_data(email=email, token=token) except NewsletterException as e: return newsletter_exception_response(e) if not user_data: if not email: # must have email to create a user return HttpResponseJSON({ 'status': 'error', 'desc': MSG_EMAIL_OR_TOKEN_REQUIRED, 'code': errors.BASKET_USAGE_ERROR, }, 400) token, created = upsert_contact(api_call_type, data, user_data) return HttpResponseJSON({ 'status': 'ok', 'token': token, 'created': created, }) else: upsert_user.delay(api_call_type, data, start_time=time()) return HttpResponseJSON({ 'status': 'ok', })
def update_user_task(request, api_call_type, data=None, optin=False, sync=False): """Call the update_user task async with the right parameters. If sync==True, be sure to include the token in the response. Otherwise, basket can just do everything in the background. """ data = data or request.POST.dict() newsletters = parse_newsletters_csv(data.get("newsletters")) if newsletters: if api_call_type == SUBSCRIBE: all_newsletters = (newsletter_and_group_slugs() + get_transactional_message_ids()) else: all_newsletters = newsletter_slugs() private_newsletters = newsletter_private_slugs() for nl in newsletters: if nl not in all_newsletters: return HttpResponseJSON( { "status": "error", "desc": "invalid newsletter", "code": errors.BASKET_INVALID_NEWSLETTER, }, 400, ) if api_call_type != UNSUBSCRIBE and nl in private_newsletters: if not is_authorized(request, data.get("email")): return HttpResponseJSON( { "status": "error", "desc": "private newsletter subscription requires a valid API key or OAuth", "code": errors.BASKET_AUTH_ERROR, }, 401, ) if "lang" in data: if not language_code_is_valid(data["lang"]): data["lang"] = "en" elif "accept_lang" in data: lang = get_best_language(get_accept_languages(data["accept_lang"])) if lang: data["lang"] = lang del data["accept_lang"] # if lang not provided get the best one from the accept-language header else: lang = get_best_request_lang(request) if lang: data["lang"] = lang # now ensure that if we do have a lang that it's a supported one if "lang" in data: data["lang"] = get_best_supported_lang(data["lang"]) email = data.get("email") token = data.get("token") if not (email or token): return HttpResponseJSON( { "status": "error", "desc": MSG_EMAIL_OR_TOKEN_REQUIRED, "code": errors.BASKET_USAGE_ERROR, }, 400, ) if optin: data["optin"] = True if api_call_type == SUBSCRIBE and email and data.get("newsletters"): # only rate limit here so we don't rate limit errors. if is_ratelimited( request, group="basket.news.views.update_user_task.subscribe", key=lambda x, y: "%s-%s" % (data["newsletters"], email), rate=EMAIL_SUBSCRIBE_RATE_LIMIT, increment=True, ): raise Ratelimited() if api_call_type == SET and token and data.get("newsletters"): # only rate limit here so we don't rate limit errors. if is_ratelimited( request, group="basket.news.views.update_user_task.set", key=lambda x, y: "%s-%s" % (data["newsletters"], token), rate=EMAIL_SUBSCRIBE_RATE_LIMIT, increment=True, ): raise Ratelimited() if sync: statsd.incr("news.views.subscribe.sync") if settings.MAINTENANCE_MODE and not settings.MAINTENANCE_READ_ONLY: # save what we can upsert_user.delay(api_call_type, data, start_time=time()) # have to error since we can't return a token return HttpResponseJSON( { "status": "error", "desc": "sync is not available in maintenance mode", "code": errors.BASKET_NETWORK_FAILURE, }, 400, ) try: user_data = get_user_data(email=email, token=token) except NewsletterException as e: return newsletter_exception_response(e) if not user_data: if not email: # must have email to create a user return HttpResponseJSON( { "status": "error", "desc": MSG_EMAIL_OR_TOKEN_REQUIRED, "code": errors.BASKET_USAGE_ERROR, }, 400, ) token, created = upsert_contact(api_call_type, data, user_data) return HttpResponseJSON({ "status": "ok", "token": token, "created": created }) else: upsert_user.delay(api_call_type, data, start_time=time()) return HttpResponseJSON({"status": "ok"})