def test_get_window_of_first_exceeded_limit_none():
per_user_rate_def = RateDefinition(per_second=10)
min_rate_def = RateDefinition(per_second=100)
per_user_rate_def = PerUserRateDefinition(per_user_rate_def, min_rate_def)
rate_limiter = RateLimiter('my_feature', per_user_rate_def.get_rate_limits)
rate_limiter.iter_rates = Mock(return_value=[('second', 9, 10)])
expected_window = None
actual_window = rate_limiter.get_window_of_first_exceeded_limit(
'my_domain')
eq(actual_window, expected_window)
def test_rate_limit_interface():
"""
Just test that very basic usage doesn't error
"""
per_user_rate_def = RateDefinition(per_week=50000, per_day=13000, per_second=.001)
min_rate_def = RateDefinition(per_second=10)
per_user_rate_def = PerUserRateDefinition(per_user_rate_def, min_rate_def)
my_feature_rate_limiter = RateLimiter('my_feature', per_user_rate_def.get_rate_limits)
if my_feature_rate_limiter.allow_usage('my_domain'):
# ...do stuff...
my_feature_rate_limiter.report_usage('my_domain')
def test_get_window_of_first_exceeded_limit_priority():
per_user_rate_def = RateDefinition(per_second=10, per_week=10)
min_rate_def = RateDefinition(per_second=100)
per_user_rate_def = PerUserRateDefinition(per_user_rate_def, min_rate_def)
rate_limiter = RateLimiter('my_feature', per_user_rate_def.get_rate_limits)
rate_limiter.iter_rates = Mock(return_value=[('', [('week', 11,
10), ('second', 11,
10)])])
expected_window = 'week'
actual_window = rate_limiter.get_window_of_first_exceeded_limit(
'my_domain')
eq(actual_window, expected_window)
def test_rate_definition_against_fixed_user_table():
tab_delimited_table = """
# of users second minute hour day week
1 1 10 33 73 215
10 1 10 60 280 1,250
100 1 17 330 2,350 11,600
1000 6 80 3,030 23,050 115,100
10000 51 710 30,030 230,050 1,150,100
100000 501 7,010 300,030 2,300,050 11,500,100
"""
rows = tab_delimited_table.strip().splitlines()[1:]
table = [[int(cell.replace(',', '')) for cell in row.split()]
for row in rows]
def check(n_users, per_second, per_minute, per_hour, per_day, per_week):
testil.eq(
per_user_rate.times(n_users).plus(floor_rate).map(int),
RateDefinition(
per_week=per_week,
per_day=per_day,
per_hour=per_hour,
per_minute=per_minute,
per_second=per_second,
),
)
per_user_rate = RateDefinition(
per_week=115,
per_day=23,
per_hour=3,
per_minute=0.07,
per_second=0.005,
)
floor_rate = RateDefinition(
per_week=100,
per_day=50,
per_hour=30,
per_minute=10,
per_second=1,
)
for n_users, per_second, per_minute, per_hour, per_day, per_week in table:
yield check, n_users, per_second, per_minute, per_hour, per_day, per_week
def _get_random_rate_def():
numbers = random.choices([None] +
[random.expovariate(1 / 64) for _ in range(6)],
k=5)
return RateDefinition(
per_second=numbers[0],
per_minute=numbers[1],
per_hour=numbers[2],
per_day=numbers[3],
per_week=numbers[4],
)
def check(n_users, per_second, per_minute, per_hour, per_day, per_week):
testil.eq(
per_user_rate.times(n_users).plus(floor_rate).map(int),
RateDefinition(
per_week=per_week,
per_day=per_day,
per_hour=per_hour,
per_minute=per_minute,
per_second=per_second,
),
)
def test_get_standard_ratio_rate_definition():
testil.eq(
get_standard_ratio_rate_definition(events_per_day=23),
RateDefinition(
per_week=115,
per_day=23,
per_hour=3,
per_minute=0.07,
per_second=0.005,
),
)
def check(rate_def, factor):
testil.eq(
rate_def.times(factor),
RateDefinition(
per_second=times_or_none(rate_def.per_second, factor),
per_minute=times_or_none(rate_def.per_minute, factor),
per_hour=times_or_none(rate_def.per_hour, factor),
per_day=times_or_none(rate_def.per_day, factor),
per_week=times_or_none(rate_def.per_week, factor),
),
)
def check(rate_def, fn):
testil.eq(
rate_def.map(fn),
RateDefinition(
per_second=fn(rate_def.per_second),
per_minute=fn(rate_def.per_minute),
per_hour=fn(rate_def.per_hour),
per_day=fn(rate_def.per_day),
per_week=fn(rate_def.per_week),
),
)
def check(base_rate, floor_rate):
testil.eq(
base_rate.plus(floor_rate),
RateDefinition(per_second=asymmetric_sum(base_rate.per_second,
floor_rate.per_second),
per_minute=asymmetric_sum(base_rate.per_minute,
floor_rate.per_minute),
per_hour=asymmetric_sum(base_rate.per_hour,
floor_rate.per_hour),
per_day=asymmetric_sum(base_rate.per_day,
floor_rate.per_day),
per_week=asymmetric_sum(base_rate.per_week,
floor_rate.per_week)))
def test_conversion(self):
self.assertEqual(
rate_definition_from_db_object(
DynamicRateDefinition(
key='a',
per_week=10,
)), RateDefinition(per_week=10))
self.assertEqual(
rate_definition_from_db_object(
DynamicRateDefinition(
key='a',
per_week=10,
per_day=8.5,
per_hour=7,
per_minute=5.5,
per_second=4,
)),
RateDefinition(
per_week=10,
per_day=8.5,
per_hour=7,
per_minute=5.5,
per_second=4,
))
def _get_per_user_submission_rate_definition(domain):
return PerUserRateDefinition(
per_user_rate_definition=get_dynamic_rate_definition(
'submissions_per_user',
default=get_standard_ratio_rate_definition(events_per_day=46),
),
constant_rate_definition=get_dynamic_rate_definition(
'baseline_submissions_per_project',
default=RateDefinition(
per_week=100,
per_day=50,
per_hour=30,
per_minute=10,
per_second=1,
),
),
).get_rate_limits(domain)
from corehq.apps.data_dictionary.util import save_case_property
from corehq.apps.domain.decorators import login_and_domain_required
from corehq.apps.hqwebapp.decorators import use_jquery_ui
from corehq.apps.hqwebapp.utils import get_bulk_upload_form
from corehq.apps.settings.views import BaseProjectDataView
from corehq.util.files import file_extention_from_filename
from corehq.util.workbook_reading import open_any_workbook
data_dictionary_rebuild_rate_limiter = RateLimiter(
feature_key='data_dictionary_rebuilds_per_user',
get_rate_limits=lambda scope: get_dynamic_rate_definition(
'data_dictionary_rebuilds_per_user',
default=RateDefinition(
per_hour=3,
per_minute=2,
per_second=1,
)
).get_rate_limits(),
scope_length=1,
)
@login_and_domain_required
@toggles.DATA_DICTIONARY.required_decorator()
def generate_data_dictionary(request, domain):
if data_dictionary_rebuild_rate_limiter.allow_usage(domain):
data_dictionary_rebuild_rate_limiter.report_usage(domain)
try:
util.generate_data_dictionary(domain)
except util.OldExportsEnabledException:
return JsonResponse({
metrics_counter('commcare.two_factor.setup_requests', 1, tags={
'status': status,
'method': method,
})
return status != _status_accepted
two_factor_setup_rate_limiter = RateLimiter(
feature_key='two_factor_setup_attempts',
get_rate_limits=lambda scope: get_dynamic_rate_definition(
'two_factor_setup_attempts',
default=RateDefinition(
per_week=15,
per_day=8,
per_hour=5,
per_minute=3,
per_second=1,
)
).get_rate_limits(),
scope_length=1, # per user OR per IP
)
global_two_factor_setup_rate_limiter = RateLimiter(
feature_key='global_two_factor_setup_attempts',
get_rate_limits=lambda: get_dynamic_rate_definition(
'global_two_factor_setup_attempts',
default=RateDefinition(
per_day=100,
)
).get_rate_limits(),
1,
tags=[
'status:{}'.format(status),
'method:{}'.format(method),
])
return status != _status_accepted
two_factor_setup_rate_limiter = RateLimiter(
feature_key='two_factor_setup_attempts',
get_rate_limits=lambda scope: get_dynamic_rate_definition(
'two_factor_setup_attempts',
default=RateDefinition(
per_week=15,
per_day=8,
per_hour=5,
per_minute=3,
per_second=1,
)).get_rate_limits(),
scope_length=1, # per user OR per IP
)
global_two_factor_setup_rate_limiter = RateLimiter(
feature_key='global_two_factor_setup_attempts',
get_rate_limits=lambda: get_dynamic_rate_definition(
'global_two_factor_setup_attempts',
default=RateDefinition(per_day=100, )).get_rate_limits(),
scope_length=0,
)
)
from corehq.apps.data_dictionary.util import save_case_property
from corehq.apps.domain.decorators import login_and_domain_required
from corehq.apps.hqwebapp.decorators import use_jquery_ui
from corehq.apps.hqwebapp.utils import get_bulk_upload_form
from corehq.apps.settings.views import BaseProjectDataView
from corehq.util.files import file_extention_from_filename
from corehq.util.workbook_reading import open_any_workbook
data_dictionary_rebuild_rate_limiter = RateLimiter(
feature_key='data_dictionary_rebuilds_per_user',
get_rate_limits=lambda scope: get_dynamic_rate_definition(
'data_dictionary_rebuilds_per_user',
default=RateDefinition(
per_hour=3,
per_minute=2,
per_second=1,
)).get_rate_limits(),
scope_length=1,
)
@login_and_domain_required
@toggles.DATA_DICTIONARY.required_decorator()
def generate_data_dictionary(request, domain):
if data_dictionary_rebuild_rate_limiter.allow_usage(domain):
data_dictionary_rebuild_rate_limiter.report_usage(domain)
try:
util.generate_data_dictionary(domain)
except util.OldExportsEnabledException:
return JsonResponse(
per_week=100,
per_day=50,
per_hour=30,
per_minute=10,
per_second=1,
),
),
).get_rate_limits(domain)
global_submission_rate_limiter = RateLimiter(
feature_key='global_submissions',
get_rate_limits=lambda: get_dynamic_rate_definition('global_submissions',
default=RateDefinition(
per_hour=17000,
per_minute=400,
per_second=30,
)).get_rate_limits(),
scope_length=0,
)
SHOULD_RATE_LIMIT_SUBMISSIONS = settings.RATE_LIMIT_SUBMISSIONS and not settings.UNIT_TESTING
@run_only_when(SHOULD_RATE_LIMIT_SUBMISSIONS)
@silence_and_report_error("Exception raised in the submission rate limiter",
'commcare.xform_submissions.rate_limiter_errors')
def rate_limit_submission(domain):
if TEST_FORM_SUBMISSION_RATE_LIMIT_RESPONSE.enabled(domain):
return True
should_allow_usage = (global_submission_rate_limiter.allow_usage()
from corehq.project_limits.shortcuts import get_standard_ratio_rate_definition
from corehq.toggles import RATE_LIMIT_RESTORES, NAMESPACE_DOMAIN, BLOCK_RESTORES
from corehq.util.decorators import run_only_when, silence_and_report_error
from corehq.util.metrics import metrics_counter
RESTORES_PER_DAY = 3
restore_rate_limiter = RateLimiter(
feature_key='restores',
get_rate_limits=PerUserRateDefinition(
per_user_rate_definition=get_standard_ratio_rate_definition(
events_per_day=RESTORES_PER_DAY),
constant_rate_definition=RateDefinition(
per_week=50,
per_day=25,
per_hour=15,
per_minute=5,
per_second=1,
),
).get_rate_limits)
SHOULD_RATE_LIMIT_RESTORES = not settings.ENTERPRISE_MODE and not settings.UNIT_TESTING
@run_only_when(SHOULD_RATE_LIMIT_RESTORES)
@silence_and_report_error("Exception raised in the restore rate limiter",
'commcare.restores.rate_limiter_errors')
def rate_limit_restore(domain):
if RATE_LIMIT_RESTORES.enabled(domain, namespace=NAMESPACE_DOMAIN):
return _rate_limit_restore(domain)
elif BLOCK_RESTORES.enabled(domain, namespace=NAMESPACE_DOMAIN):
from django.conf import settings
from corehq.project_limits.rate_limiter import (
PerUserRateDefinition,
RateDefinition,
RateLimiter,
)
from corehq.toggles import RATE_LIMIT_RESTORES, NAMESPACE_DOMAIN
from corehq.util.datadog.gauges import datadog_counter
from corehq.util.decorators import run_only_when, silence_and_report_error
restores_per_user = RateDefinition(
per_week=5.75,
per_day=1.15,
per_hour=0.15,
per_minute=0.0035,
per_second=0.00025,
)
base_restores_per_domain = RateDefinition(
per_week=50,
per_day=25,
per_hour=15,
per_minute=5,
per_second=1,
)
restore_rates = PerUserRateDefinition(
per_user_rate_definition=restores_per_user,
constant_rate_definition=base_restores_per_domain,
)
# per_second=0.005,
# ) == get_standard_ratio_rate_definition(events_per_day=23)
# If we as a team end up regretting this decision, we'll have to reset expectations
# with the Dimagi NDoH team.
SUBMISSIONS_PER_DAY = 46
submission_rate_limiter = RateLimiter(
feature_key='submissions',
get_rate_limits=PerUserRateDefinition(
per_user_rate_definition=get_standard_ratio_rate_definition(
events_per_day=SUBMISSIONS_PER_DAY),
constant_rate_definition=RateDefinition(
per_week=100,
per_day=50,
per_hour=30,
per_minute=10,
per_second=1,
),
).get_rate_limits
)
global_submission_rate_limiter = RateLimiter(
feature_key='global_submissions',
get_rate_limits=lambda: get_dynamic_rate_definition(
'global_submissions',
default=RateDefinition(
per_hour=17000,
per_minute=400,
per_second=30,
)
metrics_counter('commcare.two_factor.setup_requests', 1, tags={
'status': status,
'method': method,
'window': window or 'none',
})
return status != _status_accepted
two_factor_rate_limiter_per_ip = RateLimiter(
feature_key='two_factor_attempts_per_ip',
get_rate_limits=lambda scope: get_dynamic_rate_definition(
'two_factor_attempts_per_ip',
default=RateDefinition(
per_week=20000,
per_day=2000,
per_hour=1200,
per_minute=700,
per_second=60,
)
).get_rate_limits(),
scope_length=1,
)
two_factor_rate_limiter_per_user = RateLimiter(
feature_key='two_factor_attempts_per_user',
get_rate_limits=lambda scope: get_dynamic_rate_definition(
'two_factor_attempts_per_user',
default=RateDefinition(
per_week=120,
per_day=40,
per_hour=8,
'status': status,
'method': method,
'window': window or 'none',
'domain': domain or 'none',
})
return status != _status_accepted
two_factor_rate_limiter_per_ip = RateLimiter(
feature_key='two_factor_attempts_per_ip',
get_rate_limits=lambda scope: get_dynamic_rate_definition(
'two_factor_attempts_per_ip',
default=RateDefinition(
per_week=20000,
per_day=2000,
per_hour=1200,
per_minute=700,
per_second=60,
)).get_rate_limits(scope),
)
two_factor_rate_limiter_per_user = RateLimiter(
feature_key='two_factor_attempts_per_user',
get_rate_limits=lambda scope: get_dynamic_rate_definition(
'two_factor_attempts_per_user',
default=RateDefinition(
per_week=120,
per_day=40,
per_hour=8,
per_minute=2,
per_second=1,
RateDefinition,
RateLimiter,
)
from corehq.toggles import RATE_LIMIT_SUBMISSIONS, NAMESPACE_DOMAIN
from corehq.util.datadog.gauges import datadog_counter
from corehq.util.datadog.utils import bucket_value
from corehq.util.decorators import run_only_when, silence_and_report_error
from corehq.util.timer import TimingContext
# Danny promised in an Aug 2019 email not to enforce limits that were lower than this.
# If we as a team end up regretting this decision, we'll have to reset expectations
# with the Dimagi NDoH team.
rates_promised_not_to_go_lower_than = RateDefinition(
per_week=115,
per_day=23,
per_hour=3,
per_minute=0.07,
per_second=0.005,
)
floor_for_small_domain = RateDefinition(
per_week=100,
per_day=50,
per_hour=30,
per_minute=10,
per_second=1,
)
test_rates = PerUserRateDefinition(
per_user_rate_definition=rates_promised_not_to_go_lower_than.times(2.0),
constant_rate_definition=floor_for_small_domain,
from corehq.project_limits.rate_limiter import RateDefinition
STANDARD_RATIO = RateDefinition(
per_week=115,
per_day=23,
per_hour=3,
per_minute=0.07,
per_second=0.005,
).times(1 / 23)
def get_standard_ratio_rate_definition(events_per_day):
return STANDARD_RATIO.times(events_per_day)
