def test_get_window_of_first_exceeded_limit_none(): per_user_rate_def = RateDefinition(per_second=10) min_rate_def = RateDefinition(per_second=100) per_user_rate_def = PerUserRateDefinition(per_user_rate_def, min_rate_def) rate_limiter = RateLimiter('my_feature', per_user_rate_def.get_rate_limits) rate_limiter.iter_rates = Mock(return_value=[('second', 9, 10)]) expected_window = None actual_window = rate_limiter.get_window_of_first_exceeded_limit( 'my_domain') eq(actual_window, expected_window)
def test_rate_limit_interface(): """ Just test that very basic usage doesn't error """ per_user_rate_def = RateDefinition(per_week=50000, per_day=13000, per_second=.001) min_rate_def = RateDefinition(per_second=10) per_user_rate_def = PerUserRateDefinition(per_user_rate_def, min_rate_def) my_feature_rate_limiter = RateLimiter('my_feature', per_user_rate_def.get_rate_limits) if my_feature_rate_limiter.allow_usage('my_domain'): # ...do stuff... my_feature_rate_limiter.report_usage('my_domain')
def test_get_window_of_first_exceeded_limit_priority(): per_user_rate_def = RateDefinition(per_second=10, per_week=10) min_rate_def = RateDefinition(per_second=100) per_user_rate_def = PerUserRateDefinition(per_user_rate_def, min_rate_def) rate_limiter = RateLimiter('my_feature', per_user_rate_def.get_rate_limits) rate_limiter.iter_rates = Mock(return_value=[('', [('week', 11, 10), ('second', 11, 10)])]) expected_window = 'week' actual_window = rate_limiter.get_window_of_first_exceeded_limit( 'my_domain') eq(actual_window, expected_window)
def test_rate_definition_against_fixed_user_table(): tab_delimited_table = """ # of users second minute hour day week 1 1 10 33 73 215 10 1 10 60 280 1,250 100 1 17 330 2,350 11,600 1000 6 80 3,030 23,050 115,100 10000 51 710 30,030 230,050 1,150,100 100000 501 7,010 300,030 2,300,050 11,500,100 """ rows = tab_delimited_table.strip().splitlines()[1:] table = [[int(cell.replace(',', '')) for cell in row.split()] for row in rows] def check(n_users, per_second, per_minute, per_hour, per_day, per_week): testil.eq( per_user_rate.times(n_users).plus(floor_rate).map(int), RateDefinition( per_week=per_week, per_day=per_day, per_hour=per_hour, per_minute=per_minute, per_second=per_second, ), ) per_user_rate = RateDefinition( per_week=115, per_day=23, per_hour=3, per_minute=0.07, per_second=0.005, ) floor_rate = RateDefinition( per_week=100, per_day=50, per_hour=30, per_minute=10, per_second=1, ) for n_users, per_second, per_minute, per_hour, per_day, per_week in table: yield check, n_users, per_second, per_minute, per_hour, per_day, per_week
def _get_random_rate_def(): numbers = random.choices([None] + [random.expovariate(1 / 64) for _ in range(6)], k=5) return RateDefinition( per_second=numbers[0], per_minute=numbers[1], per_hour=numbers[2], per_day=numbers[3], per_week=numbers[4], )
def check(n_users, per_second, per_minute, per_hour, per_day, per_week): testil.eq( per_user_rate.times(n_users).plus(floor_rate).map(int), RateDefinition( per_week=per_week, per_day=per_day, per_hour=per_hour, per_minute=per_minute, per_second=per_second, ), )
def test_get_standard_ratio_rate_definition(): testil.eq( get_standard_ratio_rate_definition(events_per_day=23), RateDefinition( per_week=115, per_day=23, per_hour=3, per_minute=0.07, per_second=0.005, ), )
def check(rate_def, factor): testil.eq( rate_def.times(factor), RateDefinition( per_second=times_or_none(rate_def.per_second, factor), per_minute=times_or_none(rate_def.per_minute, factor), per_hour=times_or_none(rate_def.per_hour, factor), per_day=times_or_none(rate_def.per_day, factor), per_week=times_or_none(rate_def.per_week, factor), ), )
def check(rate_def, fn): testil.eq( rate_def.map(fn), RateDefinition( per_second=fn(rate_def.per_second), per_minute=fn(rate_def.per_minute), per_hour=fn(rate_def.per_hour), per_day=fn(rate_def.per_day), per_week=fn(rate_def.per_week), ), )
def check(base_rate, floor_rate): testil.eq( base_rate.plus(floor_rate), RateDefinition(per_second=asymmetric_sum(base_rate.per_second, floor_rate.per_second), per_minute=asymmetric_sum(base_rate.per_minute, floor_rate.per_minute), per_hour=asymmetric_sum(base_rate.per_hour, floor_rate.per_hour), per_day=asymmetric_sum(base_rate.per_day, floor_rate.per_day), per_week=asymmetric_sum(base_rate.per_week, floor_rate.per_week)))
def test_conversion(self): self.assertEqual( rate_definition_from_db_object( DynamicRateDefinition( key='a', per_week=10, )), RateDefinition(per_week=10)) self.assertEqual( rate_definition_from_db_object( DynamicRateDefinition( key='a', per_week=10, per_day=8.5, per_hour=7, per_minute=5.5, per_second=4, )), RateDefinition( per_week=10, per_day=8.5, per_hour=7, per_minute=5.5, per_second=4, ))
def _get_per_user_submission_rate_definition(domain): return PerUserRateDefinition( per_user_rate_definition=get_dynamic_rate_definition( 'submissions_per_user', default=get_standard_ratio_rate_definition(events_per_day=46), ), constant_rate_definition=get_dynamic_rate_definition( 'baseline_submissions_per_project', default=RateDefinition( per_week=100, per_day=50, per_hour=30, per_minute=10, per_second=1, ), ), ).get_rate_limits(domain)
from corehq.apps.data_dictionary.util import save_case_property from corehq.apps.domain.decorators import login_and_domain_required from corehq.apps.hqwebapp.decorators import use_jquery_ui from corehq.apps.hqwebapp.utils import get_bulk_upload_form from corehq.apps.settings.views import BaseProjectDataView from corehq.util.files import file_extention_from_filename from corehq.util.workbook_reading import open_any_workbook data_dictionary_rebuild_rate_limiter = RateLimiter( feature_key='data_dictionary_rebuilds_per_user', get_rate_limits=lambda scope: get_dynamic_rate_definition( 'data_dictionary_rebuilds_per_user', default=RateDefinition( per_hour=3, per_minute=2, per_second=1, ) ).get_rate_limits(), scope_length=1, ) @login_and_domain_required @toggles.DATA_DICTIONARY.required_decorator() def generate_data_dictionary(request, domain): if data_dictionary_rebuild_rate_limiter.allow_usage(domain): data_dictionary_rebuild_rate_limiter.report_usage(domain) try: util.generate_data_dictionary(domain) except util.OldExportsEnabledException: return JsonResponse({
metrics_counter('commcare.two_factor.setup_requests', 1, tags={ 'status': status, 'method': method, }) return status != _status_accepted two_factor_setup_rate_limiter = RateLimiter( feature_key='two_factor_setup_attempts', get_rate_limits=lambda scope: get_dynamic_rate_definition( 'two_factor_setup_attempts', default=RateDefinition( per_week=15, per_day=8, per_hour=5, per_minute=3, per_second=1, ) ).get_rate_limits(), scope_length=1, # per user OR per IP ) global_two_factor_setup_rate_limiter = RateLimiter( feature_key='global_two_factor_setup_attempts', get_rate_limits=lambda: get_dynamic_rate_definition( 'global_two_factor_setup_attempts', default=RateDefinition( per_day=100, ) ).get_rate_limits(),
1, tags=[ 'status:{}'.format(status), 'method:{}'.format(method), ]) return status != _status_accepted two_factor_setup_rate_limiter = RateLimiter( feature_key='two_factor_setup_attempts', get_rate_limits=lambda scope: get_dynamic_rate_definition( 'two_factor_setup_attempts', default=RateDefinition( per_week=15, per_day=8, per_hour=5, per_minute=3, per_second=1, )).get_rate_limits(), scope_length=1, # per user OR per IP ) global_two_factor_setup_rate_limiter = RateLimiter( feature_key='global_two_factor_setup_attempts', get_rate_limits=lambda: get_dynamic_rate_definition( 'global_two_factor_setup_attempts', default=RateDefinition(per_day=100, )).get_rate_limits(), scope_length=0, )
) from corehq.apps.data_dictionary.util import save_case_property from corehq.apps.domain.decorators import login_and_domain_required from corehq.apps.hqwebapp.decorators import use_jquery_ui from corehq.apps.hqwebapp.utils import get_bulk_upload_form from corehq.apps.settings.views import BaseProjectDataView from corehq.util.files import file_extention_from_filename from corehq.util.workbook_reading import open_any_workbook data_dictionary_rebuild_rate_limiter = RateLimiter( feature_key='data_dictionary_rebuilds_per_user', get_rate_limits=lambda scope: get_dynamic_rate_definition( 'data_dictionary_rebuilds_per_user', default=RateDefinition( per_hour=3, per_minute=2, per_second=1, )).get_rate_limits(), scope_length=1, ) @login_and_domain_required @toggles.DATA_DICTIONARY.required_decorator() def generate_data_dictionary(request, domain): if data_dictionary_rebuild_rate_limiter.allow_usage(domain): data_dictionary_rebuild_rate_limiter.report_usage(domain) try: util.generate_data_dictionary(domain) except util.OldExportsEnabledException: return JsonResponse(
per_week=100, per_day=50, per_hour=30, per_minute=10, per_second=1, ), ), ).get_rate_limits(domain) global_submission_rate_limiter = RateLimiter( feature_key='global_submissions', get_rate_limits=lambda: get_dynamic_rate_definition('global_submissions', default=RateDefinition( per_hour=17000, per_minute=400, per_second=30, )).get_rate_limits(), scope_length=0, ) SHOULD_RATE_LIMIT_SUBMISSIONS = settings.RATE_LIMIT_SUBMISSIONS and not settings.UNIT_TESTING @run_only_when(SHOULD_RATE_LIMIT_SUBMISSIONS) @silence_and_report_error("Exception raised in the submission rate limiter", 'commcare.xform_submissions.rate_limiter_errors') def rate_limit_submission(domain): if TEST_FORM_SUBMISSION_RATE_LIMIT_RESPONSE.enabled(domain): return True should_allow_usage = (global_submission_rate_limiter.allow_usage()
from corehq.project_limits.shortcuts import get_standard_ratio_rate_definition from corehq.toggles import RATE_LIMIT_RESTORES, NAMESPACE_DOMAIN, BLOCK_RESTORES from corehq.util.decorators import run_only_when, silence_and_report_error from corehq.util.metrics import metrics_counter RESTORES_PER_DAY = 3 restore_rate_limiter = RateLimiter( feature_key='restores', get_rate_limits=PerUserRateDefinition( per_user_rate_definition=get_standard_ratio_rate_definition( events_per_day=RESTORES_PER_DAY), constant_rate_definition=RateDefinition( per_week=50, per_day=25, per_hour=15, per_minute=5, per_second=1, ), ).get_rate_limits) SHOULD_RATE_LIMIT_RESTORES = not settings.ENTERPRISE_MODE and not settings.UNIT_TESTING @run_only_when(SHOULD_RATE_LIMIT_RESTORES) @silence_and_report_error("Exception raised in the restore rate limiter", 'commcare.restores.rate_limiter_errors') def rate_limit_restore(domain): if RATE_LIMIT_RESTORES.enabled(domain, namespace=NAMESPACE_DOMAIN): return _rate_limit_restore(domain) elif BLOCK_RESTORES.enabled(domain, namespace=NAMESPACE_DOMAIN):
from django.conf import settings from corehq.project_limits.rate_limiter import ( PerUserRateDefinition, RateDefinition, RateLimiter, ) from corehq.toggles import RATE_LIMIT_RESTORES, NAMESPACE_DOMAIN from corehq.util.datadog.gauges import datadog_counter from corehq.util.decorators import run_only_when, silence_and_report_error restores_per_user = RateDefinition( per_week=5.75, per_day=1.15, per_hour=0.15, per_minute=0.0035, per_second=0.00025, ) base_restores_per_domain = RateDefinition( per_week=50, per_day=25, per_hour=15, per_minute=5, per_second=1, ) restore_rates = PerUserRateDefinition( per_user_rate_definition=restores_per_user, constant_rate_definition=base_restores_per_domain, )
# per_second=0.005, # ) == get_standard_ratio_rate_definition(events_per_day=23) # If we as a team end up regretting this decision, we'll have to reset expectations # with the Dimagi NDoH team. SUBMISSIONS_PER_DAY = 46 submission_rate_limiter = RateLimiter( feature_key='submissions', get_rate_limits=PerUserRateDefinition( per_user_rate_definition=get_standard_ratio_rate_definition( events_per_day=SUBMISSIONS_PER_DAY), constant_rate_definition=RateDefinition( per_week=100, per_day=50, per_hour=30, per_minute=10, per_second=1, ), ).get_rate_limits ) global_submission_rate_limiter = RateLimiter( feature_key='global_submissions', get_rate_limits=lambda: get_dynamic_rate_definition( 'global_submissions', default=RateDefinition( per_hour=17000, per_minute=400, per_second=30, )
metrics_counter('commcare.two_factor.setup_requests', 1, tags={ 'status': status, 'method': method, 'window': window or 'none', }) return status != _status_accepted two_factor_rate_limiter_per_ip = RateLimiter( feature_key='two_factor_attempts_per_ip', get_rate_limits=lambda scope: get_dynamic_rate_definition( 'two_factor_attempts_per_ip', default=RateDefinition( per_week=20000, per_day=2000, per_hour=1200, per_minute=700, per_second=60, ) ).get_rate_limits(), scope_length=1, ) two_factor_rate_limiter_per_user = RateLimiter( feature_key='two_factor_attempts_per_user', get_rate_limits=lambda scope: get_dynamic_rate_definition( 'two_factor_attempts_per_user', default=RateDefinition( per_week=120, per_day=40, per_hour=8,
'status': status, 'method': method, 'window': window or 'none', 'domain': domain or 'none', }) return status != _status_accepted two_factor_rate_limiter_per_ip = RateLimiter( feature_key='two_factor_attempts_per_ip', get_rate_limits=lambda scope: get_dynamic_rate_definition( 'two_factor_attempts_per_ip', default=RateDefinition( per_week=20000, per_day=2000, per_hour=1200, per_minute=700, per_second=60, )).get_rate_limits(scope), ) two_factor_rate_limiter_per_user = RateLimiter( feature_key='two_factor_attempts_per_user', get_rate_limits=lambda scope: get_dynamic_rate_definition( 'two_factor_attempts_per_user', default=RateDefinition( per_week=120, per_day=40, per_hour=8, per_minute=2, per_second=1,
RateDefinition, RateLimiter, ) from corehq.toggles import RATE_LIMIT_SUBMISSIONS, NAMESPACE_DOMAIN from corehq.util.datadog.gauges import datadog_counter from corehq.util.datadog.utils import bucket_value from corehq.util.decorators import run_only_when, silence_and_report_error from corehq.util.timer import TimingContext # Danny promised in an Aug 2019 email not to enforce limits that were lower than this. # If we as a team end up regretting this decision, we'll have to reset expectations # with the Dimagi NDoH team. rates_promised_not_to_go_lower_than = RateDefinition( per_week=115, per_day=23, per_hour=3, per_minute=0.07, per_second=0.005, ) floor_for_small_domain = RateDefinition( per_week=100, per_day=50, per_hour=30, per_minute=10, per_second=1, ) test_rates = PerUserRateDefinition( per_user_rate_definition=rates_promised_not_to_go_lower_than.times(2.0), constant_rate_definition=floor_for_small_domain,
from corehq.project_limits.rate_limiter import RateDefinition STANDARD_RATIO = RateDefinition( per_week=115, per_day=23, per_hour=3, per_minute=0.07, per_second=0.005, ).times(1 / 23) def get_standard_ratio_rate_definition(events_per_day): return STANDARD_RATIO.times(events_per_day)