def _check_within_range( self, key: bytes, count_func: Callable[[], int], trim_func: Optional[Callable[[bytes, int], object]] = None, ) -> None: user_id = int(key.split(b":")[2]) user = get_user_profile_by_id(user_id) entity = RateLimitedUser(user) max_calls = entity.max_api_calls() age = int(client.ttl(key)) if age < 0: logging.error("Found key with age of %s, will never expire: %s", age, key) count = count_func() if count > max_calls: logging.error( "Redis health check found key with more elements \ than max_api_calls! (trying to trim) %s %s", key, count, ) if trim_func is not None: client.expire(key, entity.max_api_window()) trim_func(key, max_calls)
def handle(self, *args, **options): # type: (*Any, **Any) -> None if (not options['api_key'] and not options['email']) or \ (options['api_key'] and options['email']): print("Please enter either an email or API key to manage") exit(1) realm = self.get_realm(options) if options['email']: user_profile = self.get_user(options['email'], realm) else: try: user_profile = get_user_profile_by_api_key(options['api_key']) except UserProfile.DoesNotExist: print("Unable to get user profile for api key %s" % (options['api_key'], )) exit(1) users = [user_profile] if options['bots']: users.extend(bot for bot in UserProfile.objects.filter( is_bot=True, bot_owner=user_profile)) operation = options['operation'] for user in users: print("Applying operation to User ID: %s: %s" % (user.id, operation)) if operation == 'block': block_access(RateLimitedUser(user, domain=options['domain']), options['seconds']) elif operation == 'unblock': unblock_access(RateLimitedUser(user, domain=options['domain']))
def handle(self, *args: Any, **options: Any) -> None: if (not options['api_key'] and not options['email']) or \ (options['api_key'] and options['email']): raise CommandError("Please enter either an email or API key to manage") realm = self.get_realm(options) if options['email']: user_profile = self.get_user(options['email'], realm) else: try: user_profile = get_user_profile_by_api_key(options['api_key']) except UserProfile.DoesNotExist: raise CommandError("Unable to get user profile for api key {}".format(options['api_key'])) users = [user_profile] if options['bots']: users.extend(bot for bot in UserProfile.objects.filter(is_bot=True, bot_owner=user_profile)) operation = options['operation'] for user in users: print(f"Applying operation to User ID: {user.id}: {operation}") if operation == 'block': RateLimitedUser(user, domain=options['domain']).block_access(options['seconds']) elif operation == 'unblock': RateLimitedUser(user, domain=options['domain']).unblock_access()
def test_used_in_tornado(self) -> None: user_profile = self.example_user("hamlet") with self.settings(RUNNING_INSIDE_TORNADO=True): obj = RateLimitedUser(user_profile, domain='api_by_user') self.assertEqual(obj.backend, TornadoInMemoryRateLimiterBackend) with self.settings(RUNNING_INSIDE_TORNADO=True): obj = RateLimitedUser(user_profile, domain='some_domain') self.assertEqual(obj.backend, RedisRateLimiterBackend)
def test_used_in_tornado(self) -> None: user_profile = self.example_user("hamlet") ip_addr = "192.168.0.123" with self.settings(RUNNING_INSIDE_TORNADO=True): user_obj = RateLimitedUser(user_profile, domain="api_by_user") ip_obj = RateLimitedIPAddr(ip_addr, domain="api_by_ip") self.assertEqual(user_obj.backend, TornadoInMemoryRateLimiterBackend) self.assertEqual(ip_obj.backend, TornadoInMemoryRateLimiterBackend) with self.settings(RUNNING_INSIDE_TORNADO=True): user_obj = RateLimitedUser(user_profile, domain="some_domain") ip_obj = RateLimitedIPAddr(ip_addr, domain="some_domain") self.assertEqual(user_obj.backend, RedisRateLimiterBackend) self.assertEqual(ip_obj.backend, RedisRateLimiterBackend)
def test_hit_ratelimits(self) -> None: user = self.example_user('cordelia') email = user.email clear_history(RateLimitedUser(user)) start_time = time.time() for i in range(6): with mock.patch('time.time', return_value=(start_time + i * 0.1)): result = self.send_api_message(email, "some stuff %s" % (i, )) self.assertEqual(result.status_code, 429) json = result.json() self.assertEqual(json.get("result"), "error") self.assertIn("API usage exceeded rate limit", json.get("msg")) self.assertEqual(json.get('retry-after'), 0.5) self.assertTrue('Retry-After' in result) self.assertEqual(result['Retry-After'], '0.5') # We actually wait a second here, rather than force-clearing our history, # to make sure the rate-limiting code automatically forgives a user # after some time has passed. with mock.patch('time.time', return_value=(start_time + 1.0)): result = self.send_api_message(email, "Good message") self.assert_json_success(result)
def _check_within_range( self, key: str, count_func: Callable[[], int], trim_func: Optional[Callable[[str, int], None]] = None) -> None: user_id = int(key.split(':')[1]) try: user = get_user_profile_by_id(user_id) except Exception: user = None entity = RateLimitedUser(user) max_calls = max_api_calls(entity) age = int(client.ttl(key)) if age < 0: logging.error("Found key with age of %s, will never expire: %s" % ( age, key, )) count = count_func() if count > max_calls: logging.error("Redis health check found key with more elements \ than max_api_calls! (trying to trim) %s %s" % (key, count)) if trim_func is not None: client.expire(key, max_api_window(entity)) trim_func(key, max_calls)
def rate_limit_user(request: HttpRequest, user: UserProfile, domain: str) -> None: """Returns whether or not a user was rate limited. Will raise a RateLimited exception if the user has been rate limited, otherwise returns and modifies request to contain the rate limit information""" entity = RateLimitedUser(user, domain=domain) ratelimited, time = is_ratelimited(entity) request._ratelimit_applied_limits = True request._ratelimit_secs_to_freedom = time request._ratelimit_over_limit = ratelimited # Abort this request if the user is over their rate limits if ratelimited: statsd.incr("ratelimiter.limited.%s.%s" % (type(user), user.id)) raise RateLimited() try: incr_ratelimit(entity) except RateLimiterLockingException: logging.warning("Deadlock trying to incr_ratelimit for %s on %s" % ( user.id, request.path)) # rate-limit users who are hitting the API so hard we can't update our stats. raise RateLimited() calls_remaining, time_reset = api_calls_left(entity) request._ratelimit_remaining = calls_remaining request._ratelimit_secs_to_freedom = time_reset
def rate_limit_user(request: HttpRequest, user: UserProfile, domain: str) -> None: """Returns whether or not a user was rate limited. Will raise a RateLimited exception if the user has been rate limited, otherwise returns and modifies request to contain the rate limit information""" RateLimitedUser(user, domain=domain).rate_limit_request(request)
def test_headers(self) -> None: user = self.example_user("hamlet") RateLimitedUser(user).clear_history() result = self.send_api_message(user, "some stuff") self.assertTrue("X-RateLimit-Remaining" in result) self.assertTrue("X-RateLimit-Limit" in result) self.assertTrue("X-RateLimit-Reset" in result)
def test_headers(self) -> None: user = self.example_user('hamlet') clear_history(RateLimitedUser(user)) result = self.send_api_message(user, "some stuff") self.assertTrue('X-RateLimit-Remaining' in result) self.assertTrue('X-RateLimit-Limit' in result) self.assertTrue('X-RateLimit-Reset' in result)
def test_hit_change_email_ratelimit_as_user(self) -> None: user = self.example_user("cordelia") RateLimitedUser(user).clear_history() emails = ["new-email-{n}@zulip.com" for n in range(1, 8)] self.do_test_hit_ratelimits( lambda: self.api_patch(user, "/api/v1/settings", {"email": emails.pop()}), )
def test_hit_ratelimiterlockingexception(self) -> None: user = self.example_user('cordelia') email = user.email clear_history(RateLimitedUser(user)) with mock.patch('zerver.decorator.incr_ratelimit', side_effect=RateLimiterLockingException): result = self.send_api_message(email, "some stuff") self.assertEqual(result.status_code, 429)
def test_ratelimit_decrease(self) -> None: user = self.example_user('hamlet') clear_history(RateLimitedUser(user)) result = self.send_api_message(user, "some stuff") limit = int(result['X-RateLimit-Remaining']) result = self.send_api_message(user, "some stuff 2") newlimit = int(result['X-RateLimit-Remaining']) self.assertEqual(limit, newlimit + 1)
def test_hit_ratelimiterlockingexception(self, mock_warn: mock.MagicMock) -> None: user = self.example_user('cordelia') email = user.email clear_history(RateLimitedUser(user)) with mock.patch('zerver.lib.rate_limiter.incr_ratelimit', side_effect=RateLimiterLockingException): result = self.send_api_message(email, "some stuff") self.assertEqual(result.status_code, 429) mock_warn.assert_called_with("Deadlock trying to incr_ratelimit for RateLimitedUser:Id: %s" % (user.id,))
def test_hit_ratelimiterlockingexception(self) -> None: user = self.example_user('cordelia') RateLimitedUser(user).clear_history() with mock.patch('zerver.lib.rate_limiter.RedisRateLimiterBackend.incr_ratelimit', side_effect=RateLimiterLockingException): with self.assertLogs("zerver.lib.rate_limiter", level="WARNING") as m: result = self.send_api_message(user, "some stuff") self.assertEqual(result.status_code, 429) self.assertEqual( m.output, ["WARNING:zerver.lib.rate_limiter:Deadlock trying to incr_ratelimit for {}".format( f"RateLimitedUser:{user.id}:api_by_user")] )
def process_response(self, request: HttpRequest, response: HttpResponse) -> HttpResponse: if not settings.RATE_LIMITING: return response from zerver.lib.rate_limiter import max_api_calls, RateLimitedUser # Add X-RateLimit-*** headers if hasattr(request, '_ratelimit_applied_limits'): entity = RateLimitedUser(request.user) response['X-RateLimit-Limit'] = str(max_api_calls(entity)) if hasattr(request, '_ratelimit_secs_to_freedom'): response['X-RateLimit-Reset'] = str(int(time.time() + request._ratelimit_secs_to_freedom)) if hasattr(request, '_ratelimit_remaining'): response['X-RateLimit-Remaining'] = str(request._ratelimit_remaining) return response
def test_hit_ratelimiterlockingexception( self, mock_warn: mock.MagicMock) -> None: user = self.example_user('cordelia') RateLimitedUser(user).clear_history() with mock.patch( 'zerver.lib.rate_limiter.RedisRateLimiterBackend.incr_ratelimit', side_effect=RateLimiterLockingException): result = self.send_api_message(user, "some stuff") self.assertEqual(result.status_code, 429) mock_warn.assert_called_with( "Deadlock trying to incr_ratelimit for %s", f"RateLimitedUser:{user.id}:api_by_user", )
def process_response(self, request: HttpRequest, response: HttpResponse) -> HttpResponse: if not settings.RATE_LIMITING: return response from zerver.lib.rate_limiter import max_api_calls, RateLimitedUser # Add X-RateLimit-*** headers if hasattr(request, '_ratelimit'): # Right now, the only kind of limiting requests is user-based. ratelimit_user_results = request._ratelimit['RateLimitedUser'] entity = RateLimitedUser(request.user) response['X-RateLimit-Limit'] = str(max_api_calls(entity)) response['X-RateLimit-Reset'] = str(int(time.time() + ratelimit_user_results['secs_to_freedom'])) if 'remaining' in ratelimit_user_results: response['X-RateLimit-Remaining'] = str(ratelimit_user_results['remaining']) return response
def test_add_remove_rule(self) -> None: user_profile = self.example_user("hamlet") add_ratelimit_rule(1, 2) add_ratelimit_rule(4, 5, domain='some_new_domain') add_ratelimit_rule(10, 100, domain='some_new_domain') obj = RateLimitedUser(user_profile) self.assertEqual(obj.get_rules(), [(1, 2)]) obj.domain = 'some_new_domain' self.assertEqual(obj.get_rules(), [(4, 5), (10, 100)]) remove_ratelimit_rule(10, 100, domain='some_new_domain') self.assertEqual(obj.get_rules(), [(4, 5)])
def rate_limit_user(request: HttpRequest, user: UserProfile, domain: str) -> None: """Returns whether or not a user was rate limited. Will raise a RateLimited exception if the user has been rate limited, otherwise returns and modifies request to contain the rate limit information""" entity = RateLimitedUser(user, domain=domain) ratelimited, time = rate_limit_entity(entity) request._ratelimit_applied_limits = True request._ratelimit_secs_to_freedom = time request._ratelimit_over_limit = ratelimited # Abort this request if the user is over their rate limits if ratelimited: raise RateLimited() calls_remaining, time_reset = api_calls_left(entity) request._ratelimit_remaining = calls_remaining request._ratelimit_secs_to_freedom = time_reset
def rate_limit_user(request, user, domain): # type: (HttpRequest, UserProfile, Text) -> None """Returns whether or not a user was rate limited. Will raise a RateLimited exception if the user has been rate limited, otherwise returns and modifies request to contain the rate limit information""" entity = RateLimitedUser(user, domain=domain) ratelimited, time = is_ratelimited(entity) request._ratelimit_applied_limits = True request._ratelimit_secs_to_freedom = time request._ratelimit_over_limit = ratelimited # Abort this request if the user is over their rate limits if ratelimited: statsd.incr("ratelimiter.limited.%s.%s" % (type(user), user.id)) raise RateLimited() incr_ratelimit(entity) calls_remaining, time_reset = api_calls_left(entity) request._ratelimit_remaining = calls_remaining request._ratelimit_secs_to_freedom = time_reset
def test_hit_ratelimits(self) -> None: user = self.example_user("cordelia") RateLimitedUser(user).clear_history() start_time = time.time() for i in range(6): with mock.patch("time.time", return_value=(start_time + i * 0.1)): result = self.send_api_message(user, f"some stuff {i}") self.assertEqual(result.status_code, 429) json = result.json() self.assertEqual(json.get("result"), "error") self.assertIn("API usage exceeded rate limit", json.get("msg")) self.assertEqual(json.get("retry-after"), 0.5) self.assertTrue("Retry-After" in result) self.assertEqual(result["Retry-After"], "0.5") # We actually wait a second here, rather than force-clearing our history, # to make sure the rate-limiting code automatically forgives a user # after some time has passed. with mock.patch("time.time", return_value=(start_time + 1.01)): result = self.send_api_message(user, "Good message") self.assert_json_success(result)
def test_user_rate_limits(self) -> None: user_profile = self.example_user("hamlet") user_profile.rate_limits = "1:3,2:4" obj = RateLimitedUser(user_profile) self.assertEqual(obj.rules(), [(1, 3), (2, 4)])
def test_hit_ratelimits_as_user(self) -> None: user = self.example_user("cordelia") RateLimitedUser(user).clear_history() self.do_test_hit_ratelimits(lambda: self.send_api_message(user, "some stuff"))
def json_change_settings( request: HttpRequest, user_profile: UserProfile, full_name: str = REQ(default=""), email: str = REQ(default=""), old_password: str = REQ(default=""), new_password: str = REQ(default=""), twenty_four_hour_time: Optional[bool] = REQ(json_validator=check_bool, default=None), dense_mode: Optional[bool] = REQ(json_validator=check_bool, default=None), starred_message_counts: Optional[bool] = REQ(json_validator=check_bool, default=None), fluid_layout_width: Optional[bool] = REQ(json_validator=check_bool, default=None), high_contrast_mode: Optional[bool] = REQ(json_validator=check_bool, default=None), color_scheme: Optional[int] = REQ(json_validator=check_int_in( UserProfile.COLOR_SCHEME_CHOICES), default=None), translate_emoticons: Optional[bool] = REQ(json_validator=check_bool, default=None), default_language: Optional[str] = REQ(default=None), default_view: Optional[str] = REQ( str_validator=check_string_in(default_view_options), default=None), escape_navigates_to_default_view: Optional[bool] = REQ( json_validator=check_bool, default=None), left_side_userlist: Optional[bool] = REQ(json_validator=check_bool, default=None), emojiset: Optional[str] = REQ( str_validator=check_string_in(emojiset_choices), default=None), demote_inactive_streams: Optional[int] = REQ(json_validator=check_int_in( UserProfile.DEMOTE_STREAMS_CHOICES), default=None), timezone: Optional[str] = REQ(str_validator=check_string_in( pytz.all_timezones_set), default=None), email_notifications_batching_period_seconds: Optional[int] = REQ( json_validator=check_int, default=None), enable_drafts_synchronization: Optional[bool] = REQ( json_validator=check_bool, default=None), enable_stream_desktop_notifications: Optional[bool] = REQ( json_validator=check_bool, default=None), enable_stream_email_notifications: Optional[bool] = REQ( json_validator=check_bool, default=None), enable_stream_push_notifications: Optional[bool] = REQ( json_validator=check_bool, default=None), enable_stream_audible_notifications: Optional[bool] = REQ( json_validator=check_bool, default=None), wildcard_mentions_notify: Optional[bool] = REQ(json_validator=check_bool, default=None), notification_sound: Optional[str] = REQ(default=None), enable_desktop_notifications: Optional[bool] = REQ( json_validator=check_bool, default=None), enable_sounds: Optional[bool] = REQ(json_validator=check_bool, default=None), enable_offline_email_notifications: Optional[bool] = REQ( json_validator=check_bool, default=None), enable_offline_push_notifications: Optional[bool] = REQ( json_validator=check_bool, default=None), enable_online_push_notifications: Optional[bool] = REQ( json_validator=check_bool, default=None), enable_digest_emails: Optional[bool] = REQ(json_validator=check_bool, default=None), enable_login_emails: Optional[bool] = REQ(json_validator=check_bool, default=None), enable_marketing_emails: Optional[bool] = REQ(json_validator=check_bool, default=None), message_content_in_email_notifications: Optional[bool] = REQ( json_validator=check_bool, default=None), pm_content_in_desktop_notifications: Optional[bool] = REQ( json_validator=check_bool, default=None), desktop_icon_count_display: Optional[int] = REQ( json_validator=check_int_in( UserProfile.DESKTOP_ICON_COUNT_DISPLAY_CHOICES), default=None), realm_name_in_notifications: Optional[bool] = REQ( json_validator=check_bool, default=None), presence_enabled: Optional[bool] = REQ(json_validator=check_bool, default=None), enter_sends: Optional[bool] = REQ(json_validator=check_bool, default=None), send_private_typing_notifications: Optional[bool] = REQ( json_validator=check_bool, default=None), send_stream_typing_notifications: Optional[bool] = REQ( json_validator=check_bool, default=None), send_read_receipts: Optional[bool] = REQ(json_validator=check_bool, default=None), ) -> HttpResponse: if (default_language is not None or notification_sound is not None or email_notifications_batching_period_seconds is not None): check_settings_values(notification_sound, email_notifications_batching_period_seconds, default_language) if new_password != "": return_data: Dict[str, Any] = {} if email_belongs_to_ldap(user_profile.realm, user_profile.delivery_email): raise JsonableError(_("Your Zulip password is managed in LDAP")) try: if not authenticate( request, username=user_profile.delivery_email, password=old_password, realm=user_profile.realm, return_data=return_data, ): raise JsonableError(_("Wrong password!")) except RateLimited as e: assert e.secs_to_freedom is not None secs_to_freedom = int(e.secs_to_freedom) raise JsonableError( _("You're making too many attempts! Try again in {} seconds."). format(secs_to_freedom), ) if not check_password_strength(new_password): raise JsonableError(_("New password is too weak!")) do_change_password(user_profile, new_password) # Password changes invalidates sessions, see # https://docs.djangoproject.com/en/3.2/topics/auth/default/#session-invalidation-on-password-change # for details. To avoid this logging the user out of their own # session (which would provide a confusing UX at best), we # update the session hash here. update_session_auth_hash(request, user_profile) # We also save the session to the DB immediately to mitigate # race conditions. In theory, there is still a race condition # and to completely avoid it we will have to use some kind of # mutex lock in `django.contrib.auth.get_user` where session # is verified. To make that lock work we will have to control # the AuthenticationMiddleware which is currently controlled # by Django, request.session.save() result: Dict[str, Any] = {} new_email = email.strip() if user_profile.delivery_email != new_email and new_email != "": if user_profile.realm.email_changes_disabled and not user_profile.is_realm_admin: raise JsonableError( _("Email address changes are disabled in this organization.")) error = validate_email_is_valid( new_email, get_realm_email_validator(user_profile.realm), ) if error: raise JsonableError(error) try: validate_email_not_already_in_realm( user_profile.realm, new_email, verbose=False, ) except ValidationError as e: raise JsonableError(e.message) ratelimited, time_until_free = RateLimitedUser( user_profile, domain="email_change_by_user").rate_limit() if ratelimited: raise RateLimited(time_until_free) do_start_email_change_process(user_profile, new_email) if user_profile.full_name != full_name and full_name.strip() != "": if name_changes_disabled( user_profile.realm) and not user_profile.is_realm_admin: # Failingly silently is fine -- they can't do it through the UI, so # they'd have to be trying to break the rules. pass else: # Note that check_change_full_name strips the passed name automatically check_change_full_name(user_profile, full_name, user_profile) # Loop over user_profile.property_types request_settings = { k: v for k, v in list(locals().items()) if k in user_profile.property_types } for k, v in list(request_settings.items()): if v is not None and getattr(user_profile, k) != v: do_change_user_setting(user_profile, k, v, acting_user=user_profile) if timezone is not None and user_profile.timezone != timezone: do_change_user_setting(user_profile, "timezone", timezone, acting_user=user_profile) # TODO: Do this more generally. from zerver.lib.request import RequestNotes request_notes = RequestNotes.get_notes(request) for req_var in request.POST: if req_var not in request_notes.processed_parameters: request_notes.ignored_parameters.add(req_var) if len(request_notes.ignored_parameters) > 0: result["ignored_parameters_unsupported"] = list( request_notes.ignored_parameters) return json_success(result)