def _check_within_range(
self,
key: bytes,
count_func: Callable[[], int],
trim_func: Optional[Callable[[bytes, int], object]] = None,
) -> None:
user_id = int(key.split(b":")[2])
user = get_user_profile_by_id(user_id)
entity = RateLimitedUser(user)
max_calls = entity.max_api_calls()
age = int(client.ttl(key))
if age < 0:
logging.error("Found key with age of %s, will never expire: %s",
age, key)
count = count_func()
if count > max_calls:
logging.error(
"Redis health check found key with more elements \
than max_api_calls! (trying to trim) %s %s",
key,
count,
)
if trim_func is not None:
client.expire(key, entity.max_api_window())
trim_func(key, max_calls)
def handle(self, *args, **options):
# type: (*Any, **Any) -> None
if (not options['api_key'] and not options['email']) or \
(options['api_key'] and options['email']):
print("Please enter either an email or API key to manage")
exit(1)
realm = self.get_realm(options)
if options['email']:
user_profile = self.get_user(options['email'], realm)
else:
try:
user_profile = get_user_profile_by_api_key(options['api_key'])
except UserProfile.DoesNotExist:
print("Unable to get user profile for api key %s" %
(options['api_key'], ))
exit(1)
users = [user_profile]
if options['bots']:
users.extend(bot for bot in UserProfile.objects.filter(
is_bot=True, bot_owner=user_profile))
operation = options['operation']
for user in users:
print("Applying operation to User ID: %s: %s" %
(user.id, operation))
if operation == 'block':
block_access(RateLimitedUser(user, domain=options['domain']),
options['seconds'])
elif operation == 'unblock':
unblock_access(RateLimitedUser(user, domain=options['domain']))
def handle(self, *args: Any, **options: Any) -> None:
if (not options['api_key'] and not options['email']) or \
(options['api_key'] and options['email']):
raise CommandError("Please enter either an email or API key to manage")
realm = self.get_realm(options)
if options['email']:
user_profile = self.get_user(options['email'], realm)
else:
try:
user_profile = get_user_profile_by_api_key(options['api_key'])
except UserProfile.DoesNotExist:
raise CommandError("Unable to get user profile for api key {}".format(options['api_key']))
users = [user_profile]
if options['bots']:
users.extend(bot for bot in UserProfile.objects.filter(is_bot=True,
bot_owner=user_profile))
operation = options['operation']
for user in users:
print(f"Applying operation to User ID: {user.id}: {operation}")
if operation == 'block':
RateLimitedUser(user, domain=options['domain']).block_access(options['seconds'])
elif operation == 'unblock':
RateLimitedUser(user, domain=options['domain']).unblock_access()
def test_used_in_tornado(self) -> None:
user_profile = self.example_user("hamlet")
with self.settings(RUNNING_INSIDE_TORNADO=True):
obj = RateLimitedUser(user_profile, domain='api_by_user')
self.assertEqual(obj.backend, TornadoInMemoryRateLimiterBackend)
with self.settings(RUNNING_INSIDE_TORNADO=True):
obj = RateLimitedUser(user_profile, domain='some_domain')
self.assertEqual(obj.backend, RedisRateLimiterBackend)
def test_used_in_tornado(self) -> None:
user_profile = self.example_user("hamlet")
ip_addr = "192.168.0.123"
with self.settings(RUNNING_INSIDE_TORNADO=True):
user_obj = RateLimitedUser(user_profile, domain="api_by_user")
ip_obj = RateLimitedIPAddr(ip_addr, domain="api_by_ip")
self.assertEqual(user_obj.backend, TornadoInMemoryRateLimiterBackend)
self.assertEqual(ip_obj.backend, TornadoInMemoryRateLimiterBackend)
with self.settings(RUNNING_INSIDE_TORNADO=True):
user_obj = RateLimitedUser(user_profile, domain="some_domain")
ip_obj = RateLimitedIPAddr(ip_addr, domain="some_domain")
self.assertEqual(user_obj.backend, RedisRateLimiterBackend)
self.assertEqual(ip_obj.backend, RedisRateLimiterBackend)
def test_hit_ratelimits(self) -> None:
user = self.example_user('cordelia')
email = user.email
clear_history(RateLimitedUser(user))
start_time = time.time()
for i in range(6):
with mock.patch('time.time', return_value=(start_time + i * 0.1)):
result = self.send_api_message(email, "some stuff %s" % (i, ))
self.assertEqual(result.status_code, 429)
json = result.json()
self.assertEqual(json.get("result"), "error")
self.assertIn("API usage exceeded rate limit", json.get("msg"))
self.assertEqual(json.get('retry-after'), 0.5)
self.assertTrue('Retry-After' in result)
self.assertEqual(result['Retry-After'], '0.5')
# We actually wait a second here, rather than force-clearing our history,
# to make sure the rate-limiting code automatically forgives a user
# after some time has passed.
with mock.patch('time.time', return_value=(start_time + 1.0)):
result = self.send_api_message(email, "Good message")
self.assert_json_success(result)
def _check_within_range(
self,
key: str,
count_func: Callable[[], int],
trim_func: Optional[Callable[[str, int], None]] = None) -> None:
user_id = int(key.split(':')[1])
try:
user = get_user_profile_by_id(user_id)
except Exception:
user = None
entity = RateLimitedUser(user)
max_calls = max_api_calls(entity)
age = int(client.ttl(key))
if age < 0:
logging.error("Found key with age of %s, will never expire: %s" % (
age,
key,
))
count = count_func()
if count > max_calls:
logging.error("Redis health check found key with more elements \
than max_api_calls! (trying to trim) %s %s" % (key, count))
if trim_func is not None:
client.expire(key, max_api_window(entity))
trim_func(key, max_calls)
def rate_limit_user(request: HttpRequest, user: UserProfile, domain: str) -> None:
"""Returns whether or not a user was rate limited. Will raise a RateLimited exception
if the user has been rate limited, otherwise returns and modifies request to contain
the rate limit information"""
entity = RateLimitedUser(user, domain=domain)
ratelimited, time = is_ratelimited(entity)
request._ratelimit_applied_limits = True
request._ratelimit_secs_to_freedom = time
request._ratelimit_over_limit = ratelimited
# Abort this request if the user is over their rate limits
if ratelimited:
statsd.incr("ratelimiter.limited.%s.%s" % (type(user), user.id))
raise RateLimited()
try:
incr_ratelimit(entity)
except RateLimiterLockingException:
logging.warning("Deadlock trying to incr_ratelimit for %s on %s" % (
user.id, request.path))
# rate-limit users who are hitting the API so hard we can't update our stats.
raise RateLimited()
calls_remaining, time_reset = api_calls_left(entity)
request._ratelimit_remaining = calls_remaining
request._ratelimit_secs_to_freedom = time_reset
def rate_limit_user(request: HttpRequest, user: UserProfile,
domain: str) -> None:
"""Returns whether or not a user was rate limited. Will raise a RateLimited exception
if the user has been rate limited, otherwise returns and modifies request to contain
the rate limit information"""
RateLimitedUser(user, domain=domain).rate_limit_request(request)
def test_headers(self) -> None:
user = self.example_user("hamlet")
RateLimitedUser(user).clear_history()
result = self.send_api_message(user, "some stuff")
self.assertTrue("X-RateLimit-Remaining" in result)
self.assertTrue("X-RateLimit-Limit" in result)
self.assertTrue("X-RateLimit-Reset" in result)
def test_headers(self) -> None:
user = self.example_user('hamlet')
clear_history(RateLimitedUser(user))
result = self.send_api_message(user, "some stuff")
self.assertTrue('X-RateLimit-Remaining' in result)
self.assertTrue('X-RateLimit-Limit' in result)
self.assertTrue('X-RateLimit-Reset' in result)
def test_hit_change_email_ratelimit_as_user(self) -> None:
user = self.example_user("cordelia")
RateLimitedUser(user).clear_history()
emails = ["new-email-{n}@zulip.com" for n in range(1, 8)]
self.do_test_hit_ratelimits(
lambda: self.api_patch(user, "/api/v1/settings", {"email": emails.pop()}),
)
def test_hit_ratelimiterlockingexception(self) -> None:
user = self.example_user('cordelia')
email = user.email
clear_history(RateLimitedUser(user))
with mock.patch('zerver.decorator.incr_ratelimit',
side_effect=RateLimiterLockingException):
result = self.send_api_message(email, "some stuff")
self.assertEqual(result.status_code, 429)
def test_ratelimit_decrease(self) -> None:
user = self.example_user('hamlet')
clear_history(RateLimitedUser(user))
result = self.send_api_message(user, "some stuff")
limit = int(result['X-RateLimit-Remaining'])
result = self.send_api_message(user, "some stuff 2")
newlimit = int(result['X-RateLimit-Remaining'])
self.assertEqual(limit, newlimit + 1)
def test_hit_ratelimiterlockingexception(self, mock_warn: mock.MagicMock) -> None:
user = self.example_user('cordelia')
email = user.email
clear_history(RateLimitedUser(user))
with mock.patch('zerver.lib.rate_limiter.incr_ratelimit',
side_effect=RateLimiterLockingException):
result = self.send_api_message(email, "some stuff")
self.assertEqual(result.status_code, 429)
mock_warn.assert_called_with("Deadlock trying to incr_ratelimit for RateLimitedUser:Id: %s"
% (user.id,))
def test_hit_ratelimiterlockingexception(self) -> None:
user = self.example_user('cordelia')
RateLimitedUser(user).clear_history()
with mock.patch('zerver.lib.rate_limiter.RedisRateLimiterBackend.incr_ratelimit',
side_effect=RateLimiterLockingException):
with self.assertLogs("zerver.lib.rate_limiter", level="WARNING") as m:
result = self.send_api_message(user, "some stuff")
self.assertEqual(result.status_code, 429)
self.assertEqual(
m.output,
["WARNING:zerver.lib.rate_limiter:Deadlock trying to incr_ratelimit for {}".format(
f"RateLimitedUser:{user.id}:api_by_user")]
)
def process_response(self, request: HttpRequest, response: HttpResponse) -> HttpResponse:
if not settings.RATE_LIMITING:
return response
from zerver.lib.rate_limiter import max_api_calls, RateLimitedUser
# Add X-RateLimit-*** headers
if hasattr(request, '_ratelimit_applied_limits'):
entity = RateLimitedUser(request.user)
response['X-RateLimit-Limit'] = str(max_api_calls(entity))
if hasattr(request, '_ratelimit_secs_to_freedom'):
response['X-RateLimit-Reset'] = str(int(time.time() + request._ratelimit_secs_to_freedom))
if hasattr(request, '_ratelimit_remaining'):
response['X-RateLimit-Remaining'] = str(request._ratelimit_remaining)
return response
def test_hit_ratelimiterlockingexception(
self, mock_warn: mock.MagicMock) -> None:
user = self.example_user('cordelia')
RateLimitedUser(user).clear_history()
with mock.patch(
'zerver.lib.rate_limiter.RedisRateLimiterBackend.incr_ratelimit',
side_effect=RateLimiterLockingException):
result = self.send_api_message(user, "some stuff")
self.assertEqual(result.status_code, 429)
mock_warn.assert_called_with(
"Deadlock trying to incr_ratelimit for %s",
f"RateLimitedUser:{user.id}:api_by_user",
)
def process_response(self, request: HttpRequest, response: HttpResponse) -> HttpResponse:
if not settings.RATE_LIMITING:
return response
from zerver.lib.rate_limiter import max_api_calls, RateLimitedUser
# Add X-RateLimit-*** headers
if hasattr(request, '_ratelimit'):
# Right now, the only kind of limiting requests is user-based.
ratelimit_user_results = request._ratelimit['RateLimitedUser']
entity = RateLimitedUser(request.user)
response['X-RateLimit-Limit'] = str(max_api_calls(entity))
response['X-RateLimit-Reset'] = str(int(time.time() + ratelimit_user_results['secs_to_freedom']))
if 'remaining' in ratelimit_user_results:
response['X-RateLimit-Remaining'] = str(ratelimit_user_results['remaining'])
return response
def test_add_remove_rule(self) -> None:
user_profile = self.example_user("hamlet")
add_ratelimit_rule(1, 2)
add_ratelimit_rule(4, 5, domain='some_new_domain')
add_ratelimit_rule(10, 100, domain='some_new_domain')
obj = RateLimitedUser(user_profile)
self.assertEqual(obj.get_rules(), [(1, 2)])
obj.domain = 'some_new_domain'
self.assertEqual(obj.get_rules(), [(4, 5), (10, 100)])
remove_ratelimit_rule(10, 100, domain='some_new_domain')
self.assertEqual(obj.get_rules(), [(4, 5)])
def rate_limit_user(request: HttpRequest, user: UserProfile,
domain: str) -> None:
"""Returns whether or not a user was rate limited. Will raise a RateLimited exception
if the user has been rate limited, otherwise returns and modifies request to contain
the rate limit information"""
entity = RateLimitedUser(user, domain=domain)
ratelimited, time = rate_limit_entity(entity)
request._ratelimit_applied_limits = True
request._ratelimit_secs_to_freedom = time
request._ratelimit_over_limit = ratelimited
# Abort this request if the user is over their rate limits
if ratelimited:
raise RateLimited()
calls_remaining, time_reset = api_calls_left(entity)
request._ratelimit_remaining = calls_remaining
request._ratelimit_secs_to_freedom = time_reset
def rate_limit_user(request, user, domain):
# type: (HttpRequest, UserProfile, Text) -> None
"""Returns whether or not a user was rate limited. Will raise a RateLimited exception
if the user has been rate limited, otherwise returns and modifies request to contain
the rate limit information"""
entity = RateLimitedUser(user, domain=domain)
ratelimited, time = is_ratelimited(entity)
request._ratelimit_applied_limits = True
request._ratelimit_secs_to_freedom = time
request._ratelimit_over_limit = ratelimited
# Abort this request if the user is over their rate limits
if ratelimited:
statsd.incr("ratelimiter.limited.%s.%s" % (type(user), user.id))
raise RateLimited()
incr_ratelimit(entity)
calls_remaining, time_reset = api_calls_left(entity)
request._ratelimit_remaining = calls_remaining
request._ratelimit_secs_to_freedom = time_reset
def test_hit_ratelimits(self) -> None:
user = self.example_user("cordelia")
RateLimitedUser(user).clear_history()
start_time = time.time()
for i in range(6):
with mock.patch("time.time", return_value=(start_time + i * 0.1)):
result = self.send_api_message(user, f"some stuff {i}")
self.assertEqual(result.status_code, 429)
json = result.json()
self.assertEqual(json.get("result"), "error")
self.assertIn("API usage exceeded rate limit", json.get("msg"))
self.assertEqual(json.get("retry-after"), 0.5)
self.assertTrue("Retry-After" in result)
self.assertEqual(result["Retry-After"], "0.5")
# We actually wait a second here, rather than force-clearing our history,
# to make sure the rate-limiting code automatically forgives a user
# after some time has passed.
with mock.patch("time.time", return_value=(start_time + 1.01)):
result = self.send_api_message(user, "Good message")
self.assert_json_success(result)
def test_user_rate_limits(self) -> None:
user_profile = self.example_user("hamlet")
user_profile.rate_limits = "1:3,2:4"
obj = RateLimitedUser(user_profile)
self.assertEqual(obj.rules(), [(1, 3), (2, 4)])
def test_hit_ratelimits_as_user(self) -> None:
user = self.example_user("cordelia")
RateLimitedUser(user).clear_history()
self.do_test_hit_ratelimits(lambda: self.send_api_message(user, "some stuff"))
def json_change_settings(
request: HttpRequest,
user_profile: UserProfile,
full_name: str = REQ(default=""),
email: str = REQ(default=""),
old_password: str = REQ(default=""),
new_password: str = REQ(default=""),
twenty_four_hour_time: Optional[bool] = REQ(json_validator=check_bool,
default=None),
dense_mode: Optional[bool] = REQ(json_validator=check_bool, default=None),
starred_message_counts: Optional[bool] = REQ(json_validator=check_bool,
default=None),
fluid_layout_width: Optional[bool] = REQ(json_validator=check_bool,
default=None),
high_contrast_mode: Optional[bool] = REQ(json_validator=check_bool,
default=None),
color_scheme: Optional[int] = REQ(json_validator=check_int_in(
UserProfile.COLOR_SCHEME_CHOICES),
default=None),
translate_emoticons: Optional[bool] = REQ(json_validator=check_bool,
default=None),
default_language: Optional[str] = REQ(default=None),
default_view: Optional[str] = REQ(
str_validator=check_string_in(default_view_options), default=None),
escape_navigates_to_default_view: Optional[bool] = REQ(
json_validator=check_bool, default=None),
left_side_userlist: Optional[bool] = REQ(json_validator=check_bool,
default=None),
emojiset: Optional[str] = REQ(
str_validator=check_string_in(emojiset_choices), default=None),
demote_inactive_streams: Optional[int] = REQ(json_validator=check_int_in(
UserProfile.DEMOTE_STREAMS_CHOICES),
default=None),
timezone: Optional[str] = REQ(str_validator=check_string_in(
pytz.all_timezones_set),
default=None),
email_notifications_batching_period_seconds: Optional[int] = REQ(
json_validator=check_int, default=None),
enable_drafts_synchronization: Optional[bool] = REQ(
json_validator=check_bool, default=None),
enable_stream_desktop_notifications: Optional[bool] = REQ(
json_validator=check_bool, default=None),
enable_stream_email_notifications: Optional[bool] = REQ(
json_validator=check_bool, default=None),
enable_stream_push_notifications: Optional[bool] = REQ(
json_validator=check_bool, default=None),
enable_stream_audible_notifications: Optional[bool] = REQ(
json_validator=check_bool, default=None),
wildcard_mentions_notify: Optional[bool] = REQ(json_validator=check_bool,
default=None),
notification_sound: Optional[str] = REQ(default=None),
enable_desktop_notifications: Optional[bool] = REQ(
json_validator=check_bool, default=None),
enable_sounds: Optional[bool] = REQ(json_validator=check_bool,
default=None),
enable_offline_email_notifications: Optional[bool] = REQ(
json_validator=check_bool, default=None),
enable_offline_push_notifications: Optional[bool] = REQ(
json_validator=check_bool, default=None),
enable_online_push_notifications: Optional[bool] = REQ(
json_validator=check_bool, default=None),
enable_digest_emails: Optional[bool] = REQ(json_validator=check_bool,
default=None),
enable_login_emails: Optional[bool] = REQ(json_validator=check_bool,
default=None),
enable_marketing_emails: Optional[bool] = REQ(json_validator=check_bool,
default=None),
message_content_in_email_notifications: Optional[bool] = REQ(
json_validator=check_bool, default=None),
pm_content_in_desktop_notifications: Optional[bool] = REQ(
json_validator=check_bool, default=None),
desktop_icon_count_display: Optional[int] = REQ(
json_validator=check_int_in(
UserProfile.DESKTOP_ICON_COUNT_DISPLAY_CHOICES),
default=None),
realm_name_in_notifications: Optional[bool] = REQ(
json_validator=check_bool, default=None),
presence_enabled: Optional[bool] = REQ(json_validator=check_bool,
default=None),
enter_sends: Optional[bool] = REQ(json_validator=check_bool, default=None),
send_private_typing_notifications: Optional[bool] = REQ(
json_validator=check_bool, default=None),
send_stream_typing_notifications: Optional[bool] = REQ(
json_validator=check_bool, default=None),
send_read_receipts: Optional[bool] = REQ(json_validator=check_bool,
default=None),
) -> HttpResponse:
if (default_language is not None or notification_sound is not None
or email_notifications_batching_period_seconds is not None):
check_settings_values(notification_sound,
email_notifications_batching_period_seconds,
default_language)
if new_password != "":
return_data: Dict[str, Any] = {}
if email_belongs_to_ldap(user_profile.realm,
user_profile.delivery_email):
raise JsonableError(_("Your Zulip password is managed in LDAP"))
try:
if not authenticate(
request,
username=user_profile.delivery_email,
password=old_password,
realm=user_profile.realm,
return_data=return_data,
):
raise JsonableError(_("Wrong password!"))
except RateLimited as e:
assert e.secs_to_freedom is not None
secs_to_freedom = int(e.secs_to_freedom)
raise JsonableError(
_("You're making too many attempts! Try again in {} seconds.").
format(secs_to_freedom), )
if not check_password_strength(new_password):
raise JsonableError(_("New password is too weak!"))
do_change_password(user_profile, new_password)
# Password changes invalidates sessions, see
# https://docs.djangoproject.com/en/3.2/topics/auth/default/#session-invalidation-on-password-change
# for details. To avoid this logging the user out of their own
# session (which would provide a confusing UX at best), we
# update the session hash here.
update_session_auth_hash(request, user_profile)
# We also save the session to the DB immediately to mitigate
# race conditions. In theory, there is still a race condition
# and to completely avoid it we will have to use some kind of
# mutex lock in `django.contrib.auth.get_user` where session
# is verified. To make that lock work we will have to control
# the AuthenticationMiddleware which is currently controlled
# by Django,
request.session.save()
result: Dict[str, Any] = {}
new_email = email.strip()
if user_profile.delivery_email != new_email and new_email != "":
if user_profile.realm.email_changes_disabled and not user_profile.is_realm_admin:
raise JsonableError(
_("Email address changes are disabled in this organization."))
error = validate_email_is_valid(
new_email,
get_realm_email_validator(user_profile.realm),
)
if error:
raise JsonableError(error)
try:
validate_email_not_already_in_realm(
user_profile.realm,
new_email,
verbose=False,
)
except ValidationError as e:
raise JsonableError(e.message)
ratelimited, time_until_free = RateLimitedUser(
user_profile, domain="email_change_by_user").rate_limit()
if ratelimited:
raise RateLimited(time_until_free)
do_start_email_change_process(user_profile, new_email)
if user_profile.full_name != full_name and full_name.strip() != "":
if name_changes_disabled(
user_profile.realm) and not user_profile.is_realm_admin:
# Failingly silently is fine -- they can't do it through the UI, so
# they'd have to be trying to break the rules.
pass
else:
# Note that check_change_full_name strips the passed name automatically
check_change_full_name(user_profile, full_name, user_profile)
# Loop over user_profile.property_types
request_settings = {
k: v
for k, v in list(locals().items()) if k in user_profile.property_types
}
for k, v in list(request_settings.items()):
if v is not None and getattr(user_profile, k) != v:
do_change_user_setting(user_profile,
k,
v,
acting_user=user_profile)
if timezone is not None and user_profile.timezone != timezone:
do_change_user_setting(user_profile,
"timezone",
timezone,
acting_user=user_profile)
# TODO: Do this more generally.
from zerver.lib.request import RequestNotes
request_notes = RequestNotes.get_notes(request)
for req_var in request.POST:
if req_var not in request_notes.processed_parameters:
request_notes.ignored_parameters.add(req_var)
if len(request_notes.ignored_parameters) > 0:
result["ignored_parameters_unsupported"] = list(
request_notes.ignored_parameters)
return json_success(result)
