def func_with_caching(*args, **kwargs): # type: (*Any, **Any) -> Callable[..., Any] key = keyfunc(*args, **kwargs) val = cache_get(key, cache_name=cache_name) extra = "" if cache_name == 'database': extra = ".dbcache" if with_statsd_key is not None: metric_key = with_statsd_key else: metric_key = statsd_key(key) status = "hit" if val is not None else "miss" statsd.incr("cache%s.%s.%s" % (extra, metric_key, status)) # Values are singleton tuples so that we can distinguish # a result of None from a missing key. if val is not None: return val[0] val = func(*args, **kwargs) cache_set(key, val, cache_name=cache_name, timeout=timeout) return val
def report_send_times(request: HttpRequest, user_profile: UserProfile, time: int=REQ(converter=to_non_negative_int), received: int=REQ(converter=to_non_negative_int, default=-1), displayed: int=REQ(converter=to_non_negative_int, default=-1), locally_echoed: bool=REQ(validator=check_bool, default=False), rendered_content_disparity: bool=REQ(validator=check_bool, default=False)) -> HttpResponse: received_str = "(unknown)" if received > 0: received_str = str(received) displayed_str = "(unknown)" if displayed > 0: displayed_str = str(displayed) request._log_data["extra"] = f"[{time}ms/{received_str}ms/{displayed_str}ms/echo:{locally_echoed}/diff:{rendered_content_disparity}]" base_key = statsd_key(user_profile.realm.string_id, clean_periods=True) statsd.timing(f"endtoend.send_time.{base_key}", time) if received > 0: statsd.timing(f"endtoend.receive_time.{base_key}", received) if displayed > 0: statsd.timing(f"endtoend.displayed_time.{base_key}", displayed) if locally_echoed: statsd.incr('locally_echoed') if rendered_content_disparity: statsd.incr('render_disparity') return json_success()
def json_report_send_time(request, user_profile, time=REQ(converter=to_non_negative_int), received=REQ(converter=to_non_negative_int, default="(unknown)"), displayed=REQ(converter=to_non_negative_int, default="(unknown)"), locally_echoed=REQ(validator=check_bool, default=False), rendered_content_disparity=REQ(validator=check_bool, default=False)): # type: (HttpRequest, UserProfile, int, int, int, bool, bool) -> HttpResponse request._log_data["extra"] = "[%sms/%sms/%sms/echo:%s/diff:%s]" \ % (time, received, displayed, locally_echoed, rendered_content_disparity) base_key = statsd_key(user_profile.realm.string_id, clean_periods=True) statsd.timing("endtoend.send_time.%s" % (base_key, ), time) if received != "(unknown)": statsd.timing("endtoend.receive_time.%s" % (base_key, ), received) if displayed != "(unknown)": statsd.timing("endtoend.displayed_time.%s" % (base_key, ), displayed) if locally_echoed: statsd.incr('locally_echoed') if rendered_content_disparity: statsd.incr('render_disparity') return json_success()
def func_with_caching(*args: object, **kwargs: object) -> object: key = keyfunc(*args, **kwargs) try: val = cache_get(key, cache_name=cache_name) except InvalidCacheKeyException: stack_trace = traceback.format_exc() log_invalid_cache_keys(stack_trace, [key]) return func(*args, **kwargs) extra = "" if cache_name == 'database': extra = ".dbcache" if with_statsd_key is not None: metric_key = with_statsd_key else: metric_key = statsd_key(key) status = "hit" if val is not None else "miss" statsd.incr(f"cache{extra}.{metric_key}.{status}") # Values are singleton tuples so that we can distinguish # a result of None from a missing key. if val is not None: return val[0] val = func(*args, **kwargs) cache_set(key, val, cache_name=cache_name, timeout=timeout) return val
def report_send_times(request: HttpRequest, user_profile: UserProfile, time: int=REQ(converter=to_non_negative_int), received: int=REQ(converter=to_non_negative_int, default=-1), displayed: int=REQ(converter=to_non_negative_int, default=-1), locally_echoed: bool=REQ(validator=check_bool, default=False), rendered_content_disparity: bool=REQ(validator=check_bool, default=False)) -> HttpResponse: received_str = "(unknown)" if received > 0: received_str = str(received) displayed_str = "(unknown)" if displayed > 0: displayed_str = str(displayed) request._log_data["extra"] = "[%sms/%sms/%sms/echo:%s/diff:%s]" \ % (time, received_str, displayed_str, locally_echoed, rendered_content_disparity) base_key = statsd_key(user_profile.realm.string_id, clean_periods=True) statsd.timing("endtoend.send_time.%s" % (base_key,), time) if received > 0: statsd.timing("endtoend.receive_time.%s" % (base_key,), received) if displayed > 0: statsd.timing("endtoend.displayed_time.%s" % (base_key,), displayed) if locally_echoed: statsd.incr('locally_echoed') if rendered_content_disparity: statsd.incr('render_disparity') return json_success()
def rate_limit_user(request: HttpRequest, user: UserProfile, domain: str) -> None: """Returns whether or not a user was rate limited. Will raise a RateLimited exception if the user has been rate limited, otherwise returns and modifies request to contain the rate limit information""" entity = RateLimitedUser(user, domain=domain) ratelimited, time = is_ratelimited(entity) request._ratelimit_applied_limits = True request._ratelimit_secs_to_freedom = time request._ratelimit_over_limit = ratelimited # Abort this request if the user is over their rate limits if ratelimited: statsd.incr("ratelimiter.limited.%s.%s" % (type(user), user.id)) raise RateLimited() try: incr_ratelimit(entity) except RateLimiterLockingException: # nocoverage # Should add on next rate limit pass logging.warning("Deadlock trying to incr_ratelimit for %s on %s" % ( user.id, request.path)) # rate-limit users who are hitting the API so hard we can't update our stats. raise RateLimited() calls_remaining, time_reset = api_calls_left(entity) request._ratelimit_remaining = calls_remaining request._ratelimit_secs_to_freedom = time_reset
def rate_limit_user(request: HttpRequest, user: UserProfile, domain: str) -> None: """Returns whether or not a user was rate limited. Will raise a RateLimited exception if the user has been rate limited, otherwise returns and modifies request to contain the rate limit information""" entity = RateLimitedUser(user, domain=domain) ratelimited, time = is_ratelimited(entity) request._ratelimit_applied_limits = True request._ratelimit_secs_to_freedom = time request._ratelimit_over_limit = ratelimited # Abort this request if the user is over their rate limits if ratelimited: statsd.incr("ratelimiter.limited.%s.%s" % (type(user), user.id)) raise RateLimited() try: incr_ratelimit(entity) except RateLimiterLockingException: logging.warning("Deadlock trying to incr_ratelimit for %s on %s" % ( user.id, request.path)) # rate-limit users who are hitting the API so hard we can't update our stats. raise RateLimited() calls_remaining, time_reset = api_calls_left(entity) request._ratelimit_remaining = calls_remaining request._ratelimit_secs_to_freedom = time_reset
def do_publish() -> None: self.channel.basic_publish( exchange='', routing_key=queue_name, properties=pika.BasicProperties(delivery_mode=2), body=body) statsd.incr("rabbitmq.publish.%s" % (queue_name, ))
def do_publish() -> None: self.channel.basic_publish( exchange='', routing_key=queue_name, properties=pika.BasicProperties(delivery_mode=2), body=body) statsd.incr("rabbitmq.publish.%s" % (queue_name,))
def do_publish(channel: BlockingChannel) -> None: channel.basic_publish( exchange='', routing_key=queue_name, properties=pika.BasicProperties(delivery_mode=2), body=body) statsd.incr(f"rabbitmq.publish.{queue_name}")
def rate_limit_entity(cls, entity_key: str, rules: List[Tuple[int, int]], max_api_calls: int, max_api_window: int) -> Tuple[bool, float]: ratelimited, time = cls.is_ratelimited(entity_key, rules) if ratelimited: statsd.incr("ratelimiter.limited.%s" % (entity_key,)) else: try: cls.incr_ratelimit(entity_key, rules, max_api_calls, max_api_window) except RateLimiterLockingException: logger.warning("Deadlock trying to incr_ratelimit for %s" % (entity_key,)) # rate-limit users who are hitting the API so hard we can't update our stats. ratelimited = True return ratelimited, time
def rate_limit_mirror_by_realm(recipient_realm: Realm) -> None: # Code based on the rate_limit_user function: entity = RateLimitedRealmMirror(recipient_realm) ratelimited, time = is_ratelimited(entity) if ratelimited: statsd.incr("ratelimiter.limited.%s.%s" % (type(recipient_realm), recipient_realm.id)) raise RateLimited() try: incr_ratelimit(entity) except RateLimiterLockingException: logger.warning("Email mirror rate limiter: Deadlock trying to " "incr_ratelimit for realm %s" % (recipient_realm.name, )) raise RateLimited()
def rate_limit_entity(entity: RateLimitedObject) -> Tuple[bool, float]: # Returns (ratelimited, secs_to_freedom) ratelimited, time = is_ratelimited(entity) if ratelimited: statsd.incr("ratelimiter.limited.%s.%s" % (type(entity), str(entity))) else: try: incr_ratelimit(entity) except RateLimiterLockingException: logger.warning("Deadlock trying to incr_ratelimit for %s:%s" % (type(entity).__name__, str(entity))) # rate-limit users who are hitting the API so hard we can't update our stats. ratelimited = True return ratelimited, time
def json_report_send_time(request, user_profile, time=REQ(converter=to_non_negative_int), received=REQ(converter=to_non_negative_int, default="(unknown)"), displayed=REQ(converter=to_non_negative_int, default="(unknown)"), locally_echoed=REQ(validator=check_bool, default=False), rendered_content_disparity=REQ(validator=check_bool, default=False)): request._log_data["extra"] = "[%sms/%sms/%sms/echo:%s/diff:%s]" \ % (time, received, displayed, locally_echoed, rendered_content_disparity) statsd.timing("endtoend.send_time.%s" % (statsd_key(user_profile.realm.domain, clean_periods=True),), time) if received != "(unknown)": statsd.timing("endtoend.receive_time.%s" % (statsd_key(user_profile.realm.domain, clean_periods=True),), received) if displayed != "(unknown)": statsd.timing("endtoend.displayed_time.%s" % (statsd_key(user_profile.realm.domain, clean_periods=True),), displayed) if locally_echoed: statsd.incr('locally_echoed') if rendered_content_disparity: statsd.incr('render_disparity') return json_success()
def rate_limit_user(request, user, domain): """Returns whether or not a user was rate limited. Will raise a RateLimited exception if the user has been rate limited, otherwise returns and modifies request to contain the rate limit information""" ratelimited, time = is_ratelimited(user, domain) request._ratelimit_applied_limits = True request._ratelimit_secs_to_freedom = time request._ratelimit_over_limit = ratelimited # Abort this request if the user is over her rate limits if ratelimited: statsd.incr("ratelimiter.limited.%s.%s" % (type(user), user.id)) raise RateLimited() incr_ratelimit(user, domain) calls_remaining, time_reset = api_calls_left(user, domain) request._ratelimit_remaining = calls_remaining request._ratelimit_secs_to_freedom = time_reset
def rate_limit_entity(cls, entity_key: str, rules: List[Tuple[int, int]], max_api_calls: int, max_api_window: int) -> Tuple[bool, float]: now = time.time() if entity_key in cls.timestamps_blocked_until: # Check whether the key is manually blocked. if now < cls.timestamps_blocked_until[entity_key]: blocking_ttl = cls.timestamps_blocked_until[entity_key] - now return True, blocking_ttl else: del cls.timestamps_blocked_until[entity_key] assert rules for time_window, max_count in rules: ratelimited, time_till_free = cls.need_to_limit( entity_key, time_window, max_count) if ratelimited: statsd.incr("ratelimiter.limited.%s" % (entity_key, )) break return ratelimited, time_till_free
def wrapped_func(*args: object, **kwargs: object) -> object: ret = func(*args, **kwargs) statsd.incr(counter, val) return ret
def home_real(request): # type: (HttpRequest) -> HttpResponse # We need to modify the session object every two weeks or it will expire. # This line makes reloading the page a sufficient action to keep the # session alive. request.session.modified = True user_profile = request.user # If a user hasn't signed the current Terms of Service, send them there if settings.TERMS_OF_SERVICE is not None and settings.TOS_VERSION is not None and \ int(settings.TOS_VERSION.split('.')[0]) > user_profile.major_tos_version(): return accounts_accept_terms(request) narrow = [] # type: List[List[Text]] narrow_stream = None narrow_topic = request.GET.get("topic") if request.GET.get("stream"): try: narrow_stream_name = request.GET.get("stream") (narrow_stream, ignored_rec, ignored_sub) = access_stream_by_name(user_profile, narrow_stream_name) narrow = [["stream", narrow_stream.name]] except Exception: logging.exception("Narrow parsing") if narrow_stream is not None and narrow_topic is not None: narrow.append(["topic", narrow_topic]) register_ret = do_events_register(user_profile, request.client, apply_markdown=True, narrow=narrow) user_has_messages = (register_ret['max_message_id'] != -1) # Reset our don't-spam-users-with-email counter since the # user has since logged in if user_profile.last_reminder is not None: user_profile.last_reminder = None user_profile.save(update_fields=["last_reminder"]) # Brand new users get the tutorial needs_tutorial = settings.TUTORIAL_ENABLED and \ user_profile.tutorial_status != UserProfile.TUTORIAL_FINISHED first_in_realm = realm_user_count(user_profile.realm) == 1 # If you are the only person in the realm and you didn't invite # anyone, we'll continue to encourage you to do so on the frontend. prompt_for_invites = first_in_realm and \ not PreregistrationUser.objects.filter(referred_by=user_profile).count() if user_profile.pointer == -1 and user_has_messages: # Put the new user's pointer at the bottom # # This improves performance, because we limit backfilling of messages # before the pointer. It's also likely that someone joining an # organization is interested in recent messages more than the very # first messages on the system. register_ret['pointer'] = register_ret['max_message_id'] user_profile.last_pointer_updater = request.session.session_key if user_profile.pointer == -1: latest_read = None else: try: latest_read = UserMessage.objects.get( user_profile=user_profile, message__id=user_profile.pointer) except UserMessage.DoesNotExist: # Don't completely fail if your saved pointer ID is invalid logging.warning("%s has invalid pointer %s" % (user_profile.email, user_profile.pointer)) latest_read = None desktop_notifications_enabled = user_profile.enable_desktop_notifications if narrow_stream is not None: desktop_notifications_enabled = False if user_profile.realm.notifications_stream: notifications_stream = user_profile.realm.notifications_stream.name else: notifications_stream = "" # Set default language and make it persist default_language = register_ret['default_language'] url_lang = '/{}'.format(request.LANGUAGE_CODE) if not request.path.startswith(url_lang): translation.activate(default_language) request.session[translation.LANGUAGE_SESSION_KEY] = default_language # Pass parameters to the client-side JavaScript code. # These end up in a global JavaScript Object named 'page_params'. page_params = dict( # Server settings. share_the_love=settings.SHARE_THE_LOVE, development_environment=settings.DEVELOPMENT, debug_mode=settings.DEBUG, test_suite=settings.TEST_SUITE, poll_timeout=settings.POLL_TIMEOUT, login_page=settings.HOME_NOT_LOGGED_IN, server_uri=settings.SERVER_URI, maxfilesize=settings.MAX_FILE_UPLOAD_SIZE, server_generation=settings.SERVER_GENERATION, use_websockets=settings.USE_WEBSOCKETS, save_stacktraces=settings.SAVE_FRONTEND_STACKTRACES, # realm data. # TODO: Move all of these data to register_ret and pull from there realm_uri=user_profile.realm.uri, password_auth_enabled=password_auth_enabled(user_profile.realm), domain=user_profile.realm.domain, domains=list_of_domains_for_realm(user_profile.realm), realm_icon_url=realm_icon_url(user_profile.realm), realm_icon_source=user_profile.realm.icon_source, name_changes_disabled=name_changes_disabled(user_profile.realm), mandatory_topics=user_profile.realm.mandatory_topics, show_digest_email=user_profile.realm.show_digest_email, realm_presence_disabled=user_profile.realm.presence_disabled, is_zephyr_mirror_realm=user_profile.realm.is_zephyr_mirror_realm, # user_profile data. # TODO: Move all of these data to register_ret and pull from there fullname=user_profile.full_name, email=user_profile.email, enter_sends=user_profile.enter_sends, user_id=user_profile.id, is_admin=user_profile.is_realm_admin, can_create_streams=user_profile.can_create_streams(), autoscroll_forever=user_profile.autoscroll_forever, default_desktop_notifications=user_profile. default_desktop_notifications, avatar_url=avatar_url(user_profile), avatar_url_medium=avatar_url(user_profile, medium=True), avatar_source=user_profile.avatar_source, # Stream message notification settings: stream_desktop_notifications_enabled=user_profile. enable_stream_desktop_notifications, stream_sounds_enabled=user_profile.enable_stream_sounds, # Private message and @-mention notification settings: desktop_notifications_enabled=desktop_notifications_enabled, sounds_enabled=user_profile.enable_sounds, enable_offline_email_notifications=user_profile. enable_offline_email_notifications, pm_content_in_desktop_notifications=user_profile. pm_content_in_desktop_notifications, enable_offline_push_notifications=user_profile. enable_offline_push_notifications, enable_online_push_notifications=user_profile. enable_online_push_notifications, enable_digest_emails=user_profile.enable_digest_emails, # Realm foreign key data from register_ret. # TODO: Rename these to match register_ret values. subbed_info=register_ret['subscriptions'], unsubbed_info=register_ret['unsubscribed'], neversubbed_info=register_ret['never_subscribed'], people_list=register_ret['realm_users'], bot_list=register_ret['realm_bots'], initial_pointer=register_ret['pointer'], initial_presences=register_ret['presences'], event_queue_id=register_ret['queue_id'], # Misc. extra data. have_initial_messages=user_has_messages, initial_servertime=time.time( ), # Used for calculating relative presence age default_language_name=get_language_name( register_ret['default_language']), language_list_dbl_col=get_language_list_for_templates( register_ret['default_language']), language_list=get_language_list(), needs_tutorial=needs_tutorial, first_in_realm=first_in_realm, prompt_for_invites=prompt_for_invites, notifications_stream=notifications_stream, cross_realm_bots=list(get_cross_realm_dicts()), unread_count=approximate_unread_count(user_profile), furthest_read_time=sent_time_in_epoch_seconds(latest_read), has_mobile_devices=num_push_devices_for_user(user_profile) > 0, ) # These fields will be automatically copied from register_ret into # page_params. It is a goal to move more of the page_params list # into this sort of cleaner structure. page_params_core_fields = [ 'alert_words', 'attachments', 'default_language', 'last_event_id', 'left_side_userlist', 'max_message_id', 'muted_topics', 'realm_add_emoji_by_admins_only', 'realm_allow_message_editing', 'realm_authentication_methods', 'realm_create_stream_by_admins_only', 'realm_default_language', 'realm_default_streams', 'realm_emoji', 'realm_message_content_edit_limit_seconds', 'realm_name', 'realm_invite_by_admins_only', 'realm_invite_required', 'realm_filters', 'realm_restricted_to_domain', 'realm_waiting_period_threshold', 'referrals', 'twenty_four_hour_time', 'zulip_version', ] for field_name in page_params_core_fields: page_params[field_name] = register_ret[field_name] if narrow_stream is not None: # In narrow_stream context, initial pointer is just latest message recipient = get_recipient(Recipient.STREAM, narrow_stream.id) try: initial_pointer = Message.objects.filter( recipient=recipient).order_by('id').reverse()[0].id except IndexError: initial_pointer = -1 page_params["narrow_stream"] = narrow_stream.name if narrow_topic is not None: page_params["narrow_topic"] = narrow_topic page_params["narrow"] = [ dict(operator=term[0], operand=term[1]) for term in narrow ] page_params["max_message_id"] = initial_pointer page_params["initial_pointer"] = initial_pointer page_params["have_initial_messages"] = (initial_pointer != -1) statsd.incr('views.home') show_invites = True # Some realms only allow admins to invite users if user_profile.realm.invite_by_admins_only and not user_profile.is_realm_admin: show_invites = False product_name = "Zulip" page_params['product_name'] = product_name request._log_data['extra'] = "[%s]" % (register_ret["queue_id"], ) response = render_to_response('zerver/index.html', { 'user_profile': user_profile, 'page_params': simplejson.encoder.JSONEncoderForHTML().encode(page_params), 'nofontface': is_buggy_ua(request.META.get("HTTP_USER_AGENT", "Unspecified")), 'avatar_url': avatar_url(user_profile), 'show_debug': settings.DEBUG and ('show_debug' in request.GET), 'pipeline': settings.PIPELINE_ENABLED, 'show_invites': show_invites, 'is_admin': user_profile.is_realm_admin, 'show_webathena': user_profile.realm.webathena_enabled, 'enable_feedback': settings.ENABLE_FEEDBACK, 'embedded': narrow_stream is not None, 'product_name': product_name }, request=request) patch_cache_control(response, no_cache=True, no_store=True, must_revalidate=True) return response
def write_log_line( log_data: MutableMapping[str, Any], path: str, method: str, remote_ip: str, requestor_for_logs: str, client_name: str, status_code: int = 200, error_content: Optional[AnyStr] = None, error_content_iter: Optional[Iterable[AnyStr]] = None) -> None: assert error_content is None or error_content_iter is None if error_content is not None: error_content_iter = (error_content, ) if settings.STATSD_HOST != '': # For statsd timer name if path == '/': statsd_path = 'webreq' else: statsd_path = "webreq.{}".format(path[1:].replace('/', '.')) # Remove non-ascii chars from path (there should be none, if there are it's # because someone manually entered a nonexistent path), as UTF-8 chars make # statsd sad when it sends the key name over the socket statsd_path = statsd_path.encode('ascii', errors='ignore').decode("ascii") # TODO: This could probably be optimized to use a regular expression rather than a loop. suppress_statsd = any(blacklisted in statsd_path for blacklisted in statsd_blacklisted_requests) else: suppress_statsd = True statsd_path = '' time_delta = -1 # A time duration of -1 means the StartLogRequests middleware # didn't run for some reason optional_orig_delta = "" if 'time_started' in log_data: time_delta = time.time() - log_data['time_started'] if 'time_stopped' in log_data: orig_time_delta = time_delta time_delta = ((log_data['time_stopped'] - log_data['time_started']) + (time.time() - log_data['time_restarted'])) optional_orig_delta = f" (lp: {format_timedelta(orig_time_delta)})" remote_cache_output = "" if 'remote_cache_time_start' in log_data: remote_cache_time_delta = get_remote_cache_time( ) - log_data['remote_cache_time_start'] remote_cache_count_delta = get_remote_cache_requests( ) - log_data['remote_cache_requests_start'] if 'remote_cache_requests_stopped' in log_data: # (now - restarted) + (stopped - start) = (now - start) + (stopped - restarted) remote_cache_time_delta += ( log_data['remote_cache_time_stopped'] - log_data['remote_cache_time_restarted']) remote_cache_count_delta += ( log_data['remote_cache_requests_stopped'] - log_data['remote_cache_requests_restarted']) if (remote_cache_time_delta > 0.005): remote_cache_output = f" (mem: {format_timedelta(remote_cache_time_delta)}/{remote_cache_count_delta})" if not suppress_statsd: statsd.timing(f"{statsd_path}.remote_cache.time", timedelta_ms(remote_cache_time_delta)) statsd.incr(f"{statsd_path}.remote_cache.querycount", remote_cache_count_delta) startup_output = "" if 'startup_time_delta' in log_data and log_data[ "startup_time_delta"] > 0.005: startup_output = " (+start: {})".format( format_timedelta(log_data["startup_time_delta"])) markdown_output = "" if 'markdown_time_start' in log_data: markdown_time_delta = get_markdown_time( ) - log_data['markdown_time_start'] markdown_count_delta = get_markdown_requests( ) - log_data['markdown_requests_start'] if 'markdown_requests_stopped' in log_data: # (now - restarted) + (stopped - start) = (now - start) + (stopped - restarted) markdown_time_delta += (log_data['markdown_time_stopped'] - log_data['markdown_time_restarted']) markdown_count_delta += (log_data['markdown_requests_stopped'] - log_data['markdown_requests_restarted']) if (markdown_time_delta > 0.005): markdown_output = f" (md: {format_timedelta(markdown_time_delta)}/{markdown_count_delta})" if not suppress_statsd: statsd.timing(f"{statsd_path}.markdown.time", timedelta_ms(markdown_time_delta)) statsd.incr(f"{statsd_path}.markdown.count", markdown_count_delta) # Get the amount of time spent doing database queries db_time_output = "" queries = connection.connection.queries if connection.connection is not None else [] if len(queries) > 0: query_time = sum(float(query.get('time', 0)) for query in queries) db_time_output = f" (db: {format_timedelta(query_time)}/{len(queries)}q)" if not suppress_statsd: # Log ms, db ms, and num queries to statsd statsd.timing(f"{statsd_path}.dbtime", timedelta_ms(query_time)) statsd.incr(f"{statsd_path}.dbq", len(queries)) statsd.timing(f"{statsd_path}.total", timedelta_ms(time_delta)) if 'extra' in log_data: extra_request_data = " {}".format(log_data['extra']) else: extra_request_data = "" logger_client = f"({requestor_for_logs} via {client_name})" logger_timing = f'{format_timedelta(time_delta):>5}{optional_orig_delta}{remote_cache_output}{markdown_output}{db_time_output}{startup_output} {path}' logger_line = f'{remote_ip:<15} {method:<7} {status_code:3} {logger_timing}{extra_request_data} {logger_client}' if (status_code in [200, 304] and method == "GET" and path.startswith("/static")): logger.debug(logger_line) else: logger.info(logger_line) if (is_slow_query(time_delta, path)): slow_query_logger.info(logger_line) if settings.PROFILE_ALL_REQUESTS: log_data["prof"].disable() profile_path = "/tmp/profile.data.{}.{}".format( path.split("/")[-1], int(time_delta * 1000)) log_data["prof"].dump_stats(profile_path) # Log some additional data whenever we return certain 40x errors if 400 <= status_code < 500 and status_code not in [401, 404, 405]: assert error_content_iter is not None error_content_list = list(error_content_iter) if not error_content_list: error_data = '' elif isinstance(error_content_list[0], str): error_data = ''.join(error_content_list) elif isinstance(error_content_list[0], bytes): error_data = repr(b''.join(error_content_list)) if len(error_data) > 200: error_data = "[content more than 200 characters]" logger.info('status=%3d, data=%s, uid=%s', status_code, error_data, requestor_for_logs)
def home_real(request): # type: (HttpRequest) -> HttpResponse # We need to modify the session object every two weeks or it will expire. # This line makes reloading the page a sufficient action to keep the # session alive. request.session.modified = True user_profile = request.user # If a user hasn't signed the current Terms of Service, send them there if settings.TERMS_OF_SERVICE is not None and settings.TOS_VERSION is not None and \ int(settings.TOS_VERSION.split('.')[0]) > user_profile.major_tos_version(): return accounts_accept_terms(request) narrow = [] # type: List[List[Text]] narrow_stream = None narrow_topic = request.GET.get("topic") if request.GET.get("stream"): try: narrow_stream_name = request.GET.get("stream") (narrow_stream, ignored_rec, ignored_sub) = access_stream_by_name( user_profile, narrow_stream_name) narrow = [["stream", narrow_stream.name]] except Exception: logging.exception("Narrow parsing") if narrow_stream is not None and narrow_topic is not None: narrow.append(["topic", narrow_topic]) register_ret = do_events_register(user_profile, request.client, apply_markdown=True, narrow=narrow) user_has_messages = (register_ret['max_message_id'] != -1) # Reset our don't-spam-users-with-email counter since the # user has since logged in if user_profile.last_reminder is not None: user_profile.last_reminder = None user_profile.save(update_fields=["last_reminder"]) # Brand new users get narrowed to PM with welcome-bot needs_tutorial = user_profile.tutorial_status == UserProfile.TUTORIAL_WAITING first_in_realm = realm_user_count(user_profile.realm) == 1 # If you are the only person in the realm and you didn't invite # anyone, we'll continue to encourage you to do so on the frontend. prompt_for_invites = first_in_realm and \ not PreregistrationUser.objects.filter(referred_by=user_profile).count() if user_profile.pointer == -1 and user_has_messages: # Put the new user's pointer at the bottom # # This improves performance, because we limit backfilling of messages # before the pointer. It's also likely that someone joining an # organization is interested in recent messages more than the very # first messages on the system. register_ret['pointer'] = register_ret['max_message_id'] user_profile.last_pointer_updater = request.session.session_key if user_profile.pointer == -1: latest_read = None else: try: latest_read = UserMessage.objects.get(user_profile=user_profile, message__id=user_profile.pointer) except UserMessage.DoesNotExist: # Don't completely fail if your saved pointer ID is invalid logging.warning("%s has invalid pointer %s" % (user_profile.email, user_profile.pointer)) latest_read = None # Set default language and make it persist default_language = register_ret['default_language'] url_lang = '/{}'.format(request.LANGUAGE_CODE) if not request.path.startswith(url_lang): translation.activate(default_language) request.session[translation.LANGUAGE_SESSION_KEY] = default_language # Pass parameters to the client-side JavaScript code. # These end up in a global JavaScript Object named 'page_params'. page_params = dict( # Server settings. development_environment = settings.DEVELOPMENT, debug_mode = settings.DEBUG, test_suite = settings.TEST_SUITE, poll_timeout = settings.POLL_TIMEOUT, login_page = settings.HOME_NOT_LOGGED_IN, root_domain_uri = settings.ROOT_DOMAIN_URI, maxfilesize = settings.MAX_FILE_UPLOAD_SIZE, max_avatar_file_size = settings.MAX_AVATAR_FILE_SIZE, server_generation = settings.SERVER_GENERATION, use_websockets = settings.USE_WEBSOCKETS, save_stacktraces = settings.SAVE_FRONTEND_STACKTRACES, server_inline_image_preview = settings.INLINE_IMAGE_PREVIEW, server_inline_url_embed_preview = settings.INLINE_URL_EMBED_PREVIEW, password_min_length = settings.PASSWORD_MIN_LENGTH, password_min_guesses = settings.PASSWORD_MIN_GUESSES, # Misc. extra data. have_initial_messages = user_has_messages, initial_servertime = time.time(), # Used for calculating relative presence age default_language_name = get_language_name(register_ret['default_language']), language_list_dbl_col = get_language_list_for_templates(register_ret['default_language']), language_list = get_language_list(), needs_tutorial = needs_tutorial, first_in_realm = first_in_realm, prompt_for_invites = prompt_for_invites, furthest_read_time = sent_time_in_epoch_seconds(latest_read), has_mobile_devices = num_push_devices_for_user(user_profile) > 0, ) undesired_register_ret_fields = [ 'streams', ] for field_name in set(register_ret.keys()) - set(undesired_register_ret_fields): page_params[field_name] = register_ret[field_name] if narrow_stream is not None: # In narrow_stream context, initial pointer is just latest message recipient = get_recipient(Recipient.STREAM, narrow_stream.id) try: initial_pointer = Message.objects.filter(recipient=recipient).order_by('id').reverse()[0].id except IndexError: initial_pointer = -1 page_params["narrow_stream"] = narrow_stream.name if narrow_topic is not None: page_params["narrow_topic"] = narrow_topic page_params["narrow"] = [dict(operator=term[0], operand=term[1]) for term in narrow] page_params["max_message_id"] = initial_pointer page_params["pointer"] = initial_pointer page_params["have_initial_messages"] = (initial_pointer != -1) page_params["enable_desktop_notifications"] = False statsd.incr('views.home') show_invites = True # Some realms only allow admins to invite users if user_profile.realm.invite_by_admins_only and not user_profile.is_realm_admin: show_invites = False request._log_data['extra'] = "[%s]" % (register_ret["queue_id"],) response = render(request, 'zerver/index.html', context={'user_profile': user_profile, 'page_params': simplejson.encoder.JSONEncoderForHTML().encode(page_params), 'nofontface': is_buggy_ua(request.META.get("HTTP_USER_AGENT", "Unspecified")), 'avatar_url': avatar_url(user_profile), 'show_debug': settings.DEBUG and ('show_debug' in request.GET), 'pipeline': settings.PIPELINE_ENABLED, 'show_invites': show_invites, 'is_admin': user_profile.is_realm_admin, 'show_webathena': user_profile.realm.webathena_enabled, 'enable_feedback': settings.ENABLE_FEEDBACK, 'embedded': narrow_stream is not None, },) patch_cache_control(response, no_cache=True, no_store=True, must_revalidate=True) return response
def home(request): # type: (HttpRequest) -> HttpResponse # We need to modify the session object every two weeks or it will expire. # This line makes reloading the page a sufficient action to keep the # session alive. request.session.modified = True user_profile = request.user request._email = request.user.email request.client = get_client("website") # If a user hasn't signed the current Terms of Service, send them there if settings.TERMS_OF_SERVICE is not None and settings.TOS_VERSION is not None and \ int(settings.TOS_VERSION.split('.')[0]) > user_profile.major_tos_version(): return accounts_accept_terms(request) narrow = [] # type: List[List[text_type]] narrow_stream = None narrow_topic = request.GET.get("topic") if request.GET.get("stream"): try: narrow_stream = get_stream(request.GET.get("stream"), user_profile.realm) assert(narrow_stream is not None) assert(narrow_stream.is_public()) narrow = [["stream", narrow_stream.name]] except Exception: logging.exception("Narrow parsing") if narrow_topic is not None: narrow.append(["topic", narrow_topic]) register_ret = do_events_register(user_profile, request.client, apply_markdown=True, narrow=narrow) user_has_messages = (register_ret['max_message_id'] != -1) # Reset our don't-spam-users-with-email counter since the # user has since logged in if not user_profile.last_reminder is None: user_profile.last_reminder = None user_profile.save(update_fields=["last_reminder"]) # Brand new users get the tutorial needs_tutorial = settings.TUTORIAL_ENABLED and \ user_profile.tutorial_status != UserProfile.TUTORIAL_FINISHED first_in_realm = realm_user_count(user_profile.realm) == 1 # If you are the only person in the realm and you didn't invite # anyone, we'll continue to encourage you to do so on the frontend. prompt_for_invites = first_in_realm and \ not PreregistrationUser.objects.filter(referred_by=user_profile).count() if user_profile.pointer == -1 and user_has_messages: # Put the new user's pointer at the bottom # # This improves performance, because we limit backfilling of messages # before the pointer. It's also likely that someone joining an # organization is interested in recent messages more than the very # first messages on the system. register_ret['pointer'] = register_ret['max_message_id'] user_profile.last_pointer_updater = request.session.session_key if user_profile.pointer == -1: latest_read = None else: try: latest_read = UserMessage.objects.get(user_profile=user_profile, message__id=user_profile.pointer) except UserMessage.DoesNotExist: # Don't completely fail if your saved pointer ID is invalid logging.warning("%s has invalid pointer %s" % (user_profile.email, user_profile.pointer)) latest_read = None desktop_notifications_enabled = user_profile.enable_desktop_notifications if narrow_stream is not None: desktop_notifications_enabled = False if user_profile.realm.notifications_stream: notifications_stream = user_profile.realm.notifications_stream.name else: notifications_stream = "" # Set default language and make it persist default_language = register_ret['default_language'] url_lang = '/{}'.format(request.LANGUAGE_CODE) if not request.path.startswith(url_lang): translation.activate(default_language) request.session[translation.LANGUAGE_SESSION_KEY] = default_language # Pass parameters to the client-side JavaScript code. # These end up in a global JavaScript Object named 'page_params'. page_params = dict( zulip_version = ZULIP_VERSION, share_the_love = settings.SHARE_THE_LOVE, development_environment = settings.DEVELOPMENT, debug_mode = settings.DEBUG, test_suite = settings.TEST_SUITE, poll_timeout = settings.POLL_TIMEOUT, login_page = settings.HOME_NOT_LOGGED_IN, server_uri = settings.SERVER_URI, realm_uri = user_profile.realm.uri, maxfilesize = settings.MAX_FILE_UPLOAD_SIZE, server_generation = settings.SERVER_GENERATION, password_auth_enabled = password_auth_enabled(user_profile.realm), have_initial_messages = user_has_messages, subbed_info = register_ret['subscriptions'], unsubbed_info = register_ret['unsubscribed'], neversubbed_info = register_ret['never_subscribed'], email_dict = register_ret['email_dict'], people_list = register_ret['realm_users'], bot_list = register_ret['realm_bots'], initial_pointer = register_ret['pointer'], initial_presences = register_ret['presences'], initial_servertime = time.time(), # Used for calculating relative presence age fullname = user_profile.full_name, email = user_profile.email, domain = user_profile.realm.domain, domains = list_of_domains_for_realm(user_profile.realm), realm_name = register_ret['realm_name'], realm_invite_required = register_ret['realm_invite_required'], realm_invite_by_admins_only = register_ret['realm_invite_by_admins_only'], realm_create_stream_by_admins_only = register_ret['realm_create_stream_by_admins_only'], realm_allow_message_editing = register_ret['realm_allow_message_editing'], realm_message_content_edit_limit_seconds = register_ret['realm_message_content_edit_limit_seconds'], realm_restricted_to_domain = register_ret['realm_restricted_to_domain'], realm_default_language = register_ret['realm_default_language'], enter_sends = user_profile.enter_sends, user_id = user_profile.id, left_side_userlist = register_ret['left_side_userlist'], default_language = register_ret['default_language'], default_language_name = get_language_name(register_ret['default_language']), language_list_dbl_col = get_language_list_for_templates(register_ret['default_language']), language_list = get_language_list(), referrals = register_ret['referrals'], realm_emoji = register_ret['realm_emoji'], needs_tutorial = needs_tutorial, first_in_realm = first_in_realm, prompt_for_invites = prompt_for_invites, notifications_stream = notifications_stream, cross_realm_user_emails = list(get_cross_realm_users()), # Stream message notification settings: stream_desktop_notifications_enabled = user_profile.enable_stream_desktop_notifications, stream_sounds_enabled = user_profile.enable_stream_sounds, # Private message and @-mention notification settings: desktop_notifications_enabled = desktop_notifications_enabled, sounds_enabled = user_profile.enable_sounds, enable_offline_email_notifications = user_profile.enable_offline_email_notifications, enable_offline_push_notifications = user_profile.enable_offline_push_notifications, enable_online_push_notifications = user_profile.enable_online_push_notifications, twenty_four_hour_time = register_ret['twenty_four_hour_time'], enable_digest_emails = user_profile.enable_digest_emails, event_queue_id = register_ret['queue_id'], last_event_id = register_ret['last_event_id'], max_message_id = register_ret['max_message_id'], unread_count = approximate_unread_count(user_profile), furthest_read_time = sent_time_in_epoch_seconds(latest_read), save_stacktraces = settings.SAVE_FRONTEND_STACKTRACES, alert_words = register_ret['alert_words'], muted_topics = register_ret['muted_topics'], realm_filters = register_ret['realm_filters'], realm_default_streams = register_ret['realm_default_streams'], is_admin = user_profile.is_realm_admin, can_create_streams = user_profile.can_create_streams(), name_changes_disabled = name_changes_disabled(user_profile.realm), has_mobile_devices = num_push_devices_for_user(user_profile) > 0, autoscroll_forever = user_profile.autoscroll_forever, default_desktop_notifications = user_profile.default_desktop_notifications, avatar_url = avatar_url(user_profile), avatar_url_medium = avatar_url(user_profile, medium=True), mandatory_topics = user_profile.realm.mandatory_topics, show_digest_email = user_profile.realm.show_digest_email, presence_disabled = user_profile.realm.presence_disabled, is_zephyr_mirror_realm = user_profile.realm.is_zephyr_mirror_realm, ) if narrow_stream is not None: # In narrow_stream context, initial pointer is just latest message recipient = get_recipient(Recipient.STREAM, narrow_stream.id) try: initial_pointer = Message.objects.filter(recipient=recipient).order_by('id').reverse()[0].id except IndexError: initial_pointer = -1 page_params["narrow_stream"] = narrow_stream.name if narrow_topic is not None: page_params["narrow_topic"] = narrow_topic page_params["narrow"] = [dict(operator=term[0], operand=term[1]) for term in narrow] page_params["max_message_id"] = initial_pointer page_params["initial_pointer"] = initial_pointer page_params["have_initial_messages"] = (initial_pointer != -1) statsd.incr('views.home') show_invites = True # Some realms only allow admins to invite users if user_profile.realm.invite_by_admins_only and not user_profile.is_realm_admin: show_invites = False product_name = "Zulip" page_params['product_name'] = product_name request._log_data['extra'] = "[%s]" % (register_ret["queue_id"],) response = render_to_response('zerver/index.html', {'user_profile': user_profile, 'page_params' : simplejson.encoder.JSONEncoderForHTML().encode(page_params), 'nofontface': is_buggy_ua(request.META.get("HTTP_USER_AGENT", "Unspecified")), 'avatar_url': avatar_url(user_profile), 'show_debug': settings.DEBUG and ('show_debug' in request.GET), 'pipeline': settings.PIPELINE_ENABLED, 'show_invites': show_invites, 'is_admin': user_profile.is_realm_admin, 'show_webathena': user_profile.realm.webathena_enabled, 'enable_feedback': settings.ENABLE_FEEDBACK, 'embedded': narrow_stream is not None, 'product_name': product_name }, request=request) patch_cache_control(response, no_cache=True, no_store=True, must_revalidate=True) return response
def home_real(request: HttpRequest) -> HttpResponse: # Before we do any real work, check if the app is banned. client_user_agent = request.META.get("HTTP_USER_AGENT", "") (insecure_desktop_app, banned_desktop_app, auto_update_broken) = is_outdated_desktop_app(client_user_agent) if banned_desktop_app: return render( request, 'zerver/insecure_desktop_app.html', context={ "auto_update_broken": auto_update_broken, }, ) (unsupported_browser, browser_name) = is_unsupported_browser(client_user_agent) if unsupported_browser: return render( request, 'zerver/unsupported_browser.html', context={ "browser_name": browser_name, }, ) # We need to modify the session object every two weeks or it will expire. # This line makes reloading the page a sufficient action to keep the # session alive. request.session.modified = True if request.user.is_authenticated: user_profile = request.user else: # nocoverage # This code path should not be reachable because of zulip_login_required above. user_profile = None update_last_reminder(user_profile) statsd.incr('views.home') # If a user hasn't signed the current Terms of Service, send them there if need_accept_tos(user_profile): return accounts_accept_terms(request) narrow, narrow_stream, narrow_topic = detect_narrowed_window( request, user_profile) if user_profile is not None: first_in_realm = realm_user_count(user_profile.realm) == 1 # If you are the only person in the realm and you didn't invite # anyone, we'll continue to encourage you to do so on the frontend. prompt_for_invites = (first_in_realm and not PreregistrationUser.objects.filter( referred_by=user_profile).count()) needs_tutorial = user_profile.tutorial_status == UserProfile.TUTORIAL_WAITING else: # nocoverage first_in_realm = False prompt_for_invites = False # The current tutorial doesn't super make sense for logged-out users. needs_tutorial = False has_mobile_devices = user_profile is not None and num_push_devices_for_user( user_profile) > 0 queue_id, page_params = build_page_params_for_home_page_load( request=request, user_profile=user_profile, insecure_desktop_app=insecure_desktop_app, has_mobile_devices=has_mobile_devices, narrow=narrow, narrow_stream=narrow_stream, narrow_topic=narrow_topic, first_in_realm=first_in_realm, prompt_for_invites=prompt_for_invites, needs_tutorial=needs_tutorial, ) show_invites, show_add_streams = compute_show_invites_and_add_streams( user_profile) billing_info = get_billing_info(user_profile) request._log_data['extra'] = "[{}]".format(queue_id) csp_nonce = secrets.token_hex(24) user_permission_info = get_user_permission_info(user_profile) navbar_logo_url = compute_navbar_logo_url(page_params) response = render(request, 'zerver/app/index.html', context={ 'user_profile': user_profile, 'page_params': page_params, 'csp_nonce': csp_nonce, 'search_pills_enabled': settings.SEARCH_PILLS_ENABLED, 'show_invites': show_invites, 'show_add_streams': show_add_streams, 'show_billing': billing_info.show_billing, 'corporate_enabled': settings.CORPORATE_ENABLED, 'show_plans': billing_info.show_plans, 'is_owner': user_permission_info.is_realm_owner, 'is_admin': user_permission_info.is_realm_admin, 'is_guest': user_permission_info.is_guest, 'color_scheme': user_permission_info.color_scheme, 'navbar_logo_url': navbar_logo_url, 'show_webathena': user_permission_info.show_webathena, 'embedded': narrow_stream is not None, 'invite_as': PreregistrationUser.INVITE_AS, 'max_file_upload_size_mib': settings.MAX_FILE_UPLOAD_SIZE, }) patch_cache_control(response, no_cache=True, no_store=True, must_revalidate=True) return response
def write_log_line(log_data, path, method, remote_ip, email, client_name, status_code=200, error_content=None, error_content_iter=None): # type: (MutableMapping[str, Any], Text, str, str, Text, Text, int, Optional[AnyStr], Optional[Iterable[AnyStr]]) -> None assert error_content is None or error_content_iter is None if error_content is not None: error_content_iter = (error_content,) # For statsd timer name if path == '/': statsd_path = u'webreq' else: statsd_path = u"webreq.%s" % (path[1:].replace('/', '.'),) # Remove non-ascii chars from path (there should be none, if there are it's # because someone manually entered a nonexistant path), as UTF-8 chars make # statsd sad when it sends the key name over the socket statsd_path = statsd_path.encode('ascii', errors='ignore').decode("ascii") blacklisted_requests = ['do_confirm', 'send_confirm', 'eventslast_event_id', 'webreq.content', 'avatar', 'user_uploads', 'password.reset', 'static', 'json.bots', 'json.users', 'json.streams', 'accounts.unsubscribe', 'apple-touch-icon', 'emoji', 'json.bots', 'upload_file', 'realm_activity', 'user_activity'] suppress_statsd = any((blacklisted in statsd_path for blacklisted in blacklisted_requests)) time_delta = -1 # A time duration of -1 means the StartLogRequests middleware # didn't run for some reason optional_orig_delta = "" if 'time_started' in log_data: time_delta = time.time() - log_data['time_started'] if 'time_stopped' in log_data: orig_time_delta = time_delta time_delta = ((log_data['time_stopped'] - log_data['time_started']) + (time.time() - log_data['time_restarted'])) optional_orig_delta = " (lp: %s)" % (format_timedelta(orig_time_delta),) remote_cache_output = "" if 'remote_cache_time_start' in log_data: remote_cache_time_delta = get_remote_cache_time() - log_data['remote_cache_time_start'] remote_cache_count_delta = get_remote_cache_requests() - log_data['remote_cache_requests_start'] if 'remote_cache_requests_stopped' in log_data: # (now - restarted) + (stopped - start) = (now - start) + (stopped - restarted) remote_cache_time_delta += (log_data['remote_cache_time_stopped'] - log_data['remote_cache_time_restarted']) remote_cache_count_delta += (log_data['remote_cache_requests_stopped'] - log_data['remote_cache_requests_restarted']) if (remote_cache_time_delta > 0.005): remote_cache_output = " (mem: %s/%s)" % (format_timedelta(remote_cache_time_delta), remote_cache_count_delta) if not suppress_statsd: statsd.timing("%s.remote_cache.time" % (statsd_path,), timedelta_ms(remote_cache_time_delta)) statsd.incr("%s.remote_cache.querycount" % (statsd_path,), remote_cache_count_delta) startup_output = "" if 'startup_time_delta' in log_data and log_data["startup_time_delta"] > 0.005: startup_output = " (+start: %s)" % (format_timedelta(log_data["startup_time_delta"])) bugdown_output = "" if 'bugdown_time_start' in log_data: bugdown_time_delta = get_bugdown_time() - log_data['bugdown_time_start'] bugdown_count_delta = get_bugdown_requests() - log_data['bugdown_requests_start'] if 'bugdown_requests_stopped' in log_data: # (now - restarted) + (stopped - start) = (now - start) + (stopped - restarted) bugdown_time_delta += (log_data['bugdown_time_stopped'] - log_data['bugdown_time_restarted']) bugdown_count_delta += (log_data['bugdown_requests_stopped'] - log_data['bugdown_requests_restarted']) if (bugdown_time_delta > 0.005): bugdown_output = " (md: %s/%s)" % (format_timedelta(bugdown_time_delta), bugdown_count_delta) if not suppress_statsd: statsd.timing("%s.markdown.time" % (statsd_path,), timedelta_ms(bugdown_time_delta)) statsd.incr("%s.markdown.count" % (statsd_path,), bugdown_count_delta) # Get the amount of time spent doing database queries db_time_output = "" queries = connection.connection.queries if connection.connection is not None else [] if len(queries) > 0: query_time = sum(float(query.get('time', 0)) for query in queries) db_time_output = " (db: %s/%sq)" % (format_timedelta(query_time), len(queries)) if not suppress_statsd: # Log ms, db ms, and num queries to statsd statsd.timing("%s.dbtime" % (statsd_path,), timedelta_ms(query_time)) statsd.incr("%s.dbq" % (statsd_path,), len(queries)) statsd.timing("%s.total" % (statsd_path,), timedelta_ms(time_delta)) if 'extra' in log_data: extra_request_data = " %s" % (log_data['extra'],) else: extra_request_data = "" logger_client = "(%s via %s)" % (email, client_name) logger_timing = ('%5s%s%s%s%s%s %s' % (format_timedelta(time_delta), optional_orig_delta, remote_cache_output, bugdown_output, db_time_output, startup_output, path)) logger_line = ('%-15s %-7s %3d %s%s %s' % (remote_ip, method, status_code, logger_timing, extra_request_data, logger_client)) if (status_code in [200, 304] and method == "GET" and path.startswith("/static")): logger.debug(logger_line) else: logger.info(logger_line) if (is_slow_query(time_delta, path)): queue_json_publish("slow_queries", "%s (%s)" % (logger_line, email), lambda e: None) if settings.PROFILE_ALL_REQUESTS: log_data["prof"].disable() profile_path = "/tmp/profile.data.%s.%s" % (path.split("/")[-1], int(time_delta * 1000),) log_data["prof"].dump_stats(profile_path) # Log some additional data whenever we return certain 40x errors if 400 <= status_code < 500 and status_code not in [401, 404, 405]: assert error_content_iter is not None error_content_list = list(error_content_iter) if error_content_list: error_data = u'' elif isinstance(error_content_list[0], Text): error_data = u''.join(error_content_list) elif isinstance(error_content_list[0], binary_type): error_data = repr(b''.join(error_content_list)) if len(error_data) > 100: error_data = u"[content more than 100 characters]" logger.info('status=%3d, data=%s, uid=%s' % (status_code, error_data, email))
def home_real(request: HttpRequest) -> HttpResponse: # Before we do any real work, check if the app is banned. client_user_agent = request.META.get("HTTP_USER_AGENT", "") (insecure_desktop_app, banned_desktop_app, auto_update_broken) = is_outdated_desktop_app( client_user_agent) if banned_desktop_app: return render( request, 'zerver/insecure_desktop_app.html', context={ "auto_update_broken": auto_update_broken, }, ) (unsupported_browser, browser_name) = is_unsupported_browser(client_user_agent) if unsupported_browser: return render( request, 'zerver/unsupported_browser.html', context={ "browser_name": browser_name, }, ) # We need to modify the session object every two weeks or it will expire. # This line makes reloading the page a sufficient action to keep the # session alive. request.session.modified = True if request.user.is_authenticated: user_profile = request.user else: # nocoverage # This code path should not be reachable because of zulip_login_required above. user_profile = None # If a user hasn't signed the current Terms of Service, send them there if need_accept_tos(user_profile): return accounts_accept_terms(request) narrow, narrow_stream, narrow_topic = detect_narrowed_window(request, user_profile) client_capabilities = { 'notification_settings_null': True, 'bulk_message_deletion': True, 'user_avatar_url_field_optional': True, } register_ret = do_events_register(user_profile, request.client, apply_markdown=True, client_gravatar=True, slim_presence=True, client_capabilities=client_capabilities, narrow=narrow) update_last_reminder(user_profile) if user_profile is not None: first_in_realm = realm_user_count(user_profile.realm) == 1 # If you are the only person in the realm and you didn't invite # anyone, we'll continue to encourage you to do so on the frontend. prompt_for_invites = ( first_in_realm and not PreregistrationUser.objects.filter(referred_by=user_profile).count() ) needs_tutorial = user_profile.tutorial_status == UserProfile.TUTORIAL_WAITING else: # nocoverage first_in_realm = False prompt_for_invites = False # The current tutorial doesn't super make sense for logged-out users. needs_tutorial = False furthest_read_time = get_furthest_read_time(user_profile) # We pick a language for the user as follows: # * First priority is the language in the URL, for debugging. # * If not in the URL, we use the language from the user's settings. request_language = translation.get_language_from_path(request.path_info) if request_language is None: request_language = register_ret['default_language'] translation.activate(request_language) # We also save the language to the user's session, so that # something reasonable will happen in logged-in portico pages. request.session[translation.LANGUAGE_SESSION_KEY] = translation.get_language() two_fa_enabled = settings.TWO_FACTOR_AUTHENTICATION_ENABLED and user_profile is not None # Pass parameters to the client-side JavaScript code. # These end up in a global JavaScript Object named 'page_params'. page_params = dict( # Server settings. debug_mode = settings.DEBUG, test_suite = settings.TEST_SUITE, poll_timeout = settings.POLL_TIMEOUT, insecure_desktop_app = insecure_desktop_app, login_page = settings.HOME_NOT_LOGGED_IN, root_domain_uri = settings.ROOT_DOMAIN_URI, save_stacktraces = settings.SAVE_FRONTEND_STACKTRACES, warn_no_email = settings.WARN_NO_EMAIL, search_pills_enabled = settings.SEARCH_PILLS_ENABLED, # Misc. extra data. initial_servertime = time.time(), # Used for calculating relative presence age default_language_name = get_language_name(register_ret['default_language']), language_list_dbl_col = get_language_list_for_templates(register_ret['default_language']), language_list = get_language_list(), needs_tutorial = needs_tutorial, first_in_realm = first_in_realm, prompt_for_invites = prompt_for_invites, furthest_read_time = furthest_read_time, has_mobile_devices = user_profile is not None and num_push_devices_for_user(user_profile) > 0, bot_types = get_bot_types(user_profile), two_fa_enabled = two_fa_enabled, # Adding two_fa_enabled as condition saves us 3 queries when # 2FA is not enabled. two_fa_enabled_user = two_fa_enabled and bool(default_device(user_profile)), ) undesired_register_ret_fields = [ 'streams', ] for field_name in set(register_ret.keys()) - set(undesired_register_ret_fields): page_params[field_name] = register_ret[field_name] if narrow_stream is not None: # In narrow_stream context, initial pointer is just latest message recipient = narrow_stream.recipient try: max_message_id = Message.objects.filter(recipient=recipient).order_by('id').reverse()[0].id except IndexError: max_message_id = -1 page_params["narrow_stream"] = narrow_stream.name if narrow_topic is not None: page_params["narrow_topic"] = narrow_topic page_params["narrow"] = [dict(operator=term[0], operand=term[1]) for term in narrow] page_params["max_message_id"] = max_message_id page_params["enable_desktop_notifications"] = False statsd.incr('views.home') show_invites, show_add_streams = compute_show_invites_and_add_streams(user_profile) show_billing = False show_plans = False if settings.CORPORATE_ENABLED and user_profile is not None: from corporate.models import CustomerPlan, get_customer_by_realm if user_profile.has_billing_access: customer = get_customer_by_realm(user_profile.realm) if customer is not None: if customer.sponsorship_pending: show_billing = True elif CustomerPlan.objects.filter(customer=customer).exists(): show_billing = True if user_profile.realm.plan_type == Realm.LIMITED: show_plans = True request._log_data['extra'] = "[{}]".format(register_ret["queue_id"]) page_params['translation_data'] = {} if request_language != 'en': page_params['translation_data'] = get_language_translation_data(request_language) csp_nonce = generate_random_token(48) if user_profile is not None: color_scheme = user_profile.color_scheme is_guest = user_profile.is_guest is_realm_owner = user_profile.is_realm_owner is_realm_admin = user_profile.is_realm_admin show_webathena = user_profile.realm.webathena_enabled else: # nocoverage color_scheme = UserProfile.COLOR_SCHEME_AUTOMATIC is_guest = False is_realm_admin = False is_realm_owner = False show_webathena = False navbar_logo_url = compute_navbar_logo_url(page_params) # print("----------------->") # new_realm_users = [] # user_initial = user_profile.full_name[0] # pattern = '^{}'.format(user_initial) # for _ in range(len(page_params['realm_users'])): # current_user_fullname = page_params['realm_users'][_]['full_name'] # initials_matched_user = re.match(pattern,current_user_fullname) # if initials_matched_user: # new_realm_users.append(page_params['realm_users'][_]) # page_params['realm_users'] = new_realm_users # print(page_params['realm_users']) # print('\n') # print(page_params.keys()) # print("-----------------> UserProfile objects") # print(UserProfile.objects.all()) response = render(request, 'zerver/app/index.html', context={'user_profile': user_profile, 'page_params': page_params, 'csp_nonce': csp_nonce, 'search_pills_enabled': settings.SEARCH_PILLS_ENABLED, 'show_invites': show_invites, 'show_add_streams': show_add_streams, 'show_billing': show_billing, 'corporate_enabled': settings.CORPORATE_ENABLED, 'show_plans': show_plans, 'is_owner': is_realm_owner, 'is_admin': is_realm_admin, 'is_guest': is_guest, 'color_scheme': color_scheme, 'navbar_logo_url': navbar_logo_url, 'show_webathena': show_webathena, 'embedded': narrow_stream is not None, 'invite_as': PreregistrationUser.INVITE_AS, 'max_file_upload_size_mib': settings.MAX_FILE_UPLOAD_SIZE, }) patch_cache_control(response, no_cache=True, no_store=True, must_revalidate=True) # print("########################### | These are page_params keys | ##############") # print(page_params.keys()) # print("########################### | Realm Users | #############################") # print(page_params['realm_users']) # print("############################ | Presences |############################") # print("\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/") # print(page_params['presences']) return response
def wrapped_func(*args, **kwargs): ret = func(*args, **kwargs) statsd.incr(counter, val) return ret
def home_real(request: HttpRequest) -> HttpResponse: # We need to modify the session object every two weeks or it will expire. # This line makes reloading the page a sufficient action to keep the # session alive. request.session.modified = True user_profile = request.user # If a user hasn't signed the current Terms of Service, send them there if settings.TERMS_OF_SERVICE is not None and settings.TOS_VERSION is not None and \ int(settings.TOS_VERSION.split('.')[0]) > user_profile.major_tos_version(): return accounts_accept_terms(request) narrow = [] # type: List[List[str]] narrow_stream = None narrow_topic = request.GET.get("topic") if request.GET.get("stream"): try: narrow_stream_name = request.GET.get("stream") (narrow_stream, ignored_rec, ignored_sub) = access_stream_by_name(user_profile, narrow_stream_name) narrow = [["stream", narrow_stream.name]] except Exception: logging.exception("Narrow parsing exception", extra=dict(request=request)) if narrow_stream is not None and narrow_topic is not None: narrow.append(["topic", narrow_topic]) register_ret = do_events_register(user_profile, request.client, apply_markdown=True, client_gravatar=True, narrow=narrow) user_has_messages = (register_ret['max_message_id'] != -1) # Reset our don't-spam-users-with-email counter since the # user has since logged in if user_profile.last_reminder is not None: # nocoverage # TODO: Look into the history of last_reminder; we may have # eliminated that as a useful concept for non-bot users. user_profile.last_reminder = None user_profile.save(update_fields=["last_reminder"]) # Brand new users get narrowed to PM with welcome-bot needs_tutorial = user_profile.tutorial_status == UserProfile.TUTORIAL_WAITING first_in_realm = realm_user_count(user_profile.realm) == 1 # If you are the only person in the realm and you didn't invite # anyone, we'll continue to encourage you to do so on the frontend. prompt_for_invites = first_in_realm and \ not PreregistrationUser.objects.filter(referred_by=user_profile).count() if user_profile.pointer == -1 and user_has_messages: # Put the new user's pointer at the bottom # # This improves performance, because we limit backfilling of messages # before the pointer. It's also likely that someone joining an # organization is interested in recent messages more than the very # first messages on the system. register_ret['pointer'] = register_ret['max_message_id'] user_profile.last_pointer_updater = request.session.session_key if user_profile.pointer == -1: latest_read = None else: latest_read = get_usermessage_by_message_id(user_profile, user_profile.pointer) if latest_read is None: # Don't completely fail if your saved pointer ID is invalid logging.warning("%s has invalid pointer %s" % (user_profile.email, user_profile.pointer)) # We pick a language for the user as follows: # * First priority is the language in the URL, for debugging. # * If not in the URL, we use the language from the user's settings. request_language = translation.get_language_from_path(request.path_info) if request_language is None: request_language = register_ret['default_language'] translation.activate(request_language) # We also save the language to the user's session, so that # something reasonable will happen in logged-in portico pages. request.session[ translation.LANGUAGE_SESSION_KEY] = translation.get_language() two_fa_enabled = settings.TWO_FACTOR_AUTHENTICATION_ENABLED # Pass parameters to the client-side JavaScript code. # These end up in a global JavaScript Object named 'page_params'. page_params = dict( # Server settings. development_environment=settings.DEVELOPMENT, debug_mode=settings.DEBUG, test_suite=settings.TEST_SUITE, poll_timeout=settings.POLL_TIMEOUT, login_page=settings.HOME_NOT_LOGGED_IN, root_domain_uri=settings.ROOT_DOMAIN_URI, maxfilesize=settings.MAX_FILE_UPLOAD_SIZE, max_avatar_file_size=settings.MAX_AVATAR_FILE_SIZE, server_generation=settings.SERVER_GENERATION, use_websockets=settings.USE_WEBSOCKETS, save_stacktraces=settings.SAVE_FRONTEND_STACKTRACES, warn_no_email=settings.WARN_NO_EMAIL, server_inline_image_preview=settings.INLINE_IMAGE_PREVIEW, server_inline_url_embed_preview=settings.INLINE_URL_EMBED_PREVIEW, password_min_length=settings.PASSWORD_MIN_LENGTH, password_min_guesses=settings.PASSWORD_MIN_GUESSES, jitsi_server_url=settings.JITSI_SERVER_URL, search_pills_enabled=settings.SEARCH_PILLS_ENABLED, # Misc. extra data. have_initial_messages=user_has_messages, initial_servertime=time.time( ), # Used for calculating relative presence age default_language_name=get_language_name( register_ret['default_language']), language_list_dbl_col=get_language_list_for_templates( register_ret['default_language']), language_list=get_language_list(), needs_tutorial=needs_tutorial, first_in_realm=first_in_realm, prompt_for_invites=prompt_for_invites, furthest_read_time=sent_time_in_epoch_seconds(latest_read), has_mobile_devices=num_push_devices_for_user(user_profile) > 0, bot_types=get_bot_types(user_profile), two_fa_enabled=two_fa_enabled, # Adding two_fa_enabled as condition saves us 3 queries when # 2FA is not enabled. two_fa_enabled_user=two_fa_enabled and bool(default_device(user_profile)), ) undesired_register_ret_fields = [ 'streams', ] for field_name in set( register_ret.keys()) - set(undesired_register_ret_fields): page_params[field_name] = register_ret[field_name] if narrow_stream is not None: # In narrow_stream context, initial pointer is just latest message recipient = get_stream_recipient(narrow_stream.id) try: initial_pointer = Message.objects.filter( recipient=recipient).order_by('id').reverse()[0].id except IndexError: initial_pointer = -1 page_params["narrow_stream"] = narrow_stream.name if narrow_topic is not None: page_params["narrow_topic"] = narrow_topic page_params["narrow"] = [ dict(operator=term[0], operand=term[1]) for term in narrow ] page_params["max_message_id"] = initial_pointer page_params["pointer"] = initial_pointer page_params["have_initial_messages"] = (initial_pointer != -1) page_params["enable_desktop_notifications"] = False statsd.incr('views.home') show_invites = True # Some realms only allow admins to invite users if user_profile.realm.invite_by_admins_only and not user_profile.is_realm_admin: show_invites = False if user_profile.is_guest: show_invites = False show_billing = False show_plans = False if settings.CORPORATE_ENABLED: from corporate.models import Customer if user_profile.is_billing_admin or user_profile.is_realm_admin: customer = Customer.objects.filter( realm=user_profile.realm).first() if customer is not None and customer.has_billing_relationship: show_billing = True if user_profile.realm.plan_type == Realm.LIMITED: show_plans = True request._log_data['extra'] = "[%s]" % (register_ret["queue_id"], ) page_params['translation_data'] = {} if request_language != 'en': page_params['translation_data'] = get_language_translation_data( request_language) csp_nonce = generate_random_token(48) emojiset = user_profile.emojiset if emojiset == UserProfile.TEXT_EMOJISET: # If current emojiset is `TEXT_EMOJISET`, then fallback to # GOOGLE_EMOJISET for picking which spritesheet's CSS to # include (and thus how to display emojis in the emoji picker # and composebox typeahead). emojiset = UserProfile.GOOGLE_BLOB_EMOJISET response = render( request, 'zerver/app/index.html', context={ 'user_profile': user_profile, 'emojiset': emojiset, 'page_params': JSONEncoderForHTML().encode(page_params), 'csp_nonce': csp_nonce, 'avatar_url': avatar_url(user_profile), 'show_debug': settings.DEBUG and ('show_debug' in request.GET), 'pipeline': settings.PIPELINE_ENABLED, 'search_pills_enabled': settings.SEARCH_PILLS_ENABLED, 'show_invites': show_invites, 'show_billing': show_billing, 'show_plans': show_plans, 'is_admin': user_profile.is_realm_admin, 'is_guest': user_profile.is_guest, 'show_webathena': user_profile.realm.webathena_enabled, 'enable_feedback': settings.ENABLE_FEEDBACK, 'embedded': narrow_stream is not None, }, ) patch_cache_control(response, no_cache=True, no_store=True, must_revalidate=True) return response
def home_real(request): # type: (HttpRequest) -> HttpResponse # We need to modify the session object every two weeks or it will expire. # This line makes reloading the page a sufficient action to keep the # session alive. request.session.modified = True user_profile = request.user # If a user hasn't signed the current Terms of Service, send them there if settings.TERMS_OF_SERVICE is not None and settings.TOS_VERSION is not None and \ int(settings.TOS_VERSION.split('.')[0]) > user_profile.major_tos_version(): return accounts_accept_terms(request) narrow = [] # type: List[List[Text]] narrow_stream = None narrow_topic = request.GET.get("topic") if request.GET.get("stream"): try: narrow_stream_name = request.GET.get("stream") (narrow_stream, ignored_rec, ignored_sub) = access_stream_by_name( user_profile, narrow_stream_name) narrow = [["stream", narrow_stream.name]] except Exception: logging.exception("Narrow parsing") if narrow_stream is not None and narrow_topic is not None: narrow.append(["topic", narrow_topic]) register_ret = do_events_register(user_profile, request.client, apply_markdown=True, narrow=narrow) user_has_messages = (register_ret['max_message_id'] != -1) # Reset our don't-spam-users-with-email counter since the # user has since logged in if user_profile.last_reminder is not None: user_profile.last_reminder = None user_profile.save(update_fields=["last_reminder"]) # Brand new users get the tutorial needs_tutorial = settings.TUTORIAL_ENABLED and \ user_profile.tutorial_status != UserProfile.TUTORIAL_FINISHED first_in_realm = realm_user_count(user_profile.realm) == 1 # If you are the only person in the realm and you didn't invite # anyone, we'll continue to encourage you to do so on the frontend. prompt_for_invites = first_in_realm and \ not PreregistrationUser.objects.filter(referred_by=user_profile).count() if user_profile.pointer == -1 and user_has_messages: # Put the new user's pointer at the bottom # # This improves performance, because we limit backfilling of messages # before the pointer. It's also likely that someone joining an # organization is interested in recent messages more than the very # first messages on the system. register_ret['pointer'] = register_ret['max_message_id'] user_profile.last_pointer_updater = request.session.session_key if user_profile.pointer == -1: latest_read = None else: try: latest_read = UserMessage.objects.get(user_profile=user_profile, message__id=user_profile.pointer) except UserMessage.DoesNotExist: # Don't completely fail if your saved pointer ID is invalid logging.warning("%s has invalid pointer %s" % (user_profile.email, user_profile.pointer)) latest_read = None # Set default language and make it persist default_language = register_ret['default_language'] url_lang = '/{}'.format(request.LANGUAGE_CODE) if not request.path.startswith(url_lang): translation.activate(default_language) request.session[translation.LANGUAGE_SESSION_KEY] = default_language # Pass parameters to the client-side JavaScript code. # These end up in a global JavaScript Object named 'page_params'. page_params = dict( # Server settings. share_the_love = settings.SHARE_THE_LOVE, development_environment = settings.DEVELOPMENT, debug_mode = settings.DEBUG, test_suite = settings.TEST_SUITE, poll_timeout = settings.POLL_TIMEOUT, login_page = settings.HOME_NOT_LOGGED_IN, server_uri = settings.SERVER_URI, maxfilesize = settings.MAX_FILE_UPLOAD_SIZE, max_avatar_file_size = settings.MAX_AVATAR_FILE_SIZE, server_generation = settings.SERVER_GENERATION, use_websockets = settings.USE_WEBSOCKETS, save_stacktraces = settings.SAVE_FRONTEND_STACKTRACES, server_inline_image_preview = settings.INLINE_IMAGE_PREVIEW, server_inline_url_embed_preview = settings.INLINE_URL_EMBED_PREVIEW, # Misc. extra data. have_initial_messages = user_has_messages, initial_servertime = time.time(), # Used for calculating relative presence age default_language_name = get_language_name(register_ret['default_language']), language_list_dbl_col = get_language_list_for_templates(register_ret['default_language']), language_list = get_language_list(), needs_tutorial = needs_tutorial, first_in_realm = first_in_realm, prompt_for_invites = prompt_for_invites, unread_count = approximate_unread_count(user_profile), furthest_read_time = sent_time_in_epoch_seconds(latest_read), has_mobile_devices = num_push_devices_for_user(user_profile) > 0, ) undesired_register_ret_fields = [ 'streams', ] for field_name in set(register_ret.keys()) - set(undesired_register_ret_fields): page_params[field_name] = register_ret[field_name] if narrow_stream is not None: # In narrow_stream context, initial pointer is just latest message recipient = get_recipient(Recipient.STREAM, narrow_stream.id) try: initial_pointer = Message.objects.filter(recipient=recipient).order_by('id').reverse()[0].id except IndexError: initial_pointer = -1 page_params["narrow_stream"] = narrow_stream.name if narrow_topic is not None: page_params["narrow_topic"] = narrow_topic page_params["narrow"] = [dict(operator=term[0], operand=term[1]) for term in narrow] page_params["max_message_id"] = initial_pointer page_params["pointer"] = initial_pointer page_params["have_initial_messages"] = (initial_pointer != -1) page_params["enable_desktop_notifications"] = False statsd.incr('views.home') show_invites = True # Some realms only allow admins to invite users if user_profile.realm.invite_by_admins_only and not user_profile.is_realm_admin: show_invites = False request._log_data['extra'] = "[%s]" % (register_ret["queue_id"],) response = render(request, 'zerver/index.html', context={'user_profile': user_profile, 'page_params': simplejson.encoder.JSONEncoderForHTML().encode(page_params), 'nofontface': is_buggy_ua(request.META.get("HTTP_USER_AGENT", "Unspecified")), 'avatar_url': avatar_url(user_profile), 'show_debug': settings.DEBUG and ('show_debug' in request.GET), 'pipeline': settings.PIPELINE_ENABLED, 'show_invites': show_invites, 'is_admin': user_profile.is_realm_admin, 'show_webathena': user_profile.realm.webathena_enabled, 'enable_feedback': settings.ENABLE_FEEDBACK, 'embedded': narrow_stream is not None, },) patch_cache_control(response, no_cache=True, no_store=True, must_revalidate=True) return response
def home_real(request: HttpRequest) -> HttpResponse: # Before we do any real work, check if the app is banned. client_user_agent = request.headers.get("User-Agent", "") (insecure_desktop_app, banned_desktop_app, auto_update_broken) = is_outdated_desktop_app(client_user_agent) if banned_desktop_app: return render( request, "zerver/insecure_desktop_app.html", context={ "auto_update_broken": auto_update_broken, }, ) (unsupported_browser, browser_name) = is_unsupported_browser(client_user_agent) if unsupported_browser: return render( request, "zerver/unsupported_browser.html", context={ "browser_name": browser_name, }, ) # We need to modify the session object every two weeks or it will expire. # This line makes reloading the page a sufficient action to keep the # session alive. request.session.modified = True if request.user.is_authenticated: user_profile = request.user realm = user_profile.realm else: realm = get_valid_realm_from_request(request) # We load the spectator experience. We fall through to the shared code # for loading the application, with user_profile=None encoding # that we're a spectator, not a logged-in user. user_profile = None update_last_reminder(user_profile) statsd.incr("views.home") # If a user hasn't signed the current Terms of Service, send them there if need_accept_tos(user_profile): return accounts_accept_terms(request) narrow, narrow_stream, narrow_topic = detect_narrowed_window( request, user_profile) if user_profile is not None: first_in_realm = realm_user_count(user_profile.realm) == 1 # If you are the only person in the realm and you didn't invite # anyone, we'll continue to encourage you to do so on the frontend. prompt_for_invites = (first_in_realm and not PreregistrationUser.objects.filter( referred_by=user_profile).count()) needs_tutorial = user_profile.tutorial_status == UserProfile.TUTORIAL_WAITING else: first_in_realm = False prompt_for_invites = False # The current tutorial doesn't super make sense for logged-out users. needs_tutorial = False queue_id, page_params = build_page_params_for_home_page_load( request=request, user_profile=user_profile, realm=realm, insecure_desktop_app=insecure_desktop_app, narrow=narrow, narrow_stream=narrow_stream, narrow_topic=narrow_topic, first_in_realm=first_in_realm, prompt_for_invites=prompt_for_invites, needs_tutorial=needs_tutorial, ) log_data = RequestNotes.get_notes(request).log_data assert log_data is not None log_data["extra"] = f"[{queue_id}]" csp_nonce = secrets.token_hex(24) user_permission_info = get_user_permission_info(user_profile) response = render( request, "zerver/app/index.html", context={ "user_profile": user_profile, "page_params": page_params, "csp_nonce": csp_nonce, "color_scheme": user_permission_info.color_scheme, }, ) patch_cache_control(response, no_cache=True, no_store=True, must_revalidate=True) return response
def wrapped_func(*args, **kwargs): # type: (*Any, **Any) -> Any ret = func(*args, **kwargs) statsd.incr(counter, val) return ret
def wrapped_func(*args: ParamT.args, **kwargs: ParamT.kwargs) -> ReturnT: ret = func(*args, **kwargs) statsd.incr(counter, val) return ret
def home_real(request: HttpRequest) -> HttpResponse: # We need to modify the session object every two weeks or it will expire. # This line makes reloading the page a sufficient action to keep the # session alive. request.session.modified = True user_profile = request.user # If a user hasn't signed the current Terms of Service, send them there if settings.TERMS_OF_SERVICE is not None and settings.TOS_VERSION is not None and \ int(settings.TOS_VERSION.split('.')[0]) > user_profile.major_tos_version(): return accounts_accept_terms(request) narrow = [] # type: List[List[str]] narrow_stream = None narrow_topic = request.GET.get("topic") if request.GET.get("stream"): try: narrow_stream_name = request.GET.get("stream") (narrow_stream, ignored_rec, ignored_sub) = access_stream_by_name( user_profile, narrow_stream_name) narrow = [["stream", narrow_stream.name]] except Exception: logging.exception("Narrow parsing exception", extra=dict(request=request)) if narrow_stream is not None and narrow_topic is not None: narrow.append(["topic", narrow_topic]) register_ret = do_events_register(user_profile, request.client, apply_markdown=True, client_gravatar=True, narrow=narrow) user_has_messages = (register_ret['max_message_id'] != -1) # Reset our don't-spam-users-with-email counter since the # user has since logged in if user_profile.last_reminder is not None: # nocoverage # TODO: Look into the history of last_reminder; we may have # eliminated that as a useful concept for non-bot users. user_profile.last_reminder = None user_profile.save(update_fields=["last_reminder"]) # Brand new users get narrowed to PM with welcome-bot needs_tutorial = user_profile.tutorial_status == UserProfile.TUTORIAL_WAITING first_in_realm = realm_user_count(user_profile.realm) == 1 # If you are the only person in the realm and you didn't invite # anyone, we'll continue to encourage you to do so on the frontend. prompt_for_invites = first_in_realm and \ not PreregistrationUser.objects.filter(referred_by=user_profile).count() if user_profile.pointer == -1 and user_has_messages: # Put the new user's pointer at the bottom # # This improves performance, because we limit backfilling of messages # before the pointer. It's also likely that someone joining an # organization is interested in recent messages more than the very # first messages on the system. register_ret['pointer'] = register_ret['max_message_id'] user_profile.last_pointer_updater = request.session.session_key if user_profile.pointer == -1: latest_read = None else: latest_read = get_usermessage_by_message_id(user_profile, user_profile.pointer) if latest_read is None: # Don't completely fail if your saved pointer ID is invalid logging.warning("%s has invalid pointer %s" % (user_profile.email, user_profile.pointer)) # We pick a language for the user as follows: # * First priority is the language in the URL, for debugging. # * If not in the URL, we use the language from the user's settings. request_language = translation.get_language_from_path(request.path_info) if request_language is None: request_language = register_ret['default_language'] translation.activate(request_language) # We also save the language to the user's session, so that # something reasonable will happen in logged-in portico pages. request.session[translation.LANGUAGE_SESSION_KEY] = translation.get_language() two_fa_enabled = settings.TWO_FACTOR_AUTHENTICATION_ENABLED # Pass parameters to the client-side JavaScript code. # These end up in a global JavaScript Object named 'page_params'. page_params = dict( # Server settings. development_environment = settings.DEVELOPMENT, debug_mode = settings.DEBUG, test_suite = settings.TEST_SUITE, poll_timeout = settings.POLL_TIMEOUT, login_page = settings.HOME_NOT_LOGGED_IN, root_domain_uri = settings.ROOT_DOMAIN_URI, maxfilesize = settings.MAX_FILE_UPLOAD_SIZE, max_avatar_file_size = settings.MAX_AVATAR_FILE_SIZE, server_generation = settings.SERVER_GENERATION, use_websockets = settings.USE_WEBSOCKETS, save_stacktraces = settings.SAVE_FRONTEND_STACKTRACES, warn_no_email = settings.WARN_NO_EMAIL, server_inline_image_preview = settings.INLINE_IMAGE_PREVIEW, server_inline_url_embed_preview = settings.INLINE_URL_EMBED_PREVIEW, password_min_length = settings.PASSWORD_MIN_LENGTH, password_min_guesses = settings.PASSWORD_MIN_GUESSES, jitsi_server_url = settings.JITSI_SERVER_URL, search_pills_enabled = settings.SEARCH_PILLS_ENABLED, # Misc. extra data. have_initial_messages = user_has_messages, initial_servertime = time.time(), # Used for calculating relative presence age default_language_name = get_language_name(register_ret['default_language']), language_list_dbl_col = get_language_list_for_templates(register_ret['default_language']), language_list = get_language_list(), needs_tutorial = needs_tutorial, first_in_realm = first_in_realm, prompt_for_invites = prompt_for_invites, furthest_read_time = sent_time_in_epoch_seconds(latest_read), has_mobile_devices = num_push_devices_for_user(user_profile) > 0, bot_types = get_bot_types(user_profile), two_fa_enabled = two_fa_enabled, # Adding two_fa_enabled as condition saves us 3 queries when # 2FA is not enabled. two_fa_enabled_user = two_fa_enabled and bool(default_device(user_profile)), ) undesired_register_ret_fields = [ 'streams', ] for field_name in set(register_ret.keys()) - set(undesired_register_ret_fields): page_params[field_name] = register_ret[field_name] if narrow_stream is not None: # In narrow_stream context, initial pointer is just latest message recipient = get_stream_recipient(narrow_stream.id) try: initial_pointer = Message.objects.filter(recipient=recipient).order_by('id').reverse()[0].id except IndexError: initial_pointer = -1 page_params["narrow_stream"] = narrow_stream.name if narrow_topic is not None: page_params["narrow_topic"] = narrow_topic page_params["narrow"] = [dict(operator=term[0], operand=term[1]) for term in narrow] page_params["max_message_id"] = initial_pointer page_params["pointer"] = initial_pointer page_params["have_initial_messages"] = (initial_pointer != -1) page_params["enable_desktop_notifications"] = False statsd.incr('views.home') show_invites = True # Some realms only allow admins to invite users if user_profile.realm.invite_by_admins_only and not user_profile.is_realm_admin: show_invites = False if user_profile.is_guest: show_invites = False show_billing = False show_plans = False if settings.CORPORATE_ENABLED: from corporate.models import Customer if user_profile.is_billing_admin or user_profile.is_realm_admin: customer = Customer.objects.filter(realm=user_profile.realm).first() if customer is not None and customer.has_billing_relationship: show_billing = True if user_profile.realm.plan_type == Realm.LIMITED: show_plans = True request._log_data['extra'] = "[%s]" % (register_ret["queue_id"],) page_params['translation_data'] = {} if request_language != 'en': page_params['translation_data'] = get_language_translation_data(request_language) csp_nonce = generate_random_token(48) emojiset = user_profile.emojiset if emojiset == UserProfile.TEXT_EMOJISET: # If current emojiset is `TEXT_EMOJISET`, then fallback to # GOOGLE_EMOJISET for picking which spritesheet's CSS to # include (and thus how to display emojis in the emoji picker # and composebox typeahead). emojiset = UserProfile.GOOGLE_BLOB_EMOJISET response = render(request, 'zerver/app/index.html', context={'user_profile': user_profile, 'emojiset': emojiset, 'page_params': JSONEncoderForHTML().encode(page_params), 'csp_nonce': csp_nonce, 'avatar_url': avatar_url(user_profile), 'show_debug': settings.DEBUG and ('show_debug' in request.GET), 'pipeline': settings.PIPELINE_ENABLED, 'search_pills_enabled': settings.SEARCH_PILLS_ENABLED, 'show_invites': show_invites, 'show_billing': show_billing, 'show_plans': show_plans, 'is_admin': user_profile.is_realm_admin, 'is_guest': user_profile.is_guest, 'show_webathena': user_profile.realm.webathena_enabled, 'enable_feedback': settings.ENABLE_FEEDBACK, 'embedded': narrow_stream is not None, 'invite_as': PreregistrationUser.INVITE_AS, },) patch_cache_control(response, no_cache=True, no_store=True, must_revalidate=True) return response
def home_real(request: HttpRequest) -> HttpResponse: # Before we do any real work, check if the app is banned. client_user_agent = request.META.get("HTTP_USER_AGENT", "") (insecure_desktop_app, banned_desktop_app, auto_update_broken) = is_outdated_desktop_app(client_user_agent) if banned_desktop_app: return render( request, "zerver/insecure_desktop_app.html", context={ "auto_update_broken": auto_update_broken, }, ) (unsupported_browser, browser_name) = is_unsupported_browser(client_user_agent) if unsupported_browser: return render( request, "zerver/unsupported_browser.html", context={ "browser_name": browser_name, }, ) # We need to modify the session object every two weeks or it will expire. # This line makes reloading the page a sufficient action to keep the # session alive. request.session.modified = True if request.user.is_authenticated: user_profile = request.user realm = user_profile.realm # User is logged in and hence no longer `prefers_web_public_view`. if "prefers_web_public_view" in request.session.keys(): del request.session["prefers_web_public_view"] else: realm = get_valid_realm_from_request(request) # TODO: Ideally, we'd open Zulip directly as a spectator if # the URL had clicked a link to content on a web-public # stream. We could maybe do this by parsing `next`, but it's # not super convenient with Zulip's hash-based URL scheme. # The "Access without an account" button on the login page # submits a POST to this page with this hidden field set. if request.POST.get("prefers_web_public_view") == "true": request.session["prefers_web_public_view"] = True # We serve a redirect here, rather than serving a page, to # avoid browser "Confirm form resubmission" prompts on reload. redirect_to = get_safe_redirect_to(request.POST.get("next"), realm.uri) return redirect(redirect_to) prefers_web_public_view = request.session.get( "prefers_web_public_view") if not prefers_web_public_view: # For users who haven't opted into the spectator # experience, we redirect to the login page. return zulip_redirect_to_login(request, settings.HOME_NOT_LOGGED_IN) # For users who have selected public access, we load the # spectator experience. We fall through to the shared code # for loading the application, with user_profile=None encoding # that we're a spectator, not a logged-in user. user_profile = None update_last_reminder(user_profile) statsd.incr("views.home") # If a user hasn't signed the current Terms of Service, send them there if need_accept_tos(user_profile): return accounts_accept_terms(request) narrow, narrow_stream, narrow_topic = detect_narrowed_window( request, user_profile) if user_profile is not None: first_in_realm = realm_user_count(user_profile.realm) == 1 # If you are the only person in the realm and you didn't invite # anyone, we'll continue to encourage you to do so on the frontend. prompt_for_invites = (first_in_realm and not PreregistrationUser.objects.filter( referred_by=user_profile).count()) needs_tutorial = user_profile.tutorial_status == UserProfile.TUTORIAL_WAITING else: first_in_realm = False prompt_for_invites = False # The current tutorial doesn't super make sense for logged-out users. needs_tutorial = False queue_id, page_params = build_page_params_for_home_page_load( request=request, user_profile=user_profile, realm=realm, insecure_desktop_app=insecure_desktop_app, narrow=narrow, narrow_stream=narrow_stream, narrow_topic=narrow_topic, first_in_realm=first_in_realm, prompt_for_invites=prompt_for_invites, needs_tutorial=needs_tutorial, ) log_data = RequestNotes.get_notes(request).log_data assert log_data is not None log_data["extra"] = f"[{queue_id}]" csp_nonce = secrets.token_hex(24) user_permission_info = get_user_permission_info(user_profile) response = render( request, "zerver/app/index.html", context={ "user_profile": user_profile, "page_params": page_params, "csp_nonce": csp_nonce, "color_scheme": user_permission_info.color_scheme, }, ) patch_cache_control(response, no_cache=True, no_store=True, must_revalidate=True) return response
def wrapped_func(*args: Any, **kwargs: Any) -> ReturnT: ret = func(*args, **kwargs) statsd.incr(counter, val) return ret
def write_log_line(log_data, path, method, remote_ip, email, client_name, status_code=200, error_content=None, error_content_iter=None): # type: (MutableMapping[str, Any], Text, str, str, Text, Text, int, Optional[AnyStr], Optional[Iterable[AnyStr]]) -> None assert error_content is None or error_content_iter is None if error_content is not None: error_content_iter = (error_content, ) # For statsd timer name if path == '/': statsd_path = u'webreq' else: statsd_path = u"webreq.%s" % (path[1:].replace('/', '.'), ) # Remove non-ascii chars from path (there should be none, if there are it's # because someone manually entered a nonexistant path), as UTF-8 chars make # statsd sad when it sends the key name over the socket statsd_path = statsd_path.encode('ascii', errors='ignore').decode("ascii") blacklisted_requests = [ 'do_confirm', 'send_confirm', 'eventslast_event_id', 'webreq.content', 'avatar', 'user_uploads', 'password.reset', 'static', 'json.bots', 'json.users', 'json.streams', 'accounts.unsubscribe', 'apple-touch-icon', 'emoji', 'json.bots', 'upload_file', 'realm_activity', 'user_activity' ] suppress_statsd = any( (blacklisted in statsd_path for blacklisted in blacklisted_requests)) time_delta = -1 # A time duration of -1 means the StartLogRequests middleware # didn't run for some reason optional_orig_delta = "" if 'time_started' in log_data: time_delta = time.time() - log_data['time_started'] if 'time_stopped' in log_data: orig_time_delta = time_delta time_delta = ((log_data['time_stopped'] - log_data['time_started']) + (time.time() - log_data['time_restarted'])) optional_orig_delta = " (lp: %s)" % ( format_timedelta(orig_time_delta), ) remote_cache_output = "" if 'remote_cache_time_start' in log_data: remote_cache_time_delta = get_remote_cache_time( ) - log_data['remote_cache_time_start'] remote_cache_count_delta = get_remote_cache_requests( ) - log_data['remote_cache_requests_start'] if 'remote_cache_requests_stopped' in log_data: # (now - restarted) + (stopped - start) = (now - start) + (stopped - restarted) remote_cache_time_delta += ( log_data['remote_cache_time_stopped'] - log_data['remote_cache_time_restarted']) remote_cache_count_delta += ( log_data['remote_cache_requests_stopped'] - log_data['remote_cache_requests_restarted']) if (remote_cache_time_delta > 0.005): remote_cache_output = " (mem: %s/%s)" % (format_timedelta( remote_cache_time_delta), remote_cache_count_delta) if not suppress_statsd: statsd.timing("%s.remote_cache.time" % (statsd_path, ), timedelta_ms(remote_cache_time_delta)) statsd.incr("%s.remote_cache.querycount" % (statsd_path, ), remote_cache_count_delta) startup_output = "" if 'startup_time_delta' in log_data and log_data[ "startup_time_delta"] > 0.005: startup_output = " (+start: %s)" % (format_timedelta( log_data["startup_time_delta"])) bugdown_output = "" if 'bugdown_time_start' in log_data: bugdown_time_delta = get_bugdown_time( ) - log_data['bugdown_time_start'] bugdown_count_delta = get_bugdown_requests( ) - log_data['bugdown_requests_start'] if 'bugdown_requests_stopped' in log_data: # (now - restarted) + (stopped - start) = (now - start) + (stopped - restarted) bugdown_time_delta += (log_data['bugdown_time_stopped'] - log_data['bugdown_time_restarted']) bugdown_count_delta += (log_data['bugdown_requests_stopped'] - log_data['bugdown_requests_restarted']) if (bugdown_time_delta > 0.005): bugdown_output = " (md: %s/%s)" % ( format_timedelta(bugdown_time_delta), bugdown_count_delta) if not suppress_statsd: statsd.timing("%s.markdown.time" % (statsd_path, ), timedelta_ms(bugdown_time_delta)) statsd.incr("%s.markdown.count" % (statsd_path, ), bugdown_count_delta) # Get the amount of time spent doing database queries db_time_output = "" queries = connection.connection.queries if connection.connection is not None else [] if len(queries) > 0: query_time = sum(float(query.get('time', 0)) for query in queries) db_time_output = " (db: %s/%sq)" % (format_timedelta(query_time), len(queries)) if not suppress_statsd: # Log ms, db ms, and num queries to statsd statsd.timing("%s.dbtime" % (statsd_path, ), timedelta_ms(query_time)) statsd.incr("%s.dbq" % (statsd_path, ), len(queries)) statsd.timing("%s.total" % (statsd_path, ), timedelta_ms(time_delta)) if 'extra' in log_data: extra_request_data = " %s" % (log_data['extra'], ) else: extra_request_data = "" logger_client = "(%s via %s)" % (email, client_name) logger_timing = ('%5s%s%s%s%s%s %s' % (format_timedelta(time_delta), optional_orig_delta, remote_cache_output, bugdown_output, db_time_output, startup_output, path)) logger_line = ('%-15s %-7s %3d %s%s %s' % (remote_ip, method, status_code, logger_timing, extra_request_data, logger_client)) if (status_code in [200, 304] and method == "GET" and path.startswith("/static")): logger.debug(logger_line) else: logger.info(logger_line) if (is_slow_query(time_delta, path)): queue_json_publish("slow_queries", "%s (%s)" % (logger_line, email), lambda e: None) if settings.PROFILE_ALL_REQUESTS: log_data["prof"].disable() profile_path = "/tmp/profile.data.%s.%s" % ( path.split("/")[-1], int(time_delta * 1000), ) log_data["prof"].dump_stats(profile_path) # Log some additional data whenever we return certain 40x errors if 400 <= status_code < 500 and status_code not in [401, 404, 405]: assert error_content_iter is not None error_content_list = list(error_content_iter) if error_content_list: error_data = u'' elif isinstance(error_content_list[0], Text): error_data = u''.join(error_content_list) elif isinstance(error_content_list[0], binary_type): error_data = repr(b''.join(error_content_list)) if len(error_data) > 100: error_data = u"[content more than 100 characters]" logger.info('status=%3d, data=%s, uid=%s' % (status_code, error_data, email))
def home_real(request: HttpRequest) -> HttpResponse: # Before we do any real work, check if the app is banned. (insecure_desktop_app, banned_desktop_app, auto_update_broken) = is_outdated_desktop_app( request.META.get("HTTP_USER_AGENT", "")) if banned_desktop_app: return render(request, 'zerver/insecure_desktop_app.html', context={ "auto_update_broken": auto_update_broken, }) # We need to modify the session object every two weeks or it will expire. # This line makes reloading the page a sufficient action to keep the # session alive. request.session.modified = True if request.user.is_authenticated: user_profile = request.user else: # nocoverage # This code path should not be reachable because of zulip_login_required above. user_profile = None # If a user hasn't signed the current Terms of Service, send them there if need_accept_tos(user_profile): return accounts_accept_terms(request) narrow, narrow_stream, narrow_topic = detect_narrowed_window( request, user_profile) register_ret = do_events_register(user_profile, request.client, apply_markdown=True, client_gravatar=True, slim_presence=True, notification_settings_null=True, narrow=narrow) user_has_messages = (register_ret['max_message_id'] != -1) update_last_reminder(user_profile) if user_profile is not None: first_in_realm = realm_user_count(user_profile.realm) == 1 # If you are the only person in the realm and you didn't invite # anyone, we'll continue to encourage you to do so on the frontend. prompt_for_invites = (first_in_realm and not PreregistrationUser.objects.filter( referred_by=user_profile).count()) needs_tutorial = user_profile.tutorial_status == UserProfile.TUTORIAL_WAITING else: # nocoverage first_in_realm = False prompt_for_invites = False # The current tutorial doesn't super make sense for logged-out users. needs_tutorial = False if user_profile is None: # nocoverage furthest_read_time = time.time() # type: Optional[float] elif user_profile.pointer == -1: if user_has_messages: # Put the new user's pointer at the bottom # # This improves performance, because we limit backfilling of messages # before the pointer. It's also likely that someone joining an # organization is interested in recent messages more than the very # first messages on the system. register_ret['pointer'] = register_ret['max_message_id'] user_profile.last_pointer_updater = request.session.session_key furthest_read_time = None else: latest_read = get_usermessage_by_message_id(user_profile, user_profile.pointer) if latest_read is None: # Don't completely fail if your saved pointer ID is invalid logging.warning("User %s has invalid pointer %s" % (user_profile.id, user_profile.pointer)) furthest_read_time = sent_time_in_epoch_seconds(latest_read) # We pick a language for the user as follows: # * First priority is the language in the URL, for debugging. # * If not in the URL, we use the language from the user's settings. request_language = translation.get_language_from_path(request.path_info) if request_language is None: request_language = register_ret['default_language'] translation.activate(request_language) # We also save the language to the user's session, so that # something reasonable will happen in logged-in portico pages. request.session[ translation.LANGUAGE_SESSION_KEY] = translation.get_language() two_fa_enabled = settings.TWO_FACTOR_AUTHENTICATION_ENABLED and user_profile is not None # Pass parameters to the client-side JavaScript code. # These end up in a global JavaScript Object named 'page_params'. page_params = dict( # Server settings. development_environment=settings.DEVELOPMENT, debug_mode=settings.DEBUG, test_suite=settings.TEST_SUITE, poll_timeout=settings.POLL_TIMEOUT, insecure_desktop_app=insecure_desktop_app, login_page=settings.HOME_NOT_LOGGED_IN, root_domain_uri=settings.ROOT_DOMAIN_URI, max_file_upload_size=settings.MAX_FILE_UPLOAD_SIZE, max_avatar_file_size=settings.MAX_AVATAR_FILE_SIZE, server_generation=settings.SERVER_GENERATION, save_stacktraces=settings.SAVE_FRONTEND_STACKTRACES, warn_no_email=settings.WARN_NO_EMAIL, server_inline_image_preview=settings.INLINE_IMAGE_PREVIEW, server_inline_url_embed_preview=settings.INLINE_URL_EMBED_PREVIEW, password_min_length=settings.PASSWORD_MIN_LENGTH, password_min_guesses=settings.PASSWORD_MIN_GUESSES, jitsi_server_url=settings.JITSI_SERVER_URL, search_pills_enabled=settings.SEARCH_PILLS_ENABLED, server_avatar_changes_disabled=settings.AVATAR_CHANGES_DISABLED, server_name_changes_disabled=settings.NAME_CHANGES_DISABLED, # Misc. extra data. have_initial_messages=user_has_messages, initial_servertime=time.time( ), # Used for calculating relative presence age default_language_name=get_language_name( register_ret['default_language']), language_list_dbl_col=get_language_list_for_templates( register_ret['default_language']), language_list=get_language_list(), needs_tutorial=needs_tutorial, first_in_realm=first_in_realm, prompt_for_invites=prompt_for_invites, furthest_read_time=furthest_read_time, has_mobile_devices=user_profile is not None and num_push_devices_for_user(user_profile) > 0, bot_types=get_bot_types(user_profile), two_fa_enabled=two_fa_enabled, # Adding two_fa_enabled as condition saves us 3 queries when # 2FA is not enabled. two_fa_enabled_user=two_fa_enabled and bool(default_device(user_profile)), ) undesired_register_ret_fields = [ 'streams', ] for field_name in set( register_ret.keys()) - set(undesired_register_ret_fields): page_params[field_name] = register_ret[field_name] if narrow_stream is not None: # In narrow_stream context, initial pointer is just latest message recipient = narrow_stream.recipient try: initial_pointer = Message.objects.filter( recipient=recipient).order_by('id').reverse()[0].id except IndexError: initial_pointer = -1 page_params["narrow_stream"] = narrow_stream.name if narrow_topic is not None: page_params["narrow_topic"] = narrow_topic page_params["narrow"] = [ dict(operator=term[0], operand=term[1]) for term in narrow ] page_params["max_message_id"] = initial_pointer page_params["pointer"] = initial_pointer page_params["have_initial_messages"] = (initial_pointer != -1) page_params["enable_desktop_notifications"] = False statsd.incr('views.home') show_invites, show_add_streams = compute_show_invites_and_add_streams( user_profile) show_billing = False show_plans = False if settings.CORPORATE_ENABLED and user_profile is not None: from corporate.models import Customer, CustomerPlan if user_profile.is_billing_admin or user_profile.is_realm_admin: customer = Customer.objects.filter( realm=user_profile.realm).first() if customer is not None and CustomerPlan.objects.filter( customer=customer).exists(): show_billing = True if user_profile.realm.plan_type == Realm.LIMITED: show_plans = True request._log_data['extra'] = "[%s]" % (register_ret["queue_id"], ) page_params['translation_data'] = {} if request_language != 'en': page_params['translation_data'] = get_language_translation_data( request_language) csp_nonce = generate_random_token(48) if user_profile is not None: night_mode = user_profile.night_mode is_guest = user_profile.is_guest is_realm_admin = user_profile.is_realm_admin show_webathena = user_profile.realm.webathena_enabled else: # nocoverage night_mode = False is_guest = False is_realm_admin = False show_webathena = False navbar_logo_url = compute_navbar_logo_url(page_params) response = render( request, 'zerver/app/index.html', context={ 'user_profile': user_profile, 'page_params': page_params, 'csp_nonce': csp_nonce, 'search_pills_enabled': settings.SEARCH_PILLS_ENABLED, 'show_invites': show_invites, 'show_add_streams': show_add_streams, 'show_billing': show_billing, 'show_plans': show_plans, 'is_admin': is_realm_admin, 'is_guest': is_guest, 'night_mode': night_mode, 'navbar_logo_url': navbar_logo_url, 'show_webathena': show_webathena, 'embedded': narrow_stream is not None, 'invite_as': PreregistrationUser.INVITE_AS, 'max_file_upload_size': settings.MAX_FILE_UPLOAD_SIZE, }, ) patch_cache_control(response, no_cache=True, no_store=True, must_revalidate=True) return response
def get_old_messages_backend(request, user_profile, anchor = REQ(converter=int), num_before = REQ(converter=to_non_negative_int), num_after = REQ(converter=to_non_negative_int), narrow = REQ('narrow', converter=narrow_parameter, default=None), use_first_unread_anchor = REQ(default=False, converter=ujson.loads), apply_markdown=REQ(default=True, converter=ujson.loads)): # type: (HttpRequest, UserProfile, int, int, int, Optional[List[Dict[str, Any]]], bool, bool) -> HttpResponse include_history = ok_to_include_history(narrow, user_profile.realm) if include_history and not use_first_unread_anchor: query = select([column("id").label("message_id")], None, table("zerver_message")) inner_msg_id_col = literal_column("zerver_message.id") elif narrow is None: query = select([column("message_id"), column("flags")], column("user_profile_id") == literal(user_profile.id), table("zerver_usermessage")) inner_msg_id_col = column("message_id") else: # TODO: Don't do this join if we're not doing a search query = select([column("message_id"), column("flags")], column("user_profile_id") == literal(user_profile.id), join(table("zerver_usermessage"), table("zerver_message"), literal_column("zerver_usermessage.message_id") == literal_column("zerver_message.id"))) inner_msg_id_col = column("message_id") num_extra_messages = 1 is_search = False if narrow is not None: # Add some metadata to our logging data for narrows verbose_operators = [] for term in narrow: if term['operator'] == "is": verbose_operators.append("is:" + term['operand']) else: verbose_operators.append(term['operator']) request._log_data['extra'] = "[%s]" % (",".join(verbose_operators),) # Build the query for the narrow num_extra_messages = 0 builder = NarrowBuilder(user_profile, inner_msg_id_col) search_term = None # type: Optional[Dict[str, Any]] for term in narrow: if term['operator'] == 'search': if not is_search: search_term = term query = query.column(column("subject")).column(column("rendered_content")) is_search = True else: # Join the search operators if there are multiple of them search_term['operand'] += ' ' + term['operand'] else: query = builder.add_term(query, term) if is_search: query = builder.add_term(query, search_term) # We add 1 to the number of messages requested if no narrow was # specified to ensure that the resulting list always contains the # anchor message. If a narrow was specified, the anchor message # might not match the narrow anyway. if num_after != 0: num_after += num_extra_messages else: num_before += num_extra_messages sa_conn = get_sqlalchemy_connection() if use_first_unread_anchor: condition = column("flags").op("&")(UserMessage.flags.read.mask) == 0 # We exclude messages on muted topics when finding the first unread # message in this narrow muting_conditions = exclude_muting_conditions(user_profile, narrow) if muting_conditions: condition = and_(condition, *muting_conditions) first_unread_query = query.where(condition) first_unread_query = first_unread_query.order_by(inner_msg_id_col.asc()).limit(1) first_unread_result = list(sa_conn.execute(first_unread_query).fetchall()) if len(first_unread_result) > 0: anchor = first_unread_result[0][0] else: anchor = LARGER_THAN_MAX_MESSAGE_ID before_query = None after_query = None if num_before != 0: before_anchor = anchor if num_after != 0: # Don't include the anchor in both the before query and the after query before_anchor = anchor - 1 before_query = query.where(inner_msg_id_col <= before_anchor) \ .order_by(inner_msg_id_col.desc()).limit(num_before) if num_after != 0: after_query = query.where(inner_msg_id_col >= anchor) \ .order_by(inner_msg_id_col.asc()).limit(num_after) if anchor == LARGER_THAN_MAX_MESSAGE_ID: # There's no need for an after_query if we're targeting just the target message. after_query = None if before_query is not None: if after_query is not None: query = union_all(before_query.self_group(), after_query.self_group()) else: query = before_query elif after_query is not None: query = after_query else: # This can happen when a narrow is specified. query = query.where(inner_msg_id_col == anchor) main_query = alias(query) query = select(main_query.c, None, main_query).order_by(column("message_id").asc()) # This is a hack to tag the query we use for testing query = query.prefix_with("/* get_old_messages */") query_result = list(sa_conn.execute(query).fetchall()) # The following is a little messy, but ensures that the code paths # are similar regardless of the value of include_history. The # 'user_messages' dictionary maps each message to the user's # UserMessage object for that message, which we will attach to the # rendered message dict before returning it. We attempt to # bulk-fetch rendered message dicts from remote cache using the # 'messages' list. search_fields = dict() # type: Dict[int, Dict[str, Text]] message_ids = [] # type: List[int] user_message_flags = {} # type: Dict[int, List[str]] if include_history: message_ids = [row[0] for row in query_result] # TODO: This could be done with an outer join instead of two queries user_message_flags = dict((user_message.message_id, user_message.flags_list()) for user_message in UserMessage.objects.filter(user_profile=user_profile, message__id__in=message_ids)) for row in query_result: message_id = row[0] if user_message_flags.get(message_id) is None: user_message_flags[message_id] = ["read", "historical"] if is_search: (_, subject, rendered_content, content_matches, subject_matches) = row search_fields[message_id] = get_search_fields(rendered_content, subject, content_matches, subject_matches) else: for row in query_result: message_id = row[0] flags = row[1] user_message_flags[message_id] = parse_usermessage_flags(flags) message_ids.append(message_id) if is_search: (_, _, subject, rendered_content, content_matches, subject_matches) = row search_fields[message_id] = get_search_fields(rendered_content, subject, content_matches, subject_matches) cache_transformer = lambda row: MessageDict.build_dict_from_raw_db_row(row, apply_markdown) id_fetcher = lambda row: row['id'] message_dicts = generic_bulk_cached_fetch(lambda message_id: to_dict_cache_key_id(message_id, apply_markdown), Message.get_raw_db_rows, message_ids, id_fetcher=id_fetcher, cache_transformer=cache_transformer, extractor=extract_message_dict, setter=stringify_message_dict) message_list = [] for message_id in message_ids: msg_dict = message_dicts[message_id] msg_dict.update({"flags": user_message_flags[message_id]}) msg_dict.update(search_fields.get(message_id, {})) message_list.append(msg_dict) statsd.incr('loaded_old_messages', len(message_list)) ret = {'messages': message_list, "result": "success", "msg": ""} return json_success(ret)
def get_messages_backend(request: HttpRequest, user_profile: UserProfile, anchor_val: Optional[str]=REQ( 'anchor', str_validator=check_string, default=None), num_before: int=REQ(converter=to_non_negative_int), num_after: int=REQ(converter=to_non_negative_int), narrow: OptionalNarrowListT=REQ('narrow', converter=narrow_parameter, default=None), use_first_unread_anchor_val: bool=REQ('use_first_unread_anchor', validator=check_bool, default=False), client_gravatar: bool=REQ(validator=check_bool, default=False), apply_markdown: bool=REQ(validator=check_bool, default=True)) -> HttpResponse: anchor = parse_anchor_value(anchor_val, use_first_unread_anchor_val) if num_before + num_after > MAX_MESSAGES_PER_FETCH: return json_error(_("Too many messages requested (maximum {}).").format( MAX_MESSAGES_PER_FETCH, )) if user_profile.realm.email_address_visibility != Realm.EMAIL_ADDRESS_VISIBILITY_EVERYONE: # If email addresses are only available to administrators, # clients cannot compute gravatars, so we force-set it to false. client_gravatar = False include_history = ok_to_include_history(narrow, user_profile) if include_history: # The initial query in this case doesn't use `zerver_usermessage`, # and isn't yet limited to messages the user is entitled to see! # # This is OK only because we've made sure this is a narrow that # will cause us to limit the query appropriately later. # See `ok_to_include_history` for details. need_message = True need_user_message = False elif narrow is None: # We need to limit to messages the user has received, but we don't actually # need any fields from Message need_message = False need_user_message = True else: need_message = True need_user_message = True query, inner_msg_id_col = get_base_query_for_search( user_profile=user_profile, need_message=need_message, need_user_message=need_user_message, ) query, is_search = add_narrow_conditions( user_profile=user_profile, inner_msg_id_col=inner_msg_id_col, query=query, narrow=narrow, ) if narrow is not None: # Add some metadata to our logging data for narrows verbose_operators = [] for term in narrow: if term['operator'] == "is": verbose_operators.append("is:" + term['operand']) else: verbose_operators.append(term['operator']) request._log_data['extra'] = "[{}]".format(",".join(verbose_operators)) sa_conn = get_sqlalchemy_connection() if anchor is None: # The use_first_unread_anchor code path anchor = find_first_unread_anchor( sa_conn, user_profile, narrow, ) anchored_to_left = (anchor == 0) # Set value that will be used to short circuit the after_query # altogether and avoid needless conditions in the before_query. anchored_to_right = (anchor >= LARGER_THAN_MAX_MESSAGE_ID) if anchored_to_right: num_after = 0 first_visible_message_id = get_first_visible_message_id(user_profile.realm) query = limit_query_to_range( query=query, num_before=num_before, num_after=num_after, anchor=anchor, anchored_to_left=anchored_to_left, anchored_to_right=anchored_to_right, id_col=inner_msg_id_col, first_visible_message_id=first_visible_message_id, ) main_query = alias(query) query = select(main_query.c, None, main_query).order_by(column("message_id").asc()) # This is a hack to tag the query we use for testing query = query.prefix_with("/* get_messages */") rows = list(sa_conn.execute(query).fetchall()) query_info = post_process_limited_query( rows=rows, num_before=num_before, num_after=num_after, anchor=anchor, anchored_to_left=anchored_to_left, anchored_to_right=anchored_to_right, first_visible_message_id=first_visible_message_id, ) rows = query_info['rows'] # The following is a little messy, but ensures that the code paths # are similar regardless of the value of include_history. The # 'user_messages' dictionary maps each message to the user's # UserMessage object for that message, which we will attach to the # rendered message dict before returning it. We attempt to # bulk-fetch rendered message dicts from remote cache using the # 'messages' list. message_ids: List[int] = [] user_message_flags: Dict[int, List[str]] = {} if include_history: message_ids = [row[0] for row in rows] # TODO: This could be done with an outer join instead of two queries um_rows = UserMessage.objects.filter(user_profile=user_profile, message__id__in=message_ids) user_message_flags = {um.message_id: um.flags_list() for um in um_rows} for message_id in message_ids: if message_id not in user_message_flags: user_message_flags[message_id] = ["read", "historical"] else: for row in rows: message_id = row[0] flags = row[1] user_message_flags[message_id] = UserMessage.flags_list_for_flags(flags) message_ids.append(message_id) search_fields: Dict[int, Dict[str, str]] = dict() if is_search: for row in rows: message_id = row[0] (topic_name, rendered_content, content_matches, topic_matches) = row[-4:] try: search_fields[message_id] = get_search_fields(rendered_content, topic_name, content_matches, topic_matches) except UnicodeDecodeError as err: # nocoverage # No coverage for this block since it should be # impossible, and we plan to remove it once we've # debugged the case that makes it happen. raise Exception(str(err), message_id, narrow) message_list = messages_for_ids( message_ids=message_ids, user_message_flags=user_message_flags, search_fields=search_fields, apply_markdown=apply_markdown, client_gravatar=client_gravatar, allow_edit_history=user_profile.realm.allow_edit_history, ) statsd.incr('loaded_old_messages', len(message_list)) ret = dict( messages=message_list, result='success', msg='', found_anchor=query_info['found_anchor'], found_oldest=query_info['found_oldest'], found_newest=query_info['found_newest'], history_limited=query_info['history_limited'], anchor=anchor, ) return json_success(ret)
def home_real(request: HttpRequest) -> HttpResponse: # Before we do any real work, check if the app is banned. client_user_agent = request.META.get("HTTP_USER_AGENT", "") (insecure_desktop_app, banned_desktop_app, auto_update_broken) = is_outdated_desktop_app(client_user_agent) if banned_desktop_app: return render( request, "zerver/insecure_desktop_app.html", context={ "auto_update_broken": auto_update_broken, }, ) (unsupported_browser, browser_name) = is_unsupported_browser(client_user_agent) if unsupported_browser: return render( request, "zerver/unsupported_browser.html", context={ "browser_name": browser_name, }, ) # We need to modify the session object every two weeks or it will expire. # This line makes reloading the page a sufficient action to keep the # session alive. request.session.modified = True if request.user.is_authenticated: user_profile = request.user realm = user_profile.realm else: # user_profile=None corresponds to the logged-out "web_public" visitor case. user_profile = None realm = get_valid_realm_from_request(request) update_last_reminder(user_profile) statsd.incr("views.home") # If a user hasn't signed the current Terms of Service, send them there if need_accept_tos(user_profile): return accounts_accept_terms(request) narrow, narrow_stream, narrow_topic = detect_narrowed_window( request, user_profile) if user_profile is not None: first_in_realm = realm_user_count(user_profile.realm) == 1 # If you are the only person in the realm and you didn't invite # anyone, we'll continue to encourage you to do so on the frontend. prompt_for_invites = (first_in_realm and not PreregistrationUser.objects.filter( referred_by=user_profile).count()) needs_tutorial = user_profile.tutorial_status == UserProfile.TUTORIAL_WAITING else: first_in_realm = False prompt_for_invites = False # The current tutorial doesn't super make sense for logged-out users. needs_tutorial = False queue_id, page_params = build_page_params_for_home_page_load( request=request, user_profile=user_profile, realm=realm, insecure_desktop_app=insecure_desktop_app, narrow=narrow, narrow_stream=narrow_stream, narrow_topic=narrow_topic, first_in_realm=first_in_realm, prompt_for_invites=prompt_for_invites, needs_tutorial=needs_tutorial, ) show_invites, show_add_streams = compute_show_invites_and_add_streams( user_profile) request._log_data["extra"] = "[{}]".format(queue_id) csp_nonce = secrets.token_hex(24) user_permission_info = get_user_permission_info(user_profile) response = render( request, "zerver/app/index.html", context={ "user_profile": user_profile, "page_params": page_params, "csp_nonce": csp_nonce, "search_pills_enabled": settings.SEARCH_PILLS_ENABLED, "show_invites": show_invites, "show_add_streams": show_add_streams, "is_owner": user_permission_info.is_realm_owner, "is_admin": user_permission_info.is_realm_admin, "is_guest": user_permission_info.is_guest, "color_scheme": user_permission_info.color_scheme, "embedded": narrow_stream is not None, "max_file_upload_size_mib": settings.MAX_FILE_UPLOAD_SIZE, }, ) patch_cache_control(response, no_cache=True, no_store=True, must_revalidate=True) return response