def handle(self, *args: Any, **options: str) -> None: queue_name = options['queue_name'] file_name = options['file_name'] if file_name == '-': f = sys.stdin # type: IO[str] else: f = open(file_name) while True: line = f.readline() if not line: break line = line.strip() try: payload = line.split('\t')[1] except IndexError: payload = line print('Queueing to queue %s: %s' % (queue_name, payload)) # Verify that payload is valid json. data = ujson.loads(payload) # This is designed to use the `error` method rather than # the call_consume_in_tests flow. queue_json_publish(queue_name, data, error)
def missedmessage_hook(user_profile_id, queue, last_for_client): # Only process missedmessage hook when the last queue for a # client has been garbage collected if not last_for_client: return message_ids_to_notify = [] for event in queue.event_queue.contents(): if not event["type"] == "message" or not event["flags"]: continue if "mentioned" in event["flags"] and "read" not in event["flags"]: notify_info = dict(message_id=event["message"]["id"]) if not event.get("push_notified", False): notify_info["send_push"] = True if not event.get("email_notified", False): notify_info["send_email"] = True message_ids_to_notify.append(notify_info) for notify_info in message_ids_to_notify: msg_id = notify_info["message_id"] notice = build_offline_notification(user_profile_id, msg_id) if notify_info.get("send_push", False): queue_json_publish("missedmessage_mobile_notifications", notice, lambda notice: None) if notify_info.get("send_email", False): queue_json_publish("missedmessage_emails", notice, lambda notice: None)
def missedmessage_hook(user_profile_id, queue, last_for_client): # Only process missedmessage hook when the last queue for a # client has been garbage collected if not last_for_client: return message_ids_to_notify = [] for event in queue.event_queue.contents(): if not event['type'] == 'message' or not event['flags']: continue if 'mentioned' in event['flags'] and not 'read' in event['flags']: notify_info = dict(message_id=event['message']['id']) if not event.get('push_notified', False): notify_info['send_push'] = True if not event.get('email_notified', False): notify_info['send_email'] = True message_ids_to_notify.append(notify_info) for notify_info in message_ids_to_notify: msg_id = notify_info['message_id'] notice = build_offline_notification(user_profile_id, msg_id) if notify_info.get('send_push', False): queue_json_publish("missedmessage_mobile_notifications", notice, lambda notice: None) if notify_info.get('send_email', False): queue_json_publish("missedmessage_emails", notice, lambda notice: None)
def mirror_email_message(data: Dict[str, str]) -> Dict[str, str]: rcpt_to = data['recipient'] if is_missed_message_address(rcpt_to): try: mark_missed_message_address_as_used(rcpt_to) except ZulipEmailForwardError: return { "status": "error", "msg": "5.1.1 Bad destination mailbox address: " "Bad or expired missed message address." } else: try: extract_and_validate(rcpt_to) except ZulipEmailForwardError: return { "status": "error", "msg": "5.1.1 Bad destination mailbox address: " "Please use the address specified in your Streams page." } queue_json_publish( "email_mirror", { "message": data['msg_text'], "rcpt_to": rcpt_to } ) return {"status": "success"}
def test_register_consumer_nack(self) -> None: output = [] count = 0 queue_client = get_queue_client() def collect(event: Dict[str, Any]) -> None: queue_client.stop_consuming() nonlocal count count += 1 if count == 1: raise Exception("Make me nack!") output.append(event) queue_client.register_json_consumer("test_suite", collect) queue_json_publish("test_suite", {"event": "my_event"}) try: queue_client.start_consuming() except Exception: queue_client.register_json_consumer("test_suite", collect) queue_client.start_consuming() # Confirm that we processed the event fully once self.assertEqual(count, 2) self.assertEqual(len(output), 1) self.assertEqual(output[0]['event'], 'my_event')
def report_error(request: HttpRequest, user_profile: UserProfile, message: str=REQ(), stacktrace: str=REQ(), ui_message: bool=REQ(validator=check_bool), user_agent: str=REQ(), href: str=REQ(), log: str=REQ(), more_info: Optional[Dict[str, Any]]=REQ(validator=check_dict([]), default=None) ) -> HttpResponse: """Accepts an error report and stores in a queue for processing. The actual error reports are later handled by do_report_error (below)""" if not settings.BROWSER_ERROR_REPORTING: return json_success() if more_info is None: more_info = {} js_source_map = get_js_source_map() if js_source_map: stacktrace = js_source_map.annotate_stacktrace(stacktrace) try: version = subprocess.check_output(["git", "log", "HEAD^..HEAD", "--oneline"], universal_newlines=True) # type: Optional[str] except Exception: version = None # Get the IP address of the request remote_ip = request.META.get('HTTP_X_REAL_IP') if remote_ip is None: remote_ip = request.META['REMOTE_ADDR'] # For the privacy of our users, we remove any actual text content # in draft_content (from drafts rendering exceptions). See the # comment on privacy_clean_markdown for more details. if more_info.get('draft_content'): more_info['draft_content'] = privacy_clean_markdown(more_info['draft_content']) if user_profile.is_authenticated: email = user_profile.delivery_email full_name = user_profile.full_name else: email = "*****@*****.**" full_name = "Anonymous User" queue_json_publish('error_reports', dict( type = "browser", report = dict( host = request.get_host().split(":")[0], ip_address = remote_ip, user_email = email, user_full_name = full_name, user_visible = ui_message, server_path = settings.DEPLOY_ROOT, version = version, user_agent = user_agent, href = href, message = message, stacktrace = stacktrace, log = log, more_info = more_info, ) )) return json_success()
def test_queue_basics_json(self) -> None: queue_json_publish("test_suite", {"event": "my_event"}) queue_client = get_queue_client() result = queue_client.drain_queue("test_suite", json=True) self.assertEqual(len(result), 1) self.assertEqual(result[0]['event'], 'my_event')
def json_report_error(request, user_profile, message=REQ(), stacktrace=REQ(), ui_message=REQ(validator=check_bool), user_agent=REQ(), href=REQ(), log=REQ(), more_info=REQ(validator=check_dict([]), default=None)): # type: (HttpRequest, UserProfile, text_type, text_type, bool, text_type, text_type, text_type, Dict[str, Any]) -> HttpResponse if not settings.ERROR_REPORTING: return json_success() if js_source_map: stacktrace = js_source_map.annotate_stacktrace(stacktrace) try: version = subprocess.check_output(["git", "log", "HEAD^..HEAD", "--oneline"], universal_newlines=True) except Exception: version = None queue_json_publish('error_reports', dict( type = "browser", report = dict( user_email = user_profile.email, user_full_name = user_profile.full_name, user_visible = ui_message, server_path = settings.DEPLOY_ROOT, version = version, user_agent = user_agent, href = href, message = message, stacktrace = stacktrace, log = log, more_info = more_info, ) ), lambda x: None) return json_success()
def email_on_new_login(sender: Any, user: UserProfile, request: Any, **kwargs: Any) -> None: # We import here to minimize the dependencies of this module, # since it runs as part of `manage.py` initialization from zerver.context_processors import common_context if not settings.SEND_LOGIN_EMAILS: return if request: # If the user's account was just created, avoid sending an email. if getattr(user, "just_registered", False): return user_agent = request.META.get('HTTP_USER_AGENT', "").lower() context = common_context(user) context['user_email'] = user.email user_tz = user.timezone if user_tz == '': user_tz = timezone_get_current_timezone_name() local_time = timezone_now().astimezone(get_timezone(user_tz)) context['login_time'] = local_time.strftime('%A, %B %d, %Y at %I:%M%p ') + user_tz context['device_ip'] = request.META.get('REMOTE_ADDR') or _("Unknown IP address") context['device_os'] = get_device_os(user_agent) context['device_browser'] = get_device_browser(user_agent) email_dict = { 'template_prefix': 'zerver/emails/notify_new_login', 'to_user_id': user.id, 'from_name': 'Zulip Account Security', 'from_address': FromAddress.NOREPLY, 'context': context} queue_json_publish("email_senders", email_dict)
def emit(self, record): # type: (ExceptionReporter) -> None try: request = record.request # type: HttpRequest exception_filter = get_exception_reporter_filter(request) if record.exc_info: stack_trace = ''.join(traceback.format_exception(*record.exc_info)) else: stack_trace = None try: user_profile = request.user user_full_name = user_profile.full_name user_email = user_profile.email except Exception: traceback.print_exc() # Error was triggered by an anonymous user. user_full_name = None user_email = None data = request.GET if request.method == 'GET' else \ exception_filter.get_post_parameters(request) report = dict( node = platform.node(), method = request.method, path = request.path, data = data, remote_addr = request.META.get('REMOTE_ADDR', None), query_string = request.META.get('QUERY_STRING', None), server_name = request.META.get('SERVER_NAME', None), message = record.getMessage(), stack_trace = stack_trace, user_full_name = user_full_name, user_email = user_email, ) except Exception: traceback.print_exc() report = dict( node = platform.node(), message = record.getMessage(), ) try: if settings.STAGING_ERROR_NOTIFICATIONS: # On staging, process the report directly so it can happen inside this # try/except to prevent looping from zilencer.error_notify import notify_server_error notify_server_error(report) else: queue_json_publish('error_reports', dict( type = "server", report = report, ), lambda x: None) except Exception: # If this breaks, complain loudly but don't pass the traceback up the stream # However, we *don't* want to use logging.exception since that could trigger a loop. logging.warning("Reporting an exception triggered an exception!", exc_info=True)
def handle(self, *args, **options): # type: (*Any, **str) -> None queue_name = options['queue_name'] file_name = options['file_name'] if file_name == '-': f = sys.stdin # type: IO[str] else: f = open(file_name) while True: line = f.readline() if not line: break line = line.strip() try: payload = line.split('\t')[1] except IndexError: payload = line print('Queueing to queue %s: %s' % (queue_name, payload)) # Verify that payload is valid json. data = ujson.loads(payload) queue_json_publish(queue_name, data, error)
def on_message(self, msg_raw): # type: (str) -> None log_data = dict(extra='[transport=%s' % (self.session.transport_name,)) record_request_start_data(log_data) msg = ujson.loads(msg_raw) if self.did_close: logger.info("Received message on already closed socket! transport=%s user=%s client_id=%s" % (self.session.transport_name, self.session.user_profile.email if self.session.user_profile is not None else 'unknown', self.client_id)) self.session.send_message({'req_id': msg['req_id'], 'type': 'ack'}) if msg['type'] == 'auth': log_data['extra'] += ']' try: self.authenticate_client(msg) # TODO: Fill in the correct client write_log_line(log_data, path='/socket/auth', method='SOCKET', remote_ip=self.session.conn_info.ip, email=self.session.user_profile.email, client_name='?') except SocketAuthError as e: response = {'result': 'error', 'msg': e.msg} self.session.send_message({'req_id': msg['req_id'], 'type': 'response', 'response': response}) write_log_line(log_data, path='/socket/auth', method='SOCKET', remote_ip=self.session.conn_info.ip, email='unknown', client_name='?', status_code=403, error_content=ujson.dumps(response)) return else: if not self.authenticated: response = {'result': 'error', 'msg': "Not yet authenticated"} self.session.send_message({'req_id': msg['req_id'], 'type': 'response', 'response': response}) write_log_line(log_data, path='/socket/service_request', method='SOCKET', remote_ip=self.session.conn_info.ip, email='unknown', client_name='?', status_code=403, error_content=ujson.dumps(response)) return redis_key = req_redis_key(msg['req_id']) with redis_client.pipeline() as pipeline: pipeline.hmset(redis_key, {'status': 'received'}) pipeline.expire(redis_key, 60 * 60 * 24) pipeline.execute() record_request_stop_data(log_data) queue_json_publish("message_sender", dict(request=msg['request'], req_id=msg['req_id'], server_meta=dict(user_id=self.session.user_profile.id, client_id=self.client_id, return_queue="tornado_return", log_data=log_data, request_environ=dict(REMOTE_ADDR=self.session.conn_info.ip))), fake_message_sender)
def send_event(event, users): # type: (Mapping[str, Any], Union[Iterable[int], Iterable[Mapping[str, Any]]]) -> None """`users` is a list of user IDs, or in the case of `message` type events, a list of dicts describing the users and metadata about the user/message pair.""" queue_json_publish("notify_tornado", dict(event=event, users=users), send_notification_http)
def send_event(realm: Realm, event: Mapping[str, Any], users: Union[Iterable[int], Iterable[Mapping[str, Any]]]) -> None: """`users` is a list of user IDs, or in the case of `message` type events, a list of dicts describing the users and metadata about the user/message pair.""" port = get_tornado_port(realm) queue_json_publish(notify_tornado_queue_name(port), dict(event=event, users=users), lambda *args, **kwargs: send_notification_http(realm, *args, **kwargs))
def request_retry(event, failure_message): # type: (Dict[str, Any], Text) -> None event['failed_tries'] += 1 if event['failed_tries'] > MAX_REQUEST_RETRIES: bot_user = get_user_profile_by_id(event['user_profile_id']) failure_message = "Maximum retries exceeded! " + failure_message fail_with_message(event, failure_message) logging.warning("Maximum retries exceeded for trigger:%s event:%s" % (bot_user.email, event['command'])) else: queue_json_publish("outgoing_webhooks", event, lambda x: None)
def consume(self, event): # type: (Mapping[str, Any]) -> None server_meta = event['server_meta'] environ = { 'REQUEST_METHOD': 'SOCKET', 'SCRIPT_NAME': '', 'PATH_INFO': '/json/messages', 'SERVER_NAME': '127.0.0.1', 'SERVER_PORT': 9993, 'SERVER_PROTOCOL': 'ZULIP_SOCKET/1.0', 'wsgi.version': (1, 0), 'wsgi.input': StringIO(), 'wsgi.errors': sys.stderr, 'wsgi.multithread': False, 'wsgi.multiprocess': True, 'wsgi.run_once': False, 'zulip.emulated_method': 'POST' } if 'socket_user_agent' in event['request']: environ['HTTP_USER_AGENT'] = event['request']['socket_user_agent'] del event['request']['socket_user_agent'] # We're mostly using a WSGIRequest for convenience environ.update(server_meta['request_environ']) request = WSGIRequest(environ) # Note: If we ever support non-POST methods, we'll need to change this. request._post = event['request'] request.csrf_processing_done = True user_profile = get_user_profile_by_id(server_meta['user_id']) request._cached_user = user_profile resp = self.handler.get_response(request) server_meta['time_request_finished'] = time.time() server_meta['worker_log_data'] = request._log_data resp_content = resp.content.decode('utf-8') response_data = ujson.loads(resp_content) if response_data['result'] == 'error': check_and_send_restart_signal() result = {'response': response_data, 'req_id': event['req_id'], 'server_meta': server_meta} redis_key = req_redis_key(event['req_id']) self.redis_client.hmset(redis_key, {'status': 'complete', 'response': resp_content}) # Since this sends back to Tornado, we can't use # call_consume_in_tests here. queue_json_publish(server_meta['return_queue'], result, lambda e: None)
def emit(self, record: logging.LogRecord) -> None: report = {} # type: Dict[str, Any] try: report['node'] = platform.node() report['host'] = platform.node() add_deployment_metadata(report) if record.exc_info: stack_trace = ''.join(traceback.format_exception(*record.exc_info)) message = str(record.exc_info[1]) else: stack_trace = 'No stack trace available' message = record.getMessage() if '\n' in message: # Some exception code paths in queue processors # seem to result in super-long messages stack_trace = message message = message.split('\n')[0] report['stack_trace'] = stack_trace report['message'] = message report['logger_name'] = record.name report['log_module'] = find_log_caller_module(record) report['log_lineno'] = record.lineno if hasattr(record, "request"): add_request_metadata(report, record.request) # type: ignore # record.request is added dynamically except Exception: report['message'] = "Exception in preparing exception report!" logging.warning(report['message'], exc_info=True) report['stack_trace'] = "See /var/log/zulip/errors.log" try: if settings.STAGING_ERROR_NOTIFICATIONS: # On staging, process the report directly so it can happen inside this # try/except to prevent looping from zerver.lib.error_notify import notify_server_error notify_server_error(report) else: queue_json_publish('error_reports', dict( type = "server", report = report, )) except Exception: # If this breaks, complain loudly but don't pass the traceback up the stream # However, we *don't* want to use logging.exception since that could trigger a loop. logging.warning("Reporting an exception triggered an exception!", exc_info=True)
def handle(self, *args, **options): # type: (*Any, **str) -> None if six.PY3: print(py3_warning) return rcpt_to = os.environ.get("ORIGINAL_RECIPIENT", options['recipient']) if rcpt_to is not None: if is_missed_message_address(rcpt_to): try: mark_missed_message_address_as_used(rcpt_to) except ZulipEmailForwardError: print("5.1.1 Bad destination mailbox address: Bad or expired missed message address.") exit(posix.EX_NOUSER) # type: ignore # There are no stubs for posix in python 3 else: try: extract_and_validate(rcpt_to) except ZulipEmailForwardError: print("5.1.1 Bad destination mailbox address: Please use the address specified " "in your Streams page.") exit(posix.EX_NOUSER) # type: ignore # There are no stubs for posix in python 3 # Read in the message, at most 25MiB. This is the limit enforced by # Gmail, which we use here as a decent metric. message = sys.stdin.read(25*1024*1024) if len(sys.stdin.read(1)) != 0: # We're not at EOF, reject large mail. print("5.3.4 Message too big for system: Max size is 25MiB") exit(posix.EX_DATAERR) # type: ignore # There are no stubs for posix in python 3 queue_json_publish( "email_mirror", { "message": message, "rcpt_to": rcpt_to }, lambda x: None ) else: # We're probably running from cron, try to batch-process mail if (not settings.EMAIL_GATEWAY_BOT or not settings.EMAIL_GATEWAY_LOGIN or not settings.EMAIL_GATEWAY_PASSWORD or not settings.EMAIL_GATEWAY_IMAP_SERVER or not settings.EMAIL_GATEWAY_IMAP_PORT or not settings.EMAIL_GATEWAY_IMAP_FOLDER): print("Please configure the Email Mirror Gateway in /etc/zulip/, " "or specify $ORIGINAL_RECIPIENT if piping a single mail.") exit(1) reactor.callLater(0, main) reactor.run()
def test_register_consumer(self) -> None: output = [] queue_client = get_queue_client() def collect(event: Dict[str, Any]) -> None: output.append(event) queue_client.stop_consuming() queue_client.register_json_consumer("test_suite", collect) queue_json_publish("test_suite", {"event": "my_event"}) queue_client.start_consuming() self.assertEqual(len(output), 1) self.assertEqual(output[0]['event'], 'my_event')
def update_user_activity(request, user_profile): # update_active_status also pushes to rabbitmq, and it seems # redundant to log that here as well. if request.META["PATH_INFO"] == '/json/users/me/presence': return if hasattr(request, '_query'): query = request._query else: query = request.META['PATH_INFO'] event={'query': query, 'user_profile_id': user_profile.id, 'time': datetime_to_timestamp(now()), 'client': request.client.name} queue_json_publish("user_activity", event, lambda event: None)
def emit(self, record): # type: (logging.LogRecord) -> None try: if record.exc_info: stack_trace = ''.join(traceback.format_exception(*record.exc_info)) # type: Optional[str] message = str(record.exc_info[1]) else: stack_trace = None message = record.getMessage() if '\n' in message: # Some exception code paths in queue processors # seem to result in super-long messages stack_trace = message message = message.split('\n')[0] report = dict( node = platform.node(), host = platform.node(), message = message, stack_trace = stack_trace, ) if hasattr(record, "request"): add_request_metadata(report, record.request) # type: ignore # record.request is added dynamically except Exception: traceback.print_exc() report = dict( node = platform.node(), host = platform.node(), message = record.getMessage(), stack_trace = "See /var/log/zulip/errors.log", ) try: if settings.STAGING_ERROR_NOTIFICATIONS: # On staging, process the report directly so it can happen inside this # try/except to prevent looping from zerver.lib.error_notify import notify_server_error notify_server_error(report) else: queue_json_publish('error_reports', dict( type = "server", report = report, ), lambda x: None) except Exception: # If this breaks, complain loudly but don't pass the traceback up the stream # However, we *don't* want to use logging.exception since that could trigger a loop. logging.warning("Reporting an exception triggered an exception!", exc_info=True)
def handle(self, *args: Any, **options: Any) -> None: print("Purging queue...") queue = SimpleQueueClient() queue_name = "noop_batch" if options["batch"] else "noop" queue.ensure_queue(queue_name, lambda channel: channel.queue_purge("noop")) count = options["count"] reps = options["reps"] with open(options["csv"], "w", newline="") as csvfile: writer = csv.DictWriter( csvfile, fieldnames=["Queue size", "Queue type", "Prefetch", "Rate"]) writer.writeheader() for prefetch in options["prefetches"]: print(f"Queue size {count}, prefetch {prefetch}...") worker: Union[NoopWorker, BatchNoopWorker] = NoopWorker( count, options["slow"]) if options["batch"]: worker = BatchNoopWorker(count, options["slow"]) if prefetch > 0 and prefetch < worker.batch_size: print( f" Skipping, as prefetch {prefetch} is less than batch size {worker.batch_size}" ) continue worker.ENABLE_TIMEOUTS = True worker.setup() assert worker.q is not None assert worker.q.channel is not None worker.q.channel.basic_qos(prefetch_count=prefetch) total_time = 0.0 for i in range(1, reps + 1): worker.consumed = 0 timeit( lambda: queue_json_publish(queue_name, {}), number=count, ) duration = timeit( lambda: worker.start(), number=1, ) print( f" {i}/{reps}: {count}/{duration}s = {count / duration}/s" ) total_time += duration writer.writerow({ "Queue size": count, "Queue type": queue_name, "Prefetch": prefetch, "Rate": count / duration, }) csvfile.flush() print( f" Overall: {reps * count}/{total_time}s = {(reps * count) / total_time}/s" )
def maybe_enqueue_notifications(user_profile_id: int, message_id: int, private_message: bool, mentioned: bool, stream_push_notify: bool, stream_email_notify: bool, stream_name: Optional[str], always_push_notify: bool, idle: bool, already_notified: Dict[str, bool]) -> Dict[str, bool]: """This function has a complete unit test suite in `test_enqueue_notifications` that should be expanded as we add more features here.""" notified = dict() # type: Dict[str, bool] if (idle or always_push_notify) and (private_message or mentioned or stream_push_notify): notice = build_offline_notification(user_profile_id, message_id) if private_message: notice['trigger'] = 'private_message' elif mentioned: notice['trigger'] = 'mentioned' elif stream_push_notify: notice['trigger'] = 'stream_push_notify' else: raise AssertionError("Unknown notification trigger!") notice['stream_name'] = stream_name if not already_notified.get("push_notified"): queue_json_publish("missedmessage_mobile_notifications", notice) notified['push_notified'] = True # Send missed_message emails if a private message or a # mention. Eventually, we'll add settings to allow email # notifications to match the model of push notifications # above. if idle and (private_message or mentioned or stream_email_notify): notice = build_offline_notification(user_profile_id, message_id) if private_message: notice['trigger'] = 'private_message' elif mentioned: notice['trigger'] = 'mentioned' elif stream_email_notify: notice['trigger'] = 'stream_email_notify' else: raise AssertionError("Unknown notification trigger!") notice['stream_name'] = stream_name if not already_notified.get("email_notified"): queue_json_publish("missedmessage_emails", notice, lambda notice: None) notified['email_notified'] = True return notified
def email_on_new_login(sender: Any, user: UserProfile, request: Any, **kwargs: Any) -> None: if not user.enable_login_emails: return # We import here to minimize the dependencies of this module, # since it runs as part of `manage.py` initialization from zerver.context_processors import common_context if not settings.SEND_LOGIN_EMAILS: return if request: # If the user's account was just created, avoid sending an email. if (timezone_now() - user.date_joined).total_seconds() <= JUST_CREATED_THRESHOLD: return user_agent = request.META.get('HTTP_USER_AGENT', "").lower() context = common_context(user) context['user_email'] = user.email user_tz = user.timezone if user_tz == '': user_tz = timezone_get_current_timezone_name() local_time = timezone_now().astimezone(get_timezone(user_tz)) if user.twenty_four_hour_time: hhmm_string = local_time.strftime('%H:%M') else: hhmm_string = local_time.strftime('%I:%M%p') context['login_time'] = local_time.strftime( '%A, %B %d, %Y at {} %Z'.format(hhmm_string)) context['device_ip'] = request.META.get('REMOTE_ADDR') or _( "Unknown IP address") context['device_os'] = get_device_os(user_agent) context['device_browser'] = get_device_browser(user_agent) context['unsubscribe_link'] = one_click_unsubscribe_link(user, 'login') email_dict = { 'template_prefix': 'zerver/emails/notify_new_login', 'to_user_id': user.id, 'from_name': 'Zulip Account Security', 'from_address': FromAddress.NOREPLY, 'context': context } queue_json_publish("email_senders", email_dict)
def handle(self, *args, **options): # type: (*Any, **str) -> None rcpt_to = force_text(os.environ.get("ORIGINAL_RECIPIENT", options['recipient'])) if rcpt_to is not None: if is_missed_message_address(rcpt_to): try: mark_missed_message_address_as_used(rcpt_to) except ZulipEmailForwardError: print("5.1.1 Bad destination mailbox address: Bad or expired missed message address.") exit(posix.EX_NOUSER) # type: ignore # There are no stubs for posix in python 3 else: try: extract_and_validate(rcpt_to) except ZulipEmailForwardError: print("5.1.1 Bad destination mailbox address: Please use the address specified " "in your Streams page.") exit(posix.EX_NOUSER) # type: ignore # There are no stubs for posix in python 3 # Read in the message, at most 25MiB. This is the limit enforced by # Gmail, which we use here as a decent metric. msg_text = sys.stdin.read(25*1024*1024) if len(sys.stdin.read(1)) != 0: # We're not at EOF, reject large mail. print("5.3.4 Message too big for system: Max size is 25MiB") exit(posix.EX_DATAERR) # type: ignore # There are no stubs for posix in python 3 queue_json_publish( "email_mirror", { "message": msg_text, "rcpt_to": rcpt_to }, lambda x: None ) else: # We're probably running from cron, try to batch-process mail if (not settings.EMAIL_GATEWAY_BOT or not settings.EMAIL_GATEWAY_LOGIN or not settings.EMAIL_GATEWAY_PASSWORD or not settings.EMAIL_GATEWAY_IMAP_SERVER or not settings.EMAIL_GATEWAY_IMAP_PORT or not settings.EMAIL_GATEWAY_IMAP_FOLDER): print("Please configure the Email Mirror Gateway in /etc/zulip/, " "or specify $ORIGINAL_RECIPIENT if piping a single mail.") exit(1) for message in get_imap_messages(): process_message(message)
def json_report_error(request, user_profile, message=REQ(), stacktrace=REQ(), ui_message=REQ(validator=check_bool), user_agent=REQ(), href=REQ(), log=REQ(), more_info=REQ(validator=check_dict([]), default=None)): # type: (HttpRequest, UserProfile, Text, Text, bool, Text, Text, Text, Dict[str, Any]) -> HttpResponse """Accepts an error report and stores in a queue for processing. The actual error reports are later handled by do_report_error (below)""" if not settings.BROWSER_ERROR_REPORTING: return json_success() js_source_map = get_js_source_map() if js_source_map: stacktrace = js_source_map.annotate_stacktrace(stacktrace) try: version = subprocess.check_output( ["git", "log", "HEAD^..HEAD", "--oneline"], universal_newlines=True) # type: Optional[Text] except Exception: version = None queue_json_publish( 'error_reports', dict(type="browser", report=dict( host=request.get_host().split(":")[0], user_email=user_profile.email, user_full_name=user_profile.full_name, user_visible=ui_message, server_path=settings.DEPLOY_ROOT, version=version, user_agent=user_agent, href=href, message=message, stacktrace=stacktrace, log=log, more_info=more_info, )), lambda x: None) return json_success()
def export_realm(request: HttpRequest, user: UserProfile) -> HttpResponse: # Currently only supports public-data-only exports. event_type = RealmAuditLog.REALM_EXPORTED event_time = timezone_now() realm = user.realm EXPORT_LIMIT = 5 # Conservative limit on the size of message history in # organizations being exported; this exists to protect Zulip # against a possible unmonitored accidental DoS caused by trying # to export an organization with huge history. MAX_MESSAGE_HISTORY = 250000 MAX_UPLOAD_QUOTA = 10 * 1024 * 1024 * 1024 # Filter based upon the number of events that have occurred in the delta # If we are at the limit, the incoming request is rejected event_time_delta = event_time - timedelta(days=7) limit_check = RealmAuditLog.objects.filter( realm=realm, event_type=event_type, event_time__gte=event_time_delta) if len(limit_check) >= EXPORT_LIMIT: return json_error(_('Exceeded rate limit.')) total_messages = sum( realm_count.value for realm_count in RealmCount.objects.filter( realm=user.realm, property='messages_sent:client:day')) if (total_messages > MAX_MESSAGE_HISTORY or user.realm.currently_used_upload_space_bytes() > MAX_UPLOAD_QUOTA): return json_error( _('Please request a manual export from %s.') % (settings.ZULIP_ADMINISTRATOR, )) row = RealmAuditLog.objects.create(realm=realm, event_type=event_type, event_time=event_time, acting_user=user) # Using the deferred_work queue processor to avoid # killing the process after 60s event = { 'type': "realm_export", 'time': event_time, 'realm_id': realm.id, 'user_profile_id': user.id, 'id': row.id } queue_json_publish('deferred_work', event) return json_success()
def consume(self, event): server_meta = event['server_meta'] environ = { 'REQUEST_METHOD': 'SOCKET', 'SCRIPT_NAME': '', 'PATH_INFO': '/json/messages', 'SERVER_NAME': 'localhost', 'SERVER_PORT': 9993, 'SERVER_PROTOCOL': 'ZULIP_SOCKET/1.0', 'wsgi.version': (1, 0), 'wsgi.input': StringIO.StringIO(), 'wsgi.errors': sys.stderr, 'wsgi.multithread': False, 'wsgi.multiprocess': True, 'wsgi.run_once': False, 'zulip.emulated_method': 'POST' } # We're mostly using a WSGIRequest for convenience environ.update(server_meta['request_environ']) request = WSGIRequest(environ) request._request = event['request'] request.csrf_processing_done = True user_profile = get_user_profile_by_id(server_meta['user_id']) request._cached_user = user_profile resp = self.handler.get_response(request) server_meta['time_request_finished'] = time.time() server_meta['worker_log_data'] = request._log_data resp_content = resp.content result = { 'response': ujson.loads(resp_content), 'req_id': event['req_id'], 'server_meta': server_meta } redis_key = req_redis_key(event['req_id']) self.redis_client.hmset(redis_key, { 'status': 'complete', 'response': resp_content }) queue_json_publish(server_meta['return_queue'], result, lambda e: None)
def update_user_presence( user_profile: UserProfile, client: Client, log_time: datetime.datetime, status: int, new_user_input: bool, ) -> None: event = { "user_profile_id": user_profile.id, "status": status, "time": datetime_to_timestamp(log_time), "client": client.name, } queue_json_publish("user_presence", event) if new_user_input: update_user_activity_interval(user_profile, log_time)
def mirror_email_message(data: Dict[str, str]) -> Dict[str, str]: rcpt_to = data['recipient'] try: validate_to_address(rcpt_to) except ZulipEmailForwardError as e: return { "status": "error", "msg": "5.1.1 Bad destination mailbox address: {}".format(e) } queue_json_publish( "email_mirror", { "message": data['msg_text'], "rcpt_to": rcpt_to } ) return {"status": "success"}
def update_user_activity(request, user_profile): # update_active_status also pushes to rabbitmq, and it seems # redundant to log that here as well. if request.META["PATH_INFO"] == '/json/update_active_status': return if hasattr(request, '_query'): query = request._query else: query = request.META['PATH_INFO'] event = { 'query': query, 'user_profile_id': user_profile.id, 'time': datetime_to_timestamp(now()), 'client': request.client.name } queue_json_publish("user_activity", event, lambda event: None)
def test_queue_basics_json_error(self) -> None: queue_json_publish("test_suite", {"event": "my_event"}) queue_client = get_queue_client() raised = False try: with queue_client.json_drain_queue("test_suite") as result: self.assertEqual(len(result), 1) self.assertEqual(result[0]['event'], 'my_event') raise ValueError() except ValueError: raised = True assert raised # Still in the queue to be fetched with queue_client.json_drain_queue("test_suite") as result: self.assertEqual(len(result), 1) self.assertEqual(result[0]['event'], 'my_event')
def one_rep() -> None: nonlocal total_enqueue_time, total_dequeue_time total_enqueue_time += timeit( lambda: queue_json_publish(queue_name, {}), number=count, ) total_dequeue_time += timeit( lambda: worker.start(), number=1, )
def update_user_activity(request, user_profile): # type: (HttpRequest, UserProfile) -> None # update_active_status also pushes to rabbitmq, and it seems # redundant to log that here as well. if request.META["PATH_INFO"] == "/json/users/me/presence": return if hasattr(request, "_query"): query = request._query else: query = request.META["PATH_INFO"] event = { "query": query, "user_profile_id": user_profile.id, "time": datetime_to_timestamp(now()), "client": request.client.name, } queue_json_publish("user_activity", event, lambda event: None)
def update_user_activity(request: HttpRequest, user_profile: UserProfile, query: Optional[str]) -> None: # update_active_status also pushes to rabbitmq, and it seems # redundant to log that here as well. if request.META["PATH_INFO"] == '/json/users/me/presence': return if query is not None: pass elif hasattr(request, '_query'): query = request._query else: query = request.META['PATH_INFO'] event = {'query': query, 'user_profile_id': user_profile.id, 'time': datetime_to_timestamp(timezone_now()), 'client_id': request.client.id} queue_json_publish("user_activity", event, lambda event: None)
def do_regenerate_api_key(user_profile: UserProfile, acting_user: UserProfile) -> str: old_api_key = user_profile.api_key new_api_key = generate_api_key() user_profile.api_key = new_api_key user_profile.save(update_fields=["api_key"]) # We need to explicitly delete the old API key from our caches, # because the on-save handler for flushing the UserProfile object # in zerver/lib/cache.py only has access to the new API key. cache_delete(user_profile_by_api_key_cache_key(old_api_key)) event_time = timezone_now() RealmAuditLog.objects.create( realm=user_profile.realm, acting_user=acting_user, modified_user=user_profile, event_type=RealmAuditLog.USER_API_KEY_CHANGED, event_time=event_time, ) if user_profile.is_bot: send_event( user_profile.realm, dict( type="realm_bot", op="update", bot=dict( user_id=user_profile.id, api_key=new_api_key, ), ), bot_owner_user_ids(user_profile), ) event = { "type": "clear_push_device_tokens", "user_profile_id": user_profile.id } queue_json_publish("deferred_work", event) return new_api_key
def test_queue_error_json(self) -> None: queue_client = get_queue_client() actual_publish = queue_client.publish self.counter = 0 def throw_connection_error_once(self_obj: Any, *args: Any, **kwargs: Any) -> None: self.counter += 1 if self.counter <= 1: raise AMQPConnectionError("test") actual_publish(*args, **kwargs) with mock.patch("zerver.lib.queue.SimpleQueueClient.publish", throw_connection_error_once): queue_json_publish("test_suite", {"event": "my_event"}) result = queue_client.drain_queue("test_suite", json=True) self.assertEqual(len(result), 1) self.assertEqual(result[0]['event'], 'my_event')
def email_on_new_login(sender: Any, user: UserProfile, request: Any, **kwargs: Any) -> None: if not user.enable_login_emails: return # We import here to minimize the dependencies of this module, # since it runs as part of `manage.py` initialization from zerver.context_processors import common_context if not settings.SEND_LOGIN_EMAILS: return if request: # If the user's account was just created, avoid sending an email. if (timezone_now() - user.date_joined).total_seconds() <= JUST_CREATED_THRESHOLD: return user_agent = request.headers.get("User-Agent", "").lower() context = common_context(user) context["user_email"] = user.delivery_email user_tz = user.timezone if user_tz == "": user_tz = timezone_get_current_timezone_name() local_time = timezone_now().astimezone(pytz.timezone(user_tz)) if user.twenty_four_hour_time: hhmm_string = local_time.strftime("%H:%M") else: hhmm_string = local_time.strftime("%I:%M%p") context["login_time"] = local_time.strftime(f"%A, %B %d, %Y at {hhmm_string} %Z") context["device_ip"] = request.META.get("REMOTE_ADDR") or _("Unknown IP address") context["device_os"] = get_device_os(user_agent) or _("an unknown operating system") context["device_browser"] = get_device_browser(user_agent) or _("An unknown browser") context["unsubscribe_link"] = one_click_unsubscribe_link(user, "login") email_dict = { "template_prefix": "zerver/emails/notify_new_login", "to_user_ids": [user.id], "from_name": FromAddress.security_email_from_name(user_profile=user), "from_address": FromAddress.NOREPLY, "context": context, } queue_json_publish("email_senders", email_dict)
def json_report_error(request, user_profile, message=REQ(), stacktrace=REQ(), ui_message=REQ(validator=check_bool), user_agent=REQ(), href=REQ(), log=REQ(), more_info=REQ(validator=check_dict([]), default=None)): # type: (HttpRequest, UserProfile, text_type, text_type, bool, text_type, text_type, text_type, Dict[str, Any]) -> HttpResponse if not settings.ERROR_REPORTING: return json_success() if js_source_map: stacktrace = js_source_map.annotate_stacktrace(stacktrace) try: version = subprocess.check_output( ["git", "log", "HEAD^..HEAD", "--oneline"], universal_newlines=True) except Exception: version = None queue_json_publish( 'error_reports', dict(type="browser", report=dict( user_email=user_profile.email, user_full_name=user_profile.full_name, user_visible=ui_message, server_path=settings.DEPLOY_ROOT, version=version, user_agent=user_agent, href=href, message=message, stacktrace=stacktrace, log=log, more_info=more_info, )), lambda x: None) return json_success()
def handle(self, *args, **options): rcpt_to = os.environ.get("ORIGINAL_RECIPIENT", options['recipient']) if rcpt_to is not None: if is_missed_message_address(rcpt_to): try: mark_missed_message_address_as_used(rcpt_to) except ZulipEmailForwardError: print "5.1.1 Bad destination mailbox address: Bad or expired missed message address." exit(posix.EX_NOUSER) else: try: extract_and_validate(rcpt_to) except ZulipEmailForwardError: print "5.1.1 Bad destination mailbox address: Please use the address specified in your Streams page." exit(posix.EX_NOUSER) # Read in the message, at most 25MiB. This is the limit enforced by # Gmail, which we use here as a decent metric. message = sys.stdin.read(25 * 1024 * 1024) if len(sys.stdin.read(1)) != 0: # We're not at EOF, reject large mail. print "5.3.4 Message too big for system: Max size is 25MiB" exit(posix.EX_DATAERR) queue_json_publish("email_mirror", { "message": message, "rcpt_to": rcpt_to }, lambda x: None) else: # We're probably running from cron, try to batch-process mail if (not settings.EMAIL_GATEWAY_BOT or not settings.EMAIL_GATEWAY_LOGIN or not settings.EMAIL_GATEWAY_PASSWORD or not settings.EMAIL_GATEWAY_IMAP_SERVER or not settings.EMAIL_GATEWAY_IMAP_PORT or not settings.EMAIL_GATEWAY_IMAP_FOLDER): print "Please configure the Email Mirror Gateway in your local_settings.py, or specify $ORIGINAL_RECIPIENT if piping a single mail." exit(1) reactor.callLater(0, main) reactor.run()
def update_user_activity(request: HttpRequest, user_profile: UserProfile, query: Optional[str]) -> None: # update_active_status also pushes to RabbitMQ, and it seems # redundant to log that here as well. if request.META["PATH_INFO"] == "/json/users/me/presence": return if query is not None: pass elif hasattr(request, "_query"): query = request._query else: query = request.META["PATH_INFO"] event = { "query": query, "user_profile_id": user_profile.id, "time": datetime_to_timestamp(timezone_now()), "client_id": request.client.id, } queue_json_publish("user_activity", event, lambda event: None)
def email_on_new_login(sender: Any, user: UserProfile, request: Any, **kwargs: Any) -> None: if not user.enable_login_emails: return # We import here to minimize the dependencies of this module, # since it runs as part of `manage.py` initialization from zerver.context_processors import common_context if not settings.SEND_LOGIN_EMAILS: return if request: # If the user's account was just created, avoid sending an email. if (timezone_now() - user.date_joined).total_seconds() <= JUST_CREATED_THRESHOLD: return user_agent = request.META.get('HTTP_USER_AGENT', "").lower() context = common_context(user) context['user_email'] = user.email user_tz = user.timezone if user_tz == '': user_tz = timezone_get_current_timezone_name() local_time = timezone_now().astimezone(get_timezone(user_tz)) if user.twenty_four_hour_time: hhmm_string = local_time.strftime('%H:%M') else: hhmm_string = local_time.strftime('%I:%M%p') context['login_time'] = local_time.strftime('%A, %B %d, %Y at {} %Z'.format(hhmm_string)) context['device_ip'] = request.META.get('REMOTE_ADDR') or _("Unknown IP address") context['device_os'] = get_device_os(user_agent) or _("an unknown operating system") context['device_browser'] = get_device_browser(user_agent) or _("An unknown browser") context['unsubscribe_link'] = one_click_unsubscribe_link(user, 'login') email_dict = { 'template_prefix': 'zerver/emails/notify_new_login', 'to_user_ids': [user.id], 'from_name': 'Zulip Account Security', 'from_address': FromAddress.NOREPLY, 'context': context} queue_json_publish("email_senders", email_dict)
def maybe_enqueue_notifications(user_profile_id, message_id, private_message, mentioned, stream_push_notify, stream_name, always_push_notify, idle, already_notified): # type: (int, int, bool, bool, bool, Optional[str], bool, bool, Dict[str, bool]) -> Dict[str, bool] """This function has a complete unit test suite in `test_enqueue_notifications` that should be expanded as we add more features here.""" notified = dict() # type: Dict[str, bool] if (idle or always_push_notify) and (private_message or mentioned or stream_push_notify): notice = build_offline_notification(user_profile_id, message_id) if private_message: notice['trigger'] = 'private_message' elif mentioned: notice['trigger'] = 'mentioned' elif stream_push_notify: notice['trigger'] = 'stream_push_notify' else: raise AssertionError("Unknown notification trigger!") notice['stream_name'] = stream_name if not already_notified.get("push_notified"): queue_json_publish("missedmessage_mobile_notifications", notice, lambda notice: None) notified['push_notified'] = True # Send missed_message emails if a private message or a # mention. Eventually, we'll add settings to allow email # notifications to match the model of push notifications # above. if idle and (private_message or mentioned): # We require RabbitMQ to do this, as we can't call the email handler # from the Tornado process. So if there's no rabbitmq support do nothing if not already_notified.get("email_notified"): queue_json_publish("missedmessage_emails", notice, lambda notice: None) notified['email_notified'] = True return notified
def email_on_new_login(sender: Any, user: UserProfile, request: Any, **kwargs: Any) -> None: # We import here to minimize the dependencies of this module, # since it runs as part of `manage.py` initialization from zerver.context_processors import common_context if not settings.SEND_LOGIN_EMAILS: return if request: # If the user's account was just created, avoid sending an email. if getattr(user, "just_registered", False): return login_time = timezone_now().strftime('%A, %B %d, %Y at %I:%M%p ') + \ timezone_get_current_timezone_name() user_agent = request.META.get('HTTP_USER_AGENT', "").lower() device_browser = get_device_browser(user_agent) device_os = get_device_os(user_agent) device_ip = request.META.get('REMOTE_ADDR') or "Uknown IP address" device_info = { "device_browser": device_browser, "device_os": device_os, "device_ip": device_ip, "login_time": login_time } context = common_context(user) context['device_info'] = device_info context['user'] = user email_dict = { 'template_prefix': 'zerver/emails/notify_new_login', 'to_user_id': user.id, 'from_name': 'Zulip Account Security', 'from_address': FromAddress.NOREPLY, 'context': context } queue_json_publish("email_senders", email_dict)
def json_report_error(request, user_profile, message=REQ(), stacktrace=REQ(), ui_message=REQ(validator=check_bool), user_agent=REQ(), href=REQ(), log=REQ(), more_info=REQ(validator=check_dict([]), default=None)): # type: (HttpRequest, UserProfile, Text, Text, bool, Text, Text, Text, Dict[str, Any]) -> HttpResponse """Accepts an error report and stores in a queue for processing. The actual error reports are later handled by do_report_error (below)""" if not settings.BROWSER_ERROR_REPORTING: return json_success() js_source_map = get_js_source_map() if js_source_map: stacktrace = js_source_map.annotate_stacktrace(stacktrace) try: version = subprocess.check_output(["git", "log", "HEAD^..HEAD", "--oneline"], universal_newlines=True) # type: Optional[Text] except Exception: version = None queue_json_publish('error_reports', dict( type = "browser", report = dict( host = request.get_host().split(":")[0], user_email = user_profile.email, user_full_name = user_profile.full_name, user_visible = ui_message, server_path = settings.DEPLOY_ROOT, version = version, user_agent = user_agent, href = href, message = message, stacktrace = stacktrace, log = log, more_info = more_info, ) ), lambda x: None) return json_success()
def consume(self, event): # type: (Mapping[str, Any]) -> None server_meta = event['server_meta'] environ = {'REQUEST_METHOD': 'SOCKET', 'SCRIPT_NAME': '', 'PATH_INFO': '/json/messages', 'SERVER_NAME': 'localhost', 'SERVER_PORT': 9993, 'SERVER_PROTOCOL': 'ZULIP_SOCKET/1.0', 'wsgi.version': (1, 0), 'wsgi.input': StringIO(), 'wsgi.errors': sys.stderr, 'wsgi.multithread': False, 'wsgi.multiprocess': True, 'wsgi.run_once': False, 'zulip.emulated_method': 'POST'} # We're mostly using a WSGIRequest for convenience environ.update(server_meta['request_environ']) request = WSGIRequest(environ) request._request = event['request'] request.csrf_processing_done = True user_profile = get_user_profile_by_id(server_meta['user_id']) request._cached_user = user_profile resp = self.handler.get_response(request) server_meta['time_request_finished'] = time.time() server_meta['worker_log_data'] = request._log_data resp_content = resp.content.decode('utf-8') result = {'response': ujson.loads(resp_content), 'req_id': event['req_id'], 'server_meta': server_meta} redis_key = req_redis_key(event['req_id']) self.redis_client.hmset(redis_key, {'status': 'complete', 'response': resp_content}); queue_json_publish(server_meta['return_queue'], result, lambda e: None)
def consume(self, event): server_meta = event["server_meta"] environ = { "REQUEST_METHOD": "SOCKET", "SCRIPT_NAME": "", "PATH_INFO": "/json/messages", "SERVER_NAME": "localhost", "SERVER_PORT": 9993, "SERVER_PROTOCOL": "ZULIP_SOCKET/1.0", "wsgi.version": (1, 0), "wsgi.input": StringIO(), "wsgi.errors": sys.stderr, "wsgi.multithread": False, "wsgi.multiprocess": True, "wsgi.run_once": False, "zulip.emulated_method": "POST", } # We're mostly using a WSGIRequest for convenience environ.update(server_meta["request_environ"]) request = WSGIRequest(environ) request._request = event["request"] request.csrf_processing_done = True user_profile = get_user_profile_by_id(server_meta["user_id"]) request._cached_user = user_profile resp = self.handler.get_response(request) server_meta["time_request_finished"] = time.time() server_meta["worker_log_data"] = request._log_data resp_content = resp.content result = {"response": ujson.loads(resp_content), "req_id": event["req_id"], "server_meta": server_meta} redis_key = req_redis_key(event["req_id"]) self.redis_client.hmset(redis_key, {"status": "complete", "response": resp_content}) queue_json_publish(server_meta["return_queue"], result, lambda e: None)
def test_queue_error_json(self) -> None: queue_client = get_queue_client() actual_publish = queue_client.publish self.counter = 0 def throw_connection_error_once(self_obj: Any, *args: Any, **kwargs: Any) -> None: self.counter += 1 if self.counter <= 1: raise AMQPConnectionError("test") actual_publish(*args, **kwargs) with mock.patch("zerver.lib.queue.SimpleQueueClient.publish", throw_connection_error_once), self.assertLogs('zulip.queue', level='WARN') as warn_logs: queue_json_publish("test_suite", {"event": "my_event"}) self.assertEqual(warn_logs.output, [ 'WARNING:zulip.queue:Failed to send to rabbitmq, trying to reconnect and send again' ]) result = queue_client.json_drain_queue("test_suite") self.assertEqual(len(result), 1) self.assertEqual(result[0]['event'], 'my_event')
def wrapped_queue_json_publish(*args: Any, **kwargs: Any) -> None: self.create_mock_response(original_url) self.create_mock_response(edited_url) with self.settings(TEST_SUITE=False): with self.assertLogs(level="INFO") as info_logs: # Run the queue processor. This will simulate the event for original_url being # processed after the message has been edited. FetchLinksEmbedData().consume(event) self.assertTrue( "INFO:root:Time spent on get_link_embed_data for http://test.org/: " in info_logs.output[0] ) msg = Message.objects.select_related("sender").get(id=msg_id) # The content of the message has changed since the event for original_url has been created, # it should not be rendered. Another, up-to-date event will have been sent (edited_url). self.assertNotIn( f'<a href="{original_url}" title="The Rock">The Rock</a>', msg.rendered_content ) self.assertTrue(responses.assert_call_count(edited_url, 0)) with self.settings(TEST_SUITE=False): with self.assertLogs(level="INFO") as info_logs: # Now proceed with the original queue_json_publish and call the # up-to-date event for edited_url. queue_json_publish(*args, **kwargs) msg = Message.objects.select_related("sender").get(id=msg_id) assert msg.rendered_content is not None self.assertIn( f'<a href="{edited_url}" title="The Rock">The Rock</a>', msg.rendered_content, ) self.assertTrue( "INFO:root:Time spent on get_link_embed_data for http://edited.org/: " in info_logs.output[0] )
def test_queue_error_json(self) -> None: queue_client = get_queue_client() assert isinstance(queue_client, SimpleQueueClient) actual_publish = queue_client.publish self.counter = 0 def throw_connection_error_once(self_obj: Any, *args: Any, **kwargs: Any) -> None: self.counter += 1 if self.counter <= 1: raise AMQPConnectionError("test") actual_publish(*args, **kwargs) with mock.patch("zerver.lib.queue.SimpleQueueClient.publish", throw_connection_error_once), self.assertLogs( "zulip.queue", level="WARN") as warn_logs: queue_json_publish("test_suite", {"event": "my_event"}) self.assertEqual( warn_logs.output, [ "WARNING:zulip.queue:Failed to send to rabbitmq, trying to reconnect and send again" ], ) assert queue_client.channel method, header, message = queue_client.channel.basic_get("test_suite") assert method is not None assert method.delivery_tag is not None assert message is not None queue_client.channel.basic_ack(method.delivery_tag) result = orjson.loads(message) self.assertEqual(result["event"], "my_event") method, header, message = queue_client.channel.basic_get("test_suite") assert message is None
def set_emoji_author(apps: StateApps, schema_editor: DatabaseSchemaEditor) -> None: """ This migration establishes the invariant that all RealmEmoji objects have .author set and queues events for reuploading all RealmEmoji. """ RealmEmoji = apps.get_model("zerver", "RealmEmoji") Realm = apps.get_model("zerver", "Realm") UserProfile = apps.get_model("zerver", "UserProfile") ROLE_REALM_OWNER = 100 realm_emoji_to_update = [] for realm_emoji in RealmEmoji.objects.all(): if realm_emoji.author_id is None: user_profile = (UserProfile.objects.filter( realm_id=realm_emoji.realm_id, is_active=True, role=ROLE_REALM_OWNER).order_by("id").first()) realm_emoji.author_id = user_profile.id realm_emoji_to_update.append(realm_emoji) RealmEmoji.objects.bulk_update(realm_emoji_to_update, ["author_id"]) if settings.TEST_SUITE: # There are no custom emoji in the test suite data set, and # the below code won't work because RabbitMQ isn't enabled for # the test suite. return for realm_id in Realm.objects.order_by("id").values_list("id", flat=True): event = { "type": "reupload_realm_emoji", "realm_id": realm_id, } queue_json_publish("deferred_work", event)
def send_notification(data): # type: (Mapping[str, Any]) -> None queue_json_publish("notify_tornado", data, send_notification_http)
def process_message_event(event_template, users): # type: (Mapping[str, Any], Iterable[Mapping[str, Any]]) -> None missed_message_userids = set(event_template.get('missed_message_userids', [])) sender_queue_id = event_template.get('sender_queue_id', None) # type: Optional[str] message_dict_markdown = event_template['message_dict_markdown'] # type: Dict[str, Any] message_dict_no_markdown = event_template['message_dict_no_markdown'] # type: Dict[str, Any] sender_id = message_dict_markdown['sender_id'] # type: int message_id = message_dict_markdown['id'] # type: int message_type = message_dict_markdown['type'] # type: str sending_client = message_dict_markdown['client'] # type: Text # To remove duplicate clients: Maps queue ID to {'client': Client, 'flags': flags} send_to_clients = {} # type: Dict[str, Dict[str, Any]] # Extra user-specific data to include extra_user_data = {} # type: Dict[int, Any] if 'stream_name' in event_template and not event_template.get("invite_only"): for client in get_client_descriptors_for_realm_all_streams(event_template['realm_id']): send_to_clients[client.event_queue.id] = {'client': client, 'flags': None} if sender_queue_id is not None and client.event_queue.id == sender_queue_id: send_to_clients[client.event_queue.id]['is_sender'] = True for user_data in users: user_profile_id = user_data['id'] # type: int flags = user_data.get('flags', []) # type: Iterable[str] for client in get_client_descriptors_for_user(user_profile_id): send_to_clients[client.event_queue.id] = {'client': client, 'flags': flags} if sender_queue_id is not None and client.event_queue.id == sender_queue_id: send_to_clients[client.event_queue.id]['is_sender'] = True # If the recipient was offline and the message was a single or group PM to them # or they were @-notified potentially notify more immediately private_message = message_type == "private" and user_profile_id != sender_id mentioned = 'mentioned' in flags stream_push_notify = user_data.get('stream_push_notify', False) # We first check if a message is potentially mentionable, # since receiver_is_off_zulip is somewhat expensive. if private_message or mentioned or stream_push_notify: idle = receiver_is_off_zulip(user_profile_id) or (user_profile_id in missed_message_userids) always_push_notify = user_data.get('always_push_notify', False) notified = dict() # type: Dict[str, bool] if (idle or always_push_notify) and (private_message or mentioned or stream_push_notify): notice = build_offline_notification(user_profile_id, message_id) notice['triggers'] = { 'private_message': private_message, 'mentioned': mentioned, 'stream_push_notify': stream_push_notify, } notice['stream_name'] = event_template.get('stream_name') queue_json_publish("missedmessage_mobile_notifications", notice, lambda notice: None) notified['push_notified'] = True # Send missed_message emails if a private message or a # mention. Eventually, we'll add settings to allow email # notifications to match the model of push notifications # above. if idle and (private_message or mentioned): # We require RabbitMQ to do this, as we can't call the email handler # from the Tornado process. So if there's no rabbitmq support do nothing queue_json_publish("missedmessage_emails", notice, lambda notice: None) notified['email_notified'] = True if len(notified) > 0: extra_user_data[user_profile_id] = notified for client_data in six.itervalues(send_to_clients): client = client_data['client'] flags = client_data['flags'] is_sender = client_data.get('is_sender', False) # type: bool extra_data = extra_user_data.get(client.user_profile_id, None) # type: Optional[Mapping[str, bool]] if not client.accepts_messages(): # The actual check is the accepts_event() check below; # this line is just an optimization to avoid copying # message data unnecessarily continue if client.apply_markdown: message_dict = message_dict_markdown else: message_dict = message_dict_no_markdown # Make sure Zephyr mirroring bots know whether stream is invite-only if "mirror" in client.client_type_name and event_template.get("invite_only"): message_dict = message_dict.copy() message_dict["invite_only_stream"] = True if flags is not None: message_dict['is_mentioned'] = 'mentioned' in flags user_event = dict(type='message', message=message_dict, flags=flags) # type: Dict[str, Any] if extra_data is not None: user_event.update(extra_data) if is_sender: local_message_id = event_template.get('local_id', None) if local_message_id is not None: user_event["local_message_id"] = local_message_id if not client.accepts_event(user_event): continue # The below prevents (Zephyr) mirroring loops. if ('mirror' in sending_client and sending_client.lower() == client.client_type_name.lower()): continue client.add_event(user_event)
def queue_digest_recipient(user_profile, cutoff): # Convert cutoff to epoch seconds for transit. event = {"user_profile_id": user_profile.id, "cutoff": cutoff.strftime('%s')} queue_json_publish("digest_emails", event, lambda event: None)
def queue_digest_recipient(user_profile: UserProfile, cutoff: datetime.datetime) -> None: # Convert cutoff to epoch seconds for transit. event = {"user_profile_id": user_profile.id, "cutoff": cutoff.strftime('%s')} queue_json_publish("digest_emails", event)
def report_error( request: HttpRequest, user_profile: UserProfile, message: str = REQ(), stacktrace: str = REQ(), ui_message: bool = REQ(validator=check_bool), user_agent: str = REQ(), href: str = REQ(), log: str = REQ(), more_info: Optional[Dict[str, Any]] = REQ(validator=check_dict([]), default=None) ) -> HttpResponse: """Accepts an error report and stores in a queue for processing. The actual error reports are later handled by do_report_error (below)""" if not settings.BROWSER_ERROR_REPORTING: return json_success() if more_info is None: more_info = {} js_source_map = get_js_source_map() if js_source_map: stacktrace = js_source_map.annotate_stacktrace(stacktrace) try: version = subprocess.check_output( ["git", "log", "HEAD^..HEAD", "--oneline"], universal_newlines=True) # type: Optional[str] except Exception: version = None # Get the IP address of the request remote_ip = request.META.get('HTTP_X_REAL_IP') if remote_ip is None: remote_ip = request.META['REMOTE_ADDR'] # For the privacy of our users, we remove any actual text content # in draft_content (from drafts rendering exceptions). See the # comment on privacy_clean_markdown for more details. if more_info.get('draft_content'): more_info['draft_content'] = privacy_clean_markdown( more_info['draft_content']) queue_json_publish( 'error_reports', dict(type="browser", report=dict( host=request.get_host().split(":")[0], ip_address=remote_ip, user_email=user_profile.email, user_full_name=user_profile.full_name, user_visible=ui_message, server_path=settings.DEPLOY_ROOT, version=version, user_agent=user_agent, href=href, message=message, stacktrace=stacktrace, log=log, more_info=more_info, ))) return json_success()