from sentry_sdk import capture_exception
from zerver.lib.logging_util import log_to_file
from zerver.models import (
Message,
Realm,
RealmAuditLog,
Recipient,
Subscription,
UserActivity,
UserMessage,
UserProfile,
)
logger = logging.getLogger("zulip.soft_deactivation")
log_to_file(logger, settings.SOFT_DEACTIVATION_LOG_PATH)
BULK_CREATE_BATCH_SIZE = 10000
def filter_by_subscription_history(
user_profile: UserProfile,
all_stream_messages: DefaultDict[int, List[Message]],
all_stream_subscription_logs: DefaultDict[int, List[RealmAuditLog]],
) -> List[UserMessage]:
user_messages_to_insert: List[UserMessage] = []
def store_user_message_to_insert(message: Message) -> None:
message = UserMessage(user_profile=user_profile,
message_id=message["id"],
flags=0)
user_messages_to_insert.append(message)
from typing import Union, Any, Callable, Sequence, Dict, Optional, TypeVar, Tuple, cast
from zerver.lib.logging_util import log_to_file
# This is a hack to ensure that RemoteZulipServer always exists even
# if Zilencer isn't enabled.
if settings.ZILENCER_ENABLED:
from zilencer.models import get_remote_server_by_uuid, RemoteZulipServer
else: # nocoverage # Hack here basically to make impossible code paths compile
from mock import Mock
get_remote_server_by_uuid = Mock()
RemoteZulipServer = Mock() # type: ignore # https://github.com/JukkaL/mypy/issues/1188
ReturnT = TypeVar('ReturnT')
webhook_logger = logging.getLogger("zulip.zerver.webhooks")
log_to_file(webhook_logger, settings.API_KEY_ONLY_WEBHOOK_LOG_PATH)
class _RespondAsynchronously:
pass
# Return RespondAsynchronously from an @asynchronous view if the
# response will be provided later by calling handler.zulip_finish(),
# or has already been provided this way. We use this for longpolling
# mode.
RespondAsynchronously = _RespondAsynchronously()
AsyncWrapperT = Callable[..., Union[HttpResponse, _RespondAsynchronously]]
def asynchronous(method: Callable[..., Union[HttpResponse, _RespondAsynchronously]]) -> AsyncWrapperT:
# TODO: this should be the correct annotation when mypy gets fixed: type:
# (Callable[[HttpRequest, base.BaseHandler, Sequence[Any], Dict[str, Any]],
# Union[HttpResponse, _RespondAsynchronously]]) ->
from zerver.lib.logging_util import log_to_file
# This is a hack to ensure that RemoteZulipServer always exists even
# if Zilencer isn't enabled.
if settings.ZILENCER_ENABLED:
from zilencer.models import get_remote_server_by_uuid, RemoteZulipServer
else:
from mock import Mock
get_remote_server_by_uuid = Mock()
RemoteZulipServer = Mock(
) # type: ignore # https://github.com/JukkaL/mypy/issues/1188
ReturnT = TypeVar('ReturnT')
webhook_logger = logging.getLogger("zulip.zerver.webhooks")
log_to_file(webhook_logger, settings.API_KEY_ONLY_WEBHOOK_LOG_PATH)
class _RespondAsynchronously:
pass
# Return RespondAsynchronously from an @asynchronous view if the
# response will be provided later by calling handler.zulip_finish(),
# or has already been provided this way. We use this for longpolling
# mode.
RespondAsynchronously = _RespondAsynchronously()
AsyncWrapperT = Callable[..., Union[HttpResponse, _RespondAsynchronously]]
# Documented in https://zulip.readthedocs.io/en/latest/subsystems/sending-messages.html#soft-deactivation
from zerver.lib.logging_util import log_to_file
from collections import defaultdict
import logging
from django.db import transaction
from django.db.models import Max
from django.conf import settings
from django.utils.timezone import now as timezone_now
from typing import DefaultDict, List, Union, Any
from zerver.models import UserProfile, UserMessage, RealmAuditLog, \
Subscription, Message, Recipient, UserActivity
logger = logging.getLogger("zulip.soft_deactivation")
log_to_file(logger, settings.SOFT_DEACTIVATION_LOG_PATH)
def filter_by_subscription_history(user_profile: UserProfile,
all_stream_messages: DefaultDict[int, List[Message]],
all_stream_subscription_logs: DefaultDict[int, List[RealmAuditLog]],
) -> List[UserMessage]:
user_messages_to_insert = [] # type: List[UserMessage]
def store_user_message_to_insert(message: Message) -> None:
message = UserMessage(user_profile=user_profile,
message_id=message['id'], flags=0)
user_messages_to_insert.append(message)
for (stream_id, stream_messages) in all_stream_messages.items():
stream_subscription_logs = all_stream_subscription_logs[stream_id]
from datetime import timedelta
from typing import Any
from django.conf import settings
from django.core.management.base import BaseCommand
from django.db import transaction
from django.utils.timezone import now as timezone_now
from zerver.actions.message_send import build_message_send_dict, do_send_messages
from zerver.lib.logging_util import log_to_file
from zerver.lib.message import SendMessageRequest
from zerver.models import Message, ScheduledMessage, get_user_by_delivery_email
## Setup ##
logger = logging.getLogger(__name__)
log_to_file(logger, settings.DELIVER_SCHEDULED_MESSAGES_LOG_PATH)
class Command(BaseCommand):
help = """Deliver scheduled messages from the ScheduledMessage table.
Run this command under supervisor.
This management command is run via supervisor.
Usage: ./manage.py deliver_scheduled_messages
"""
def construct_message(
self, scheduled_message: ScheduledMessage) -> SendMessageRequest:
message = Message()
original_sender = scheduled_message.sender
import datetime
from email.utils import parseaddr, formataddr
import logging
import ujson
import os
from typing import Any, Dict, Iterable, List, Mapping, Optional
from zerver.lib.logging_util import log_to_file
from confirmation.models import generate_key
## Logging setup ##
logger = logging.getLogger('zulip.send_email')
log_to_file(logger, settings.EMAIL_LOG_PATH)
class FromAddress:
SUPPORT = parseaddr(settings.ZULIP_ADMINISTRATOR)[1]
NOREPLY = parseaddr(settings.NOREPLY_EMAIL_ADDRESS)[1]
# Generates an unpredictable noreply address.
@staticmethod
def tokenized_no_reply_address() -> str:
if settings.ADD_TOKENS_TO_NOREPLY_ADDRESS:
return parseaddr(
settings.TOKENIZED_NOREPLY_EMAIL_ADDRESS)[1].format(
token=generate_key())
return FromAddress.NOREPLY
from zerver.lib.timestamp import datetime_to_timestamp, timestamp_to_datetime
from zerver.lib.utils import generate_random_token
from zerver.models import Realm, UserProfile, RealmAuditLog
from corporate.models import Customer, CustomerPlan, LicenseLedger, \
get_current_plan_by_customer, get_customer_by_realm, \
get_current_plan_by_realm
from zproject.config import get_secret
STRIPE_PUBLISHABLE_KEY = get_secret('stripe_publishable_key')
stripe.api_key = get_secret('stripe_secret_key')
BILLING_LOG_PATH = os.path.join(
'/var/log/zulip' if not settings.DEVELOPMENT else
settings.DEVELOPMENT_LOG_DIRECTORY, 'billing.log')
billing_logger = logging.getLogger('corporate.stripe')
log_to_file(billing_logger, BILLING_LOG_PATH)
log_to_file(logging.getLogger('stripe'), BILLING_LOG_PATH)
CallableT = TypeVar('CallableT', bound=Callable[..., Any])
MIN_INVOICED_LICENSES = 30
MAX_INVOICED_LICENSES = 1000
DEFAULT_INVOICE_DAYS_UNTIL_DUE = 30
def get_latest_seat_count(realm: Realm) -> int:
non_guests = UserProfile.objects.filter(
realm=realm, is_active=True,
is_bot=False).exclude(role=UserProfile.ROLE_GUEST).count()
guests = UserProfile.objects.filter(realm=realm,
is_active=True,
import datetime
from email.utils import parseaddr, formataddr
import logging
import ujson
import os
from typing import Any, Dict, Iterable, List, Mapping, Optional
from zerver.lib.logging_util import log_to_file
from confirmation.models import generate_key
## Logging setup ##
logger = logging.getLogger('zulip.send_email')
log_to_file(logger, settings.EMAIL_LOG_PATH)
class FromAddress:
SUPPORT = parseaddr(settings.ZULIP_ADMINISTRATOR)[1]
NOREPLY = parseaddr(settings.NOREPLY_EMAIL_ADDRESS)[1]
# Generates an unpredictable noreply address.
@staticmethod
def tokenized_no_reply_address() -> str:
if settings.ADD_TOKENS_TO_NOREPLY_ADDRESS:
return parseaddr(settings.TOKENIZED_NOREPLY_EMAIL_ADDRESS)[1].format(token=generate_key())
return FromAddress.NOREPLY
def build_email(template_prefix: str, to_user_id: Optional[int]=None,
to_email: Optional[str]=None, from_name: Optional[str]=None,
from_address: Optional[str]=None, reply_to_email: Optional[str]=None,
from zerver.lib.logging_util import log_to_file
from zerver.lib.timestamp import ceiling_to_day, ceiling_to_hour, floor_to_hour, verify_UTC
from zerver.models import (
Message,
Realm,
RealmAuditLog,
Stream,
UserActivityInterval,
UserProfile,
models,
)
## Logging setup ##
logger = logging.getLogger('zulip.management')
log_to_file(logger, settings.ANALYTICS_LOG_PATH)
# You can't subtract timedelta.max from a datetime, so use this instead
TIMEDELTA_MAX = timedelta(days=365 * 1000)
## Class definitions ##
class CountStat:
HOUR = 'hour'
DAY = 'day'
FREQUENCIES = frozenset([HOUR, DAY])
def __init__(self,
property: str,
data_collector: 'DataCollector',
from typing import Any
from django.conf import settings
from django.core.management.base import BaseCommand
from django.utils.timezone import now as timezone_now
from ujson import loads
from zerver.lib.context_managers import lockfile
from zerver.lib.logging_util import log_to_file
from zerver.lib.management import sleep_forever
from zerver.lib.send_email import EmailNotDeliveredException, send_email
from zerver.models import ScheduledEmail
## Setup ##
logger = logging.getLogger(__name__)
log_to_file(logger, settings.EMAIL_DELIVERER_LOG_PATH)
class Command(BaseCommand):
help = """Deliver emails queued by various parts of Zulip
(either for immediate sending or sending at a specified time).
Run this command under supervisor. This is for SMTP email delivery.
Usage: ./manage.py deliver_email
"""
def handle(self, *args: Any, **options: Any) -> None:
if settings.EMAIL_DELIVERER_DISABLED:
sleep_forever()
from django.utils.timezone import now as timezone_now
from psycopg2.sql import Composable, Identifier, Literal, SQL
from zerver.lib.logging_util import log_to_file
from zerver.models import (Message, UserMessage, ArchivedUserMessage, Realm,
Attachment, ArchivedAttachment, Reaction,
ArchivedReaction, SubMessage, ArchivedSubMessage,
Recipient, Stream, ArchiveTransaction,
get_user_including_cross_realm)
from typing import Any, Dict, List, Optional
import logging
logger = logging.getLogger('zulip.retention')
log_to_file(logger, settings.RETENTION_LOG_PATH)
MESSAGE_BATCH_SIZE = 1000
models_with_message_key: List[Dict[str, Any]] = [
{
'class': Reaction,
'archive_class': ArchivedReaction,
'table_name': 'zerver_reaction',
'archive_table_name': 'zerver_archivedreaction'
},
{
'class': SubMessage,
'archive_class': ArchivedSubMessage,
'table_name': 'zerver_submessage',
'archive_table_name': 'zerver_archivedsubmessage'
import logging
from argparse import ArgumentParser
from typing import Any, List
from django.conf import settings
from zerver.lib.logging_util import log_to_file
from zerver.lib.management import ZulipBaseCommand
from zerver.models import UserProfile
from zproject.backends import ZulipLDAPException, sync_user_from_ldap
## Setup ##
logger = logging.getLogger('zulip.sync_ldap_user_data')
log_to_file(logger, settings.LDAP_SYNC_LOG_PATH)
# Run this on a cronjob to pick up on name changes.
def sync_ldap_user_data(user_profiles: List[UserProfile]) -> None:
logger.info("Starting update.")
for u in user_profiles:
# This will save the user if relevant, and will do nothing if the user
# does not exist.
try:
if sync_user_from_ldap(u):
logger.info("Updated %s." % (u.email,))
else:
logger.warning("Did not find %s in LDAP." % (u.email,))
if settings.LDAP_DEACTIVATE_NON_MATCHING_USERS:
logger.info("Deactivated non-matching user: %s" % (u.email,))
except ZulipLDAPException as e:
from django.conf import settings
from django.core.management.base import BaseCommand
from django.db import transaction
from django.utils.timezone import now as timezone_now
from zerver.lib.context_managers import lockfile
from zerver.lib.logging_util import log_to_file
from zerver.lib.management import sleep_forever
from zerver.models import ScheduledMessage, Message, get_user
from zerver.lib.actions import do_send_messages
from zerver.lib.addressee import Addressee
## Setup ##
logger = logging.getLogger(__name__)
log_to_file(logger, settings.SCHEDULED_MESSAGE_DELIVERER_LOG_PATH)
class Command(BaseCommand):
help = """Deliver scheduled messages from the ScheduledMessage table.
Run this command under supervisor.
This management command is run via supervisor. Do not run on multiple
machines, as you may encounter multiple sends in a specific race
condition. (Alternatively, you can set `EMAIL_DELIVERER_DISABLED=True`
on all but one machine to make the command have no effect.)
Usage: ./manage.py deliver_scheduled_messages
"""
def construct_message(self, scheduled_message: ScheduledMessage) -> Dict[str, Any]:
message = Message()
from django.db import connection
from django.db.models import F
from analytics.models import BaseCount, \
FillState, InstallationCount, RealmCount, StreamCount, \
UserCount, installation_epoch, last_successful_fill
from zerver.lib.logging_util import log_to_file
from zerver.lib.timestamp import ceiling_to_day, \
ceiling_to_hour, floor_to_hour, verify_UTC
from zerver.models import Message, Realm, \
Stream, UserActivityInterval, UserProfile, models
## Logging setup ##
logger = logging.getLogger('zulip.management')
log_to_file(logger, settings.ANALYTICS_LOG_PATH)
# You can't subtract timedelta.max from a datetime, so use this instead
TIMEDELTA_MAX = timedelta(days=365*1000)
## Class definitions ##
class CountStat:
HOUR = 'hour'
DAY = 'day'
FREQUENCIES = frozenset([HOUR, DAY])
def __init__(self, property: str, data_collector: 'DataCollector', frequency: str,
interval: Optional[timedelta]=None) -> None:
self.property = property
self.data_collector = data_collector
from zerver.lib.timestamp import datetime_to_timestamp, timestamp_to_datetime
from zerver.lib.utils import generate_random_token
from zerver.models import Realm, UserProfile, RealmAuditLog
from corporate.models import Customer, CustomerPlan, LicenseLedger, \
get_active_plan
from zproject.settings import get_secret
STRIPE_PUBLISHABLE_KEY = get_secret('stripe_publishable_key')
stripe.api_key = get_secret('stripe_secret_key')
BILLING_LOG_PATH = os.path.join('/var/log/zulip'
if not settings.DEVELOPMENT
else settings.DEVELOPMENT_LOG_DIRECTORY,
'billing.log')
billing_logger = logging.getLogger('corporate.stripe')
log_to_file(billing_logger, BILLING_LOG_PATH)
log_to_file(logging.getLogger('stripe'), BILLING_LOG_PATH)
CallableT = TypeVar('CallableT', bound=Callable[..., Any])
MIN_INVOICED_LICENSES = 30
DEFAULT_INVOICE_DAYS_UNTIL_DUE = 30
def get_seat_count(realm: Realm) -> int:
non_guests = UserProfile.objects.filter(
realm=realm, is_active=True, is_bot=False, is_guest=False).count()
guests = UserProfile.objects.filter(
realm=realm, is_active=True, is_bot=False, is_guest=True).count()
return max(non_guests, math.ceil(guests / 5))
def sign_string(string: str) -> Tuple[str, str]:
from django.template import loader
from django.conf import settings
from django.utils.timezone import now as timezone_now
from zerver.lib.notifications import build_message_list, hash_util_encode, \
one_click_unsubscribe_link
from zerver.lib.send_email import send_future_email, FromAddress
from zerver.models import UserProfile, UserMessage, Recipient, Stream, \
Subscription, UserActivity, get_active_streams, get_user_profile_by_id, \
Realm
from zerver.context_processors import common_context
from zerver.lib.queue import queue_json_publish
from zerver.lib.logging_util import log_to_file
logger = logging.getLogger(__name__)
log_to_file(logger, settings.DIGEST_LOG_PATH)
VALID_DIGEST_DAY = 1 # Tuesdays
DIGEST_CUTOFF = 5
# Digests accumulate 4 types of interesting traffic for a user:
# 1. Missed PMs
# 2. New streams
# 3. New users
# 4. Interesting stream traffic, as determined by the longest and most
# diversely comment upon topics.
def inactive_since(user_profile: UserProfile, cutoff: datetime.datetime) -> bool:
# Hasn't used the app in the last DIGEST_CUTOFF (5) days.
most_recent_visit = [row.last_visit for row in
UserActivity.objects.filter(
from datetime import timedelta
from django.conf import settings
from django.core.management.base import BaseCommand
from django.db import transaction
from django.utils.timezone import now as timezone_now
from zerver.lib.context_managers import lockfile
from zerver.lib.logging_util import log_to_file
from zerver.models import ScheduledMessage, Message
from zerver.lib.actions import do_send_messages
from zerver.lib.addressee import Addressee
## Setup ##
logger = logging.getLogger(__name__)
log_to_file(logger, settings.SCHEDULED_MESSAGE_DELIVERER_LOG_PATH)
class Command(BaseCommand):
help = """Deliver scheduled messages from the ScheduledMessage table.
Run this command under supervisor.
Usage: ./manage.py deliver_scheduled_messages
"""
def construct_message(
self, scheduled_message: ScheduledMessage) -> Dict[str, Any]:
message = Message()
message.sender = scheduled_message.sender
message.content = scheduled_message.content
message.recipient = scheduled_message.recipient
import logging
from argparse import ArgumentParser
from typing import Any, List
from django.conf import settings
from django.db import transaction
from zerver.lib.logging_util import log_to_file
from zerver.lib.management import ZulipBaseCommand
from zerver.models import UserProfile
from zproject.backends import ZulipLDAPException, sync_user_from_ldap
## Setup ##
logger = logging.getLogger('zulip.sync_ldap_user_data')
log_to_file(logger, settings.LDAP_SYNC_LOG_PATH)
# Run this on a cronjob to pick up on name changes.
def sync_ldap_user_data(user_profiles: List[UserProfile],
deactivation_protection: bool = True) -> None:
logger.info("Starting update.")
with transaction.atomic():
realms = {u.realm.string_id for u in user_profiles}
for u in user_profiles:
# This will save the user if relevant, and will do nothing if the user
# does not exist.
try:
sync_user_from_ldap(u, logger)
except ZulipLDAPException as e:
logger.error("Error attempting to update user %s:",
import logging
import time
from typing import Any
from django.conf import settings
from django.core.management.base import BaseCommand
from django.utils.timezone import now as timezone_now
from zerver.lib.logging_util import log_to_file
from zerver.lib.management import sleep_forever
from zerver.lib.send_email import EmailNotDeliveredException, deliver_scheduled_emails
from zerver.models import ScheduledEmail
## Setup ##
logger = logging.getLogger(__name__)
log_to_file(logger, settings.EMAIL_DELIVERER_LOG_PATH)
class Command(BaseCommand):
help = """Send emails queued by various parts of Zulip
for later delivery.
Run this command under supervisor.
Usage: ./manage.py deliver_scheduled_emails
"""
def handle(self, *args: Any, **options: Any) -> None:
if settings.EMAIL_DELIVERER_DISABLED:
sleep_forever()
from zerver.lib.logging_util import log_to_file
# This is a hack to ensure that RemoteZulipServer always exists even
# if Zilencer isn't enabled.
if settings.ZILENCER_ENABLED:
from zilencer.models import get_remote_server_by_uuid, RemoteZulipServer
else: # nocoverage # Hack here basically to make impossible code paths compile
from mock import Mock
get_remote_server_by_uuid = Mock()
RemoteZulipServer = Mock(
) # type: ignore[misc] # https://github.com/JukkaL/mypy/issues/1188
ReturnT = TypeVar('ReturnT')
webhook_logger = logging.getLogger("zulip.zerver.webhooks")
log_to_file(webhook_logger, settings.API_KEY_ONLY_WEBHOOK_LOG_PATH)
webhook_unexpected_events_logger = logging.getLogger(
"zulip.zerver.lib.webhooks.common")
log_to_file(webhook_unexpected_events_logger,
settings.WEBHOOK_UNEXPECTED_EVENTS_LOG_PATH)
def cachify(method: Callable[..., ReturnT]) -> Callable[..., ReturnT]:
dct: Dict[Tuple[Any, ...], ReturnT] = {}
def cache_wrapper(*args: Any) -> ReturnT:
tup = tuple(args)
if tup in dct:
return dct[tup]
result = method(*args)
from django.conf import settings
from django.utils.timezone import now as timezone_now
from confirmation.models import one_click_unsubscribe_link
from zerver.lib.email_notifications import build_message_list
from zerver.lib.send_email import send_future_email, FromAddress
from zerver.lib.url_encoding import encode_stream
from zerver.models import UserProfile, Recipient, Subscription, UserActivity, \
get_active_streams, get_user_profile_by_id, Realm, Message, RealmAuditLog
from zerver.context_processors import common_context
from zerver.lib.queue import queue_json_publish
from zerver.lib.logging_util import log_to_file
logger = logging.getLogger(__name__)
log_to_file(logger, settings.DIGEST_LOG_PATH)
DIGEST_CUTOFF = 5
# Digests accumulate 2 types of interesting traffic for a user:
# 1. New streams
# 2. Interesting stream traffic, as determined by the longest and most
# diversely comment upon topics.
def inactive_since(user_profile: UserProfile,
cutoff: datetime.datetime) -> bool:
# Hasn't used the app in the last DIGEST_CUTOFF (5) days.
most_recent_visit = [
row.last_visit
for row in UserActivity.objects.filter(user_profile=user_profile)
from zerver.lib.queue import queue_json_publish
from zerver.lib.rate_limiter import RateLimitedUser
from zerver.lib.request import REQ, has_request_variables
from zerver.lib.response import json_error, json_method_not_allowed, json_success, json_unauthorized
from zerver.lib.subdomains import get_subdomain, user_matches_subdomain
from zerver.lib.timestamp import datetime_to_timestamp, timestamp_to_datetime
from zerver.lib.types import ViewFuncT
from zerver.lib.user_agent import parse_user_agent
from zerver.lib.utils import has_api_key_format, statsd
from zerver.models import Realm, UserProfile, get_client, get_user_profile_by_api_key
if settings.ZILENCER_ENABLED:
from zilencer.models import RemoteZulipServer, get_remote_server_by_uuid
webhook_logger = logging.getLogger("zulip.zerver.webhooks")
log_to_file(webhook_logger, settings.WEBHOOK_LOG_PATH)
webhook_unsupported_events_logger = logging.getLogger(
"zulip.zerver.webhooks.unsupported")
log_to_file(webhook_unsupported_events_logger,
settings.WEBHOOK_UNSUPPORTED_EVENTS_LOG_PATH)
FuncT = TypeVar('FuncT', bound=Callable[..., object])
def cachify(method: FuncT) -> FuncT:
dct: Dict[Tuple[object, ...], object] = {}
def cache_wrapper(*args: object) -> object:
tup = tuple(args)
if tup in dct: