from django.db.models import Q, QuerySet
from django.template import loader
from django.conf import settings
from django.utils.timezone import now as timezone_now
from zerver.lib.notifications import build_message_list, hash_util_encode, \
one_click_unsubscribe_link
from zerver.lib.send_email import send_future_email, FromAddress
from zerver.models import UserProfile, UserMessage, Recipient, Stream, \
Subscription, UserActivity, get_active_streams, get_user_profile_by_id, \
Realm
from zerver.context_processors import common_context
from zerver.lib.queue import queue_json_publish
from zerver.lib.logging_util import create_logger
logger = create_logger(__name__, settings.DIGEST_LOG_PATH, 'DEBUG')
VALID_DIGEST_DAY = 1 # Tuesdays
DIGEST_CUTOFF = 5
# Digests accumulate 4 types of interesting traffic for a user:
# 1. Missed PMs
# 2. New streams
# 3. New users
# 4. Interesting stream traffic, as determined by the longest and most
# diversely comment upon topics.
def inactive_since(user_profile: UserProfile, cutoff: datetime.datetime) -> bool:
# Hasn't used the app in the last DIGEST_CUTOFF (5) days.
most_recent_visit = [row.last_visit for row in
UserActivity.objects.filter(
from typing import Any
from django.core.management.base import BaseCommand
from django.db.utils import IntegrityError
from django.conf import settings
from zproject.backends import ZulipLDAPUserPopulator
from zerver.models import UserProfile
from zerver.lib.logging_util import create_logger
## Setup ##
logger = create_logger(__name__, settings.LDAP_SYNC_LOG_PATH, 'INFO')
# Run this on a cronjob to pick up on name changes.
def sync_ldap_user_data():
# type: () -> None
logger.info("Starting update.")
backend = ZulipLDAPUserPopulator()
for u in UserProfile.objects.select_related().filter(is_active=True,
is_bot=False).all():
# This will save the user if relevant, and will do nothing if the user
# does not exist.
try:
if backend.populate_user(backend.django_to_ldap_username(
u.email)) is not None:
logger.info("Updated %s." % (u.email, ))
else:
logger.warning("Did not find %s in LDAP." % (u.email, ))
except IntegrityError:
logger.warning("User populated did not match an existing user.")
# This is a hack to ensure that RemoteZulipServer always exists even
# if Zilencer isn't enabled.
if settings.ZILENCER_ENABLED:
from zilencer.models import get_remote_server_by_uuid, RemoteZulipServer
else:
from mock import Mock
get_remote_server_by_uuid = Mock()
RemoteZulipServer = Mock(
) # type: ignore # https://github.com/JukkaL/mypy/issues/1188
FuncT = TypeVar('FuncT', bound=Callable[..., Any])
ViewFuncT = TypeVar('ViewFuncT', bound=Callable[..., HttpResponse])
## logger setup
webhook_logger = create_logger("zulip.zerver.webhooks",
settings.API_KEY_ONLY_WEBHOOK_LOG_PATH, 'DEBUG')
class _RespondAsynchronously(object):
pass
# Return RespondAsynchronously from an @asynchronous view if the
# response will be provided later by calling handler.zulip_finish(),
# or has already been provided this way. We use this for longpolling
# mode.
RespondAsynchronously = _RespondAsynchronously()
def asynchronous(method):
# type: (Callable[..., Union[HttpResponse, _RespondAsynchronously]]) -> Callable[..., Union[HttpResponse, _RespondAsynchronously]]
from django.template.exceptions import TemplateDoesNotExist
from zerver.models import UserProfile, ScheduledEmail, get_user_profile_by_id, \
EMAIL_TYPES
import datetime
from email.utils import parseaddr, formataddr
import ujson
import os
from typing import Any, Dict, Iterable, List, Mapping, Optional, Text
from zerver.lib.logging_util import create_logger
## Logging setup ##
logger = create_logger('zulip.send_email', settings.EMAIL_LOG_PATH, 'INFO')
class FromAddress(object):
SUPPORT = parseaddr(settings.ZULIP_ADMINISTRATOR)[1]
NOREPLY = parseaddr(settings.NOREPLY_EMAIL_ADDRESS)[1]
def log_email(email, template_prefix):
# type: (EmailMultiAlternatives, str) -> None
"""Used in development to record sent emails in a nice HTML log"""
html_message = 'Missing HTML message'
if len(email.alternatives) > 0:
html_message = email.alternatives[0][0]
context = {
from django.conf import settings
from django.core.management.base import BaseCommand
from django.utils.timezone import now as timezone_now
from zerver.models import ScheduledEmail
from zerver.lib.context_managers import lockfile
from zerver.lib.send_email import send_email, EmailNotDeliveredException
import time
from zerver.lib.logging_util import create_logger
from ujson import loads
from typing import Any
## Setup ##
logger = create_logger(__name__, settings.EMAIL_DELIVERER_LOG_PATH, 'DEBUG')
class Command(BaseCommand):
help = """Deliver emails queued by various parts of Zulip
(either for immediate sending or sending at a specified time).
Run this command under supervisor. This is for SMTP email delivery.
Usage: ./manage.py deliver_email
"""
def handle(self, *args: Any, **options: Any) -> None:
if settings.EMAIL_DELIVERER_DISABLED:
while True:
last_successful_fill
from zerver.models import Realm, UserProfile, Message, Stream, \
UserActivityInterval, RealmAuditLog, models
from zerver.lib.timestamp import floor_to_day, floor_to_hour, ceiling_to_day, \
ceiling_to_hour, verify_UTC
from typing import Any, Callable, Dict, List, Optional, Text, Tuple, Type, Union
from collections import defaultdict, OrderedDict
from datetime import timedelta, datetime
from zerver.lib.logging_util import create_logger
import time
## Logging setup ##
logger = create_logger('zulip.management', settings.ANALYTICS_LOG_PATH, 'INFO')
# You can't subtract timedelta.max from a datetime, so use this instead
TIMEDELTA_MAX = timedelta(days=365*1000)
## Class definitions ##
class CountStat(object):
HOUR = 'hour'
DAY = 'day'
FREQUENCIES = frozenset([HOUR, DAY])
def __init__(self, property, data_collector, frequency, interval=None):
# type: (str, DataCollector, str, Optional[timedelta]) -> None
self.property = property
self.data_collector = data_collector
from django.template.exceptions import TemplateDoesNotExist
from zerver.models import UserProfile, ScheduledEmail, get_user_profile_by_id, \
EMAIL_TYPES
import datetime
from email.utils import parseaddr, formataddr
import ujson
import os
from typing import Any, Dict, Iterable, List, Mapping, Optional, Text
from zerver.lib.logging_util import create_logger
## Logging setup ##
logger = create_logger('zulip.send_email', settings.EMAIL_LOG_PATH, 'INFO')
class FromAddress(object):
SUPPORT = parseaddr(settings.ZULIP_ADMINISTRATOR)[1]
NOREPLY = parseaddr(settings.NOREPLY_EMAIL_ADDRESS)[1]
def build_email(template_prefix, to_user_id=None, to_email=None, from_name=None,
from_address=None, reply_to_email=None, context=None):
# type: (str, Optional[int], Optional[Text], Optional[Text], Optional[Text], Optional[Text], Optional[Dict[str, Any]]) -> EmailMultiAlternatives
# Callers should pass exactly one of to_user_id and to_email.
assert (to_user_id is None) ^ (to_email is None)
if to_user_id is not None:
to_user = get_user_profile_by_id(to_user_id)
# Change to formataddr((to_user.full_name, to_user.email)) once
# https://github.com/zulip/zulip/issues/4676 is resolved
to_email = to_user.email
from django.conf import settings
from django.core.management.base import BaseCommand
from django.utils.timezone import now as timezone_now
from zerver.models import ScheduledEmail
from zerver.lib.context_managers import lockfile
from zerver.lib.send_email import send_email, EmailNotDeliveredException
import time
from zerver.lib.logging_util import create_logger
from datetime import datetime
from ujson import loads
from typing import Any
## Setup ##
logger = create_logger(__name__, settings.EMAIL_DELIVERER_LOG_PATH, 'DEBUG')
class Command(BaseCommand):
help = """Deliver emails queued by various parts of Zulip
(either for immediate sending or sending at a specified time).
Run this command under supervisor. This is for SMTP email delivery.
Usage: ./manage.py deliver_email
"""
def handle(self, *args: Any, **options: Any) -> None:
if settings.EMAIL_DELIVERER_DISABLED:
while True:
time.sleep(10*9)
from django.db.models import Q, QuerySet
from django.template import loader
from django.conf import settings
from django.utils.timezone import now as timezone_now
from zerver.lib.notifications import build_message_list, hash_util_encode, \
one_click_unsubscribe_link
from zerver.lib.send_email import send_future_email, FromAddress
from zerver.models import UserProfile, UserMessage, Recipient, Stream, \
Subscription, UserActivity, get_active_streams, get_user_profile_by_id, \
Realm
from zerver.context_processors import common_context
from zerver.lib.queue import queue_json_publish
from zerver.lib.logging_util import create_logger
logger = create_logger(__name__, settings.DIGEST_LOG_PATH, 'DEBUG')
VALID_DIGEST_DAY = 1 # Tuesdays
DIGEST_CUTOFF = 5
# Digests accumulate 4 types of interesting traffic for a user:
# 1. Missed PMs
# 2. New streams
# 3. New users
# 4. Interesting stream traffic, as determined by the longest and most
# diversely comment upon topics.
def inactive_since(user_profile, cutoff):
# type: (UserProfile, datetime.datetime) -> bool
# Hasn't used the app in the last DIGEST_CUTOFF (5) days.
most_recent_visit = [row.last_visit for row in
from zerver.lib.logging_util import create_logger
# This is a hack to ensure that RemoteZulipServer always exists even
# if Zilencer isn't enabled.
if settings.ZILENCER_ENABLED:
from zilencer.models import get_remote_server_by_uuid, RemoteZulipServer
else:
from mock import Mock
get_remote_server_by_uuid = Mock()
RemoteZulipServer = Mock() # type: ignore # https://github.com/JukkaL/mypy/issues/1188
ViewFuncT = TypeVar('ViewFuncT', bound=Callable[..., HttpResponse])
ReturnT = TypeVar('ReturnT')
## logger setup
webhook_logger = create_logger(
"zulip.zerver.webhooks", settings.API_KEY_ONLY_WEBHOOK_LOG_PATH, 'DEBUG')
class _RespondAsynchronously(object):
pass
# Return RespondAsynchronously from an @asynchronous view if the
# response will be provided later by calling handler.zulip_finish(),
# or has already been provided this way. We use this for longpolling
# mode.
RespondAsynchronously = _RespondAsynchronously()
def asynchronous(method):
# type: (Callable[..., Union[HttpResponse, _RespondAsynchronously]]) -> Callable[..., Union[HttpResponse, _RespondAsynchronously]]
# TODO: this should be the correct annotation when mypy gets fixed: type:
# (Callable[[HttpRequest, base.BaseHandler, Sequence[Any], Dict[str, Any]],
# Union[HttpResponse, _RespondAsynchronously]]) ->
from zerver.lib.logging_util import create_logger
from collections import defaultdict
from django.db import transaction
from django.db.models import Max
from django.conf import settings
from django.utils.timezone import now as timezone_now
from typing import DefaultDict, List, Union, Any
from zerver.models import UserProfile, UserMessage, RealmAuditLog, \
Subscription, Message, Recipient, UserActivity, Realm
logger = create_logger("zulip.soft_deactivation",
settings.SOFT_DEACTIVATION_LOG_PATH, 'INFO')
def filter_by_subscription_history(user_profile, all_stream_messages,
all_stream_subscription_logs):
# type: (UserProfile, DefaultDict[int, List[Message]], DefaultDict[int, List[RealmAuditLog]]) -> List[UserMessage]
user_messages_to_insert = [] # type: List[UserMessage]
def store_user_message_to_insert(message):
# type: (Message) -> None
message = UserMessage(user_profile=user_profile,
message_id=message['id'],
flags=0)
user_messages_to_insert.append(message)
for (stream_id, stream_messages) in all_stream_messages.items():
stream_subscription_logs = all_stream_subscription_logs[stream_id]
for log_entry in stream_subscription_logs:
from typing import Any
from django.core.management.base import BaseCommand
from django.db.utils import IntegrityError
from django.conf import settings
from zproject.backends import ZulipLDAPUserPopulator
from zerver.models import UserProfile
from zerver.lib.logging_util import create_logger
## Setup ##
logger = create_logger(__name__, settings.LDAP_SYNC_LOG_PATH, 'INFO')
# Run this on a cronjob to pick up on name changes.
def sync_ldap_user_data() -> None:
logger.info("Starting update.")
backend = ZulipLDAPUserPopulator()
for u in UserProfile.objects.select_related().filter(is_active=True, is_bot=False).all():
# This will save the user if relevant, and will do nothing if the user
# does not exist.
try:
if backend.populate_user(backend.django_to_ldap_username(u.email)) is not None:
logger.info("Updated %s." % (u.email,))
else:
logger.warning("Did not find %s in LDAP." % (u.email,))
except IntegrityError:
logger.warning("User populated did not match an existing user.")
logger.info("Finished update.")
class Command(BaseCommand):
