def by_group_pm_with(self, query: Query, operand: Union[str, int],
maybe_negate: ConditionTransform) -> Query:
try:
if isinstance(operand, str):
narrow_profile = get_user_including_cross_realm(operand, self.user_realm)
else:
narrow_profile = get_user_by_id_in_realm_including_cross_realm(operand, self.user_realm)
except UserProfile.DoesNotExist:
raise BadNarrowOperator('unknown user ' + str(operand))
self_recipient_ids = [
recipient_tuple['recipient_id'] for recipient_tuple
in Subscription.objects.filter(
user_profile=self.user_profile,
recipient__type=Recipient.HUDDLE,
).values("recipient_id")]
narrow_recipient_ids = [
recipient_tuple['recipient_id'] for recipient_tuple
in Subscription.objects.filter(
user_profile=narrow_profile,
recipient__type=Recipient.HUDDLE,
).values("recipient_id")]
recipient_ids = set(self_recipient_ids) & set(narrow_recipient_ids)
cond = column("recipient_id").in_(recipient_ids)
return query.where(maybe_negate(cond))
def avatar(request: HttpRequest,
user_profile: UserProfile,
email_or_id: str,
medium: bool = False) -> HttpResponse:
"""Accepts an email address or user ID and returns the avatar"""
is_email = False
try:
int(email_or_id)
except ValueError:
is_email = True
try:
realm = user_profile.realm
if is_email:
avatar_user_profile = get_user_including_cross_realm(
email_or_id, realm)
else:
avatar_user_profile = get_user_by_id_in_realm_including_cross_realm(
int(email_or_id), realm)
# If there is a valid user account passed in, use its avatar
url = avatar_url(avatar_user_profile, medium=medium)
except UserProfile.DoesNotExist:
# If there is no such user, treat it as a new gravatar
email = email_or_id
avatar_version = 1
url = get_gravatar_url(email, avatar_version, medium)
# We can rely on the url already having query parameters. Because
# our templates depend on being able to use the ampersand to
# add query parameters to our url, get_avatar_url does '?x=x'
# hacks to prevent us from having to jump through decode/encode hoops.
assert url is not None
assert '?' in url
url += '&' + request.META['QUERY_STRING']
return redirect(url)
def avatar(request: HttpRequest, email_or_id: str, medium: bool=False) -> HttpResponse:
"""Accepts an email address or user ID and returns the avatar"""
is_email = False
try:
int(email_or_id)
except ValueError:
is_email = True
try:
if is_email:
realm = request.user.realm
user_profile = get_user_including_cross_realm(email_or_id, realm)
else:
user_profile = get_user_profile_by_id(email_or_id)
# If there is a valid user account passed in, use its avatar
url = avatar_url(user_profile, medium=medium)
except UserProfile.DoesNotExist:
# If there is no such user, treat it as a new gravatar
email = email_or_id
avatar_version = 1
url = get_gravatar_url(email, avatar_version, medium)
# We can rely on the url already having query parameters. Because
# our templates depend on being able to use the ampersand to
# add query parameters to our url, get_avatar_url does '?x=x'
# hacks to prevent us from having to jump through decode/encode hoops.
assert '?' in url
url += '&' + request.META['QUERY_STRING']
return redirect(url)
def move_expired_personal_and_huddle_messages_to_archive(realm: Realm,
chunk_size: int=MESSAGE_BATCH_SIZE
) -> int:
# This function will archive appropriate messages and their related objects.
cross_realm_bot_ids_list = [get_user_including_cross_realm(email).id
for email in settings.CROSS_REALM_BOT_EMAILS]
cross_realm_bot_ids = str(tuple(cross_realm_bot_ids_list))
recipient_types = (Recipient.PERSONAL, Recipient.HUDDLE)
# Archive expired personal and huddle Messages in the realm, except cross-realm messages:
# TODO: Remove the "zerver_userprofile.id NOT IN {cross_realm_bot_ids}" clause
# once https://github.com/zulip/zulip/issues/11015 is solved.
query = """
INSERT INTO zerver_archivedmessage ({dst_fields}, archive_transaction_id)
SELECT {src_fields}, {archive_transaction_id}
FROM zerver_message
INNER JOIN zerver_recipient ON zerver_recipient.id = zerver_message.recipient_id
INNER JOIN zerver_userprofile ON zerver_userprofile.id = zerver_message.sender_id
WHERE zerver_userprofile.id NOT IN {cross_realm_bot_ids}
AND zerver_userprofile.realm_id = {realm_id}
AND zerver_recipient.type in {recipient_types}
AND zerver_message.date_sent < '{check_date}'
LIMIT {chunk_size}
ON CONFLICT (id) DO UPDATE SET archive_transaction_id = {archive_transaction_id}
RETURNING id
"""
assert realm.message_retention_days is not None
check_date = timezone_now() - timedelta(days=realm.message_retention_days)
message_count = run_archiving_in_chunks(query, type=ArchiveTransaction.RETENTION_POLICY_BASED,
realm=realm,
cross_realm_bot_ids=cross_realm_bot_ids,
realm_id=realm.id, recipient_types=recipient_types,
check_date=check_date.isoformat(), chunk_size=chunk_size)
# Archive cross-realm personal messages to users in the realm:
# Note: Cross-realm huddle message aren't handled yet, they remain an issue
# that should be addressed.
query = """
INSERT INTO zerver_archivedmessage ({dst_fields}, archive_transaction_id)
SELECT {src_fields}, {archive_transaction_id}
FROM zerver_message
INNER JOIN zerver_recipient ON zerver_recipient.id = zerver_message.recipient_id
INNER JOIN zerver_userprofile recipient_profile ON recipient_profile.id = zerver_recipient.type_id
INNER JOIN zerver_userprofile sender_profile ON sender_profile.id = zerver_message.sender_id
WHERE sender_profile.id IN {cross_realm_bot_ids}
AND recipient_profile.realm_id = {realm_id}
AND zerver_recipient.type = {recipient_personal}
AND zerver_message.date_sent < '{check_date}'
LIMIT {chunk_size}
ON CONFLICT (id) DO UPDATE SET archive_transaction_id = {archive_transaction_id}
RETURNING id
"""
message_count += run_archiving_in_chunks(query, type=ArchiveTransaction.RETENTION_POLICY_BASED,
realm=realm,
cross_realm_bot_ids=cross_realm_bot_ids,
realm_id=realm.id, recipient_personal=Recipient.PERSONAL,
check_date=check_date.isoformat(), chunk_size=chunk_size)
return message_count
def move_expired_personal_and_huddle_messages_to_archive(realm: Realm,
chunk_size: int=MESSAGE_BATCH_SIZE
) -> Iterator[List[int]]:
# Important: This function is a generator, and you need to iterate
# through the Iterator it returns to execute the queries.
cross_realm_bot_ids_list = [get_user_including_cross_realm(email).id
for email in settings.CROSS_REALM_BOT_EMAILS]
cross_realm_bot_ids = str(tuple(cross_realm_bot_ids_list))
recipient_types = (Recipient.PERSONAL, Recipient.HUDDLE)
# Archive expired personal and huddle Messages in the realm, except cross-realm messages:
# TODO: Remove the "zerver_userprofile.id NOT IN {cross_realm_bot_ids}" clause
# once https://github.com/zulip/zulip/issues/11015 is solved.
query = """
INSERT INTO zerver_archivedmessage ({dst_fields}, archive_timestamp)
SELECT {src_fields}, '{archive_timestamp}'
FROM zerver_message
INNER JOIN zerver_recipient ON zerver_recipient.id = zerver_message.recipient_id
INNER JOIN zerver_userprofile ON zerver_userprofile.id = zerver_message.sender_id
LEFT JOIN zerver_archivedmessage ON zerver_archivedmessage.id = zerver_message.id
WHERE zerver_userprofile.id NOT IN {cross_realm_bot_ids}
AND zerver_userprofile.realm_id = {realm_id}
AND zerver_recipient.type in {recipient_types}
AND zerver_message.pub_date < '{check_date}'
AND zerver_archivedmessage.id is NULL
LIMIT {chunk_size}
RETURNING id
"""
assert realm.message_retention_days is not None
check_date = timezone_now() - timedelta(days=realm.message_retention_days)
yield from run_message_batch_query(query, returning_id=True, cross_realm_bot_ids=cross_realm_bot_ids,
realm_id=realm.id, recipient_types=recipient_types,
check_date=check_date.isoformat(), chunk_size=chunk_size)
def move_expired_personal_and_huddle_messages_to_archive(realm: Realm) -> List[int]:
cross_realm_bot_ids_list = [get_user_including_cross_realm(email).id
for email in settings.CROSS_REALM_BOT_EMAILS]
cross_realm_bot_ids = str(tuple(cross_realm_bot_ids_list))
recipient_types = (Recipient.PERSONAL, Recipient.HUDDLE)
# Archive expired personal and huddle Messages in the realm, except cross-realm messages:
# TODO: Remove the "zerver_userprofile.id NOT IN {cross_realm_bot_ids}" clause
# once https://github.com/zulip/zulip/issues/11015 is solved.
query = """
INSERT INTO zerver_archivedmessage ({dst_fields}, archive_timestamp)
SELECT {src_fields}, '{archive_timestamp}'
FROM zerver_message
INNER JOIN zerver_recipient ON zerver_recipient.id = zerver_message.recipient_id
INNER JOIN zerver_userprofile ON zerver_userprofile.id = zerver_message.sender_id
LEFT JOIN zerver_archivedmessage ON zerver_archivedmessage.id = zerver_message.id
WHERE zerver_userprofile.id NOT IN {cross_realm_bot_ids}
AND zerver_userprofile.realm_id = {realm_id}
AND zerver_recipient.type in {recipient_types}
AND zerver_message.pub_date < '{check_date}'
AND zerver_archivedmessage.id is NULL
RETURNING id
"""
assert realm.message_retention_days is not None
check_date = timezone_now() - timedelta(days=realm.message_retention_days)
return move_rows(Message, query, returning_id=True, cross_realm_bot_ids=cross_realm_bot_ids,
realm_id=realm.id, recipient_types=recipient_types, check_date=check_date.isoformat())
def create_mirrored_message_users(request: HttpRequest, user_profile: UserProfile,
recipients: Iterable[str]) -> UserProfile:
if "sender" not in request.POST:
raise InvalidMirrorInput("No sender")
sender_email = request.POST["sender"].strip().lower()
referenced_users = {sender_email}
if request.POST['type'] == 'private':
for email in recipients:
referenced_users.add(email.lower())
if request.client.name == "zephyr_mirror":
user_check = same_realm_zephyr_user
fullname_function = compute_mit_user_fullname
elif request.client.name == "irc_mirror":
user_check = same_realm_irc_user
fullname_function = compute_irc_user_fullname
elif request.client.name in ("jabber_mirror", "JabberMirror"):
user_check = same_realm_jabber_user
fullname_function = compute_jabber_user_fullname
else:
raise InvalidMirrorInput("Unrecognized mirroring client")
for email in referenced_users:
# Check that all referenced users are in our realm:
if not user_check(user_profile, email):
raise InvalidMirrorInput("At least one user cannot be mirrored")
# Create users for the referenced users, if needed.
for email in referenced_users:
create_mirror_user_if_needed(user_profile.realm, email, fullname_function)
sender = get_user_including_cross_realm(sender_email, user_profile.realm)
return sender
def user_profiles_from_unvalidated_emails(emails: Iterable[str], realm: Realm) -> List[UserProfile]:
user_profiles = [] # type: List[UserProfile]
for email in emails:
try:
user_profile = get_user_including_cross_realm(email, realm)
except UserProfile.DoesNotExist:
raise ValidationError(_("Invalid email '%s'") % (email,))
user_profiles.append(user_profile)
return user_profiles
def get_user_profiles(emails: Iterable[str], realm: Realm) -> List[UserProfile]:
user_profiles = [] # type: List[UserProfile]
for email in emails:
try:
user_profile = get_user_including_cross_realm(email, realm)
except UserProfile.DoesNotExist:
raise JsonableError(_("Invalid email '%s'") % (email,))
user_profiles.append(user_profile)
return user_profiles
def user_profiles_from_unvalidated_emails(emails: Iterable[Text], realm: Realm) -> List[UserProfile]:
user_profiles = [] # type: List[UserProfile]
for email in emails:
try:
user_profile = get_user_including_cross_realm(email, realm)
except UserProfile.DoesNotExist:
raise ValidationError(_("Invalid email '%s'") % (email,))
user_profiles.append(user_profile)
return user_profiles
def by_sender(self, query: Query, operand: Union[str, int], maybe_negate: ConditionTransform) -> Query:
try:
if isinstance(operand, str):
sender = get_user_including_cross_realm(operand, self.user_realm)
else:
sender = get_user_by_id_in_realm_including_cross_realm(operand, self.user_realm)
except UserProfile.DoesNotExist:
raise BadNarrowOperator('unknown user ' + str(operand))
cond = column("sender_id") == literal(sender.id)
return query.where(maybe_negate(cond))
def avatar(
request: HttpRequest,
maybe_user_profile: Union[UserProfile, AnonymousUser],
email_or_id: str,
medium: bool = False,
) -> HttpResponse:
"""Accepts an email address or user ID and returns the avatar"""
is_email = False
try:
int(email_or_id)
except ValueError:
is_email = True
if not maybe_user_profile.is_authenticated:
# Allow anonynous access to avatars only if spectators are
# enabled in the organization.
realm = get_valid_realm_from_request(request)
# TODO: Replace with realm.allow_web_public_streams_access()
# when the method is available.
if not realm.has_web_public_streams():
raise MissingAuthenticationError()
# We only allow the ID format for accessing a user's avatar
# for spectators. This is mainly for defense in depth, since
# email_address_visibility should mean spectators only
# interact with fake email addresses anyway.
if is_email:
raise MissingAuthenticationError()
else:
realm = maybe_user_profile.realm
try:
if is_email:
avatar_user_profile = get_user_including_cross_realm(
email_or_id, realm)
else:
avatar_user_profile = get_user_by_id_in_realm_including_cross_realm(
int(email_or_id), realm)
# If there is a valid user account passed in, use its avatar
url = avatar_url(avatar_user_profile, medium=medium)
except UserProfile.DoesNotExist:
# If there is no such user, treat it as a new gravatar
email = email_or_id
avatar_version = 1
url = get_gravatar_url(email, avatar_version, medium)
# We can rely on the URL already having query parameters. Because
# our templates depend on being able to use the ampersand to
# add query parameters to our url, get_avatar_url does '?x=x'
# hacks to prevent us from having to jump through decode/encode hoops.
assert url is not None
url = append_url_query_string(url, request.META["QUERY_STRING"])
return redirect(url)
def move_expired_personal_and_huddle_messages_to_archive(
realm: Realm, chunk_size: int = MESSAGE_BATCH_SIZE) -> int:
# This function will archive appropriate messages and their related objects.
cross_realm_bot_ids = [
get_user_including_cross_realm(email).id
for email in settings.CROSS_REALM_BOT_EMAILS
]
recipient_types = (Recipient.PERSONAL, Recipient.HUDDLE)
# Archive expired personal and huddle Messages in the realm, except cross-realm messages.
# The condition zerver_userprofile.realm_id = {realm_id} assures the row won't be
# a message sent by a cross-realm bot, because cross-realm bots have their own separate realm.
query = SQL("""
INSERT INTO zerver_archivedmessage ({dst_fields}, archive_transaction_id)
SELECT {src_fields}, {archive_transaction_id}
FROM zerver_message
INNER JOIN zerver_recipient ON zerver_recipient.id = zerver_message.recipient_id
INNER JOIN zerver_userprofile ON zerver_userprofile.id = zerver_message.sender_id
WHERE zerver_userprofile.realm_id = {realm_id}
AND zerver_recipient.type in {recipient_types}
AND zerver_message.date_sent < {check_date}
LIMIT {chunk_size}
ON CONFLICT (id) DO UPDATE SET archive_transaction_id = {archive_transaction_id}
RETURNING id
""")
assert realm.message_retention_days is not None
check_date = timezone_now() - timedelta(days=realm.message_retention_days)
message_count = run_archiving_in_chunks(
query,
type=ArchiveTransaction.RETENTION_POLICY_BASED,
realm=realm,
cross_realm_bot_ids=Literal(tuple(cross_realm_bot_ids)),
realm_id=Literal(realm.id),
recipient_types=Literal(recipient_types),
check_date=Literal(check_date.isoformat()),
chunk_size=chunk_size,
)
# Archive cross-realm personal messages to users in the realm:
# Note: Cross-realm huddle message aren't handled yet, they remain an issue
# that should be addressed.
query = SQL("""
INSERT INTO zerver_archivedmessage ({dst_fields}, archive_transaction_id)
SELECT {src_fields}, {archive_transaction_id}
FROM zerver_message
INNER JOIN zerver_recipient ON zerver_recipient.id = zerver_message.recipient_id
INNER JOIN zerver_userprofile recipient_profile ON recipient_profile.id = zerver_recipient.type_id
INNER JOIN zerver_userprofile sender_profile ON sender_profile.id = zerver_message.sender_id
WHERE sender_profile.id IN {cross_realm_bot_ids}
AND recipient_profile.realm_id = {realm_id}
AND zerver_recipient.type = {recipient_personal}
AND zerver_message.date_sent < {check_date}
LIMIT {chunk_size}
ON CONFLICT (id) DO UPDATE SET archive_transaction_id = {archive_transaction_id}
RETURNING id
""")
message_count += run_archiving_in_chunks(
query,
type=ArchiveTransaction.RETENTION_POLICY_BASED,
realm=realm,
cross_realm_bot_ids=Literal(tuple(cross_realm_bot_ids)),
realm_id=Literal(realm.id),
recipient_personal=Literal(Recipient.PERSONAL),
check_date=Literal(check_date.isoformat()),
chunk_size=chunk_size,
)
return message_count
def move_expired_personal_and_huddle_messages_to_archive(
realm: Realm,
chunk_size: int = MESSAGE_BATCH_SIZE,
) -> int:
message_retention_days = realm.message_retention_days
assert message_retention_days != -1
check_date = timezone_now() - timedelta(days=message_retention_days)
# This function will archive appropriate messages and their related objects.
cross_realm_bot_ids = [
get_user_including_cross_realm(email).id
for email in settings.CROSS_REALM_BOT_EMAILS
]
recipient_types = (Recipient.PERSONAL, Recipient.HUDDLE)
# Archive expired personal and huddle Messages in the realm, except cross-realm messages.
# The condition zerver_userprofile.realm_id = {realm_id} assures the row won't be
# a message sent by a cross-realm bot, because cross-realm bots have their own separate realm.
query = SQL("""
INSERT INTO zerver_archivedmessage ({dst_fields}, archive_transaction_id)
SELECT {src_fields}, {archive_transaction_id}
FROM zerver_message
INNER JOIN zerver_recipient ON zerver_recipient.id = zerver_message.recipient_id
INNER JOIN zerver_userprofile ON zerver_userprofile.id = zerver_message.sender_id
WHERE zerver_userprofile.realm_id = {realm_id}
AND zerver_recipient.type in {recipient_types}
AND zerver_message.date_sent < {check_date}
LIMIT {chunk_size}
ON CONFLICT (id) DO UPDATE SET archive_transaction_id = {archive_transaction_id}
RETURNING id
""")
message_count = run_archiving_in_chunks(
query,
type=ArchiveTransaction.RETENTION_POLICY_BASED,
realm=realm,
cross_realm_bot_ids=Literal(tuple(cross_realm_bot_ids)),
realm_id=Literal(realm.id),
recipient_types=Literal(recipient_types),
check_date=Literal(check_date.isoformat()),
chunk_size=chunk_size,
)
# Archive cross-realm personal messages to users in the realm. We
# don't archive cross-realm huddle messages via retention policy,
# as we don't support them as a feature in Zulip, and the query to
# find and delete them would be a lot of complexity and potential
# performance work for a case that doesn't actually happen.
query = SQL("""
INSERT INTO zerver_archivedmessage ({dst_fields}, archive_transaction_id)
SELECT {src_fields}, {archive_transaction_id}
FROM zerver_message
INNER JOIN zerver_userprofile recipient_profile ON recipient_profile.recipient_id = zerver_message.recipient_id
WHERE zerver_message.sender_id IN {cross_realm_bot_ids}
AND recipient_profile.realm_id = {realm_id}
AND zerver_message.date_sent < {check_date}
LIMIT {chunk_size}
ON CONFLICT (id) DO UPDATE SET archive_transaction_id = {archive_transaction_id}
RETURNING id
""")
message_count += run_archiving_in_chunks(
query,
type=ArchiveTransaction.RETENTION_POLICY_BASED,
realm=realm,
cross_realm_bot_ids=Literal(tuple(cross_realm_bot_ids)),
realm_id=Literal(realm.id),
check_date=Literal(check_date.isoformat()),
chunk_size=chunk_size,
)
return message_count
