def consume(self, event: Dict[str, Any]) -> None:
if event['type'] == 'mark_stream_messages_as_read':
user_profile = get_user_profile_by_id(event['user_profile_id'])
client = Client.objects.get(id=event['client_id'])
for stream_id in event['stream_ids']:
# Since the user just unsubscribed, we don't require
# an active Subscription object (otherwise, private
# streams would never be accessible)
(stream, recipient,
sub) = access_stream_by_id(user_profile,
stream_id,
require_active=False)
do_mark_stream_messages_as_read(user_profile, client, stream)
elif event['type'] == 'clear_push_device_tokens':
try:
clear_push_device_tokens(event["user_profile_id"])
except PushNotificationBouncerRetryLaterError:
def failure_processor(event: Dict[str, Any]) -> None:
logger.warning(
"Maximum retries exceeded for trigger:%s event:clear_push_device_tokens"
% (event['user_profile_id'], ))
retry_event(self.queue_name, event, failure_processor)
elif event['type'] == 'realm_export':
start = time.time()
realm = Realm.objects.get(id=event['realm_id'])
output_dir = tempfile.mkdtemp(prefix="zulip-export-")
public_url = export_realm_wrapper(realm=realm,
output_dir=output_dir,
threads=6,
upload=True,
public_only=True,
delete_after_upload=True)
assert public_url is not None
# Update the extra_data field now that the export is complete.
export_event = RealmAuditLog.objects.get(id=event['id'])
export_event.extra_data = ujson.dumps(
dict(export_path=urllib.parse.urlparse(public_url).path, ))
export_event.save(update_fields=['extra_data'])
# Send a private message notification letting the user who
# triggered the export know the export finished.
user_profile = get_user_profile_by_id(event['user_profile_id'])
content = "Your data export is complete and has been uploaded here:\n\n%s" % (
public_url, )
internal_send_private_message(realm=user_profile.realm,
sender=get_system_bot(
settings.NOTIFICATION_BOT),
recipient_user=user_profile,
content=content)
# For future frontend use, also notify administrator
# clients that the export happened.
notify_realm_export(user_profile)
logging.info("Completed data export for %s in %s" %
(user_profile.realm.string_id, time.time() - start))
def export_realm(request: HttpRequest, user: UserProfile) -> HttpResponse:
# Currently only supports public-data-only exports.
event_type = RealmAuditLog.REALM_EXPORTED
event_time = timezone_now()
realm = user.realm
EXPORT_LIMIT = 5
# Conservative limit on the size of message history in
# organizations being exported; this exists to protect Zulip
# against a possible unmonitored accidental DoS caused by trying
# to export an organization with huge history.
MAX_MESSAGE_HISTORY = 250000
MAX_UPLOAD_QUOTA = 10 * 1024 * 1024 * 1024
# Filter based upon the number of events that have occurred in the delta
# If we are at the limit, the incoming request is rejected
event_time_delta = event_time - timedelta(days=7)
limit_check = RealmAuditLog.objects.filter(
realm=realm, event_type=event_type, event_time__gte=event_time_delta
)
if len(limit_check) >= EXPORT_LIMIT:
raise JsonableError(_("Exceeded rate limit."))
total_messages = sum(
realm_count.value
for realm_count in RealmCount.objects.filter(
realm=user.realm, property="messages_sent:client:day"
)
)
if (
total_messages > MAX_MESSAGE_HISTORY
or user.realm.currently_used_upload_space_bytes() > MAX_UPLOAD_QUOTA
):
raise JsonableError(
_("Please request a manual export from {email}.").format(
email=settings.ZULIP_ADMINISTRATOR,
)
)
row = RealmAuditLog.objects.create(
realm=realm, event_type=event_type, event_time=event_time, acting_user=user
)
# Allow for UI updates on a pending export
notify_realm_export(user)
# Using the deferred_work queue processor to avoid
# killing the process after 60s
event = {
"type": "realm_export",
"time": event_time,
"realm_id": realm.id,
"user_profile_id": user.id,
"id": row.id,
}
queue_json_publish("deferred_work", event)
return json_success()
def consume(self, event: Mapping[str, Any]) -> None:
if event['type'] == 'mark_stream_messages_as_read':
user_profile = get_user_profile_by_id(event['user_profile_id'])
client = Client.objects.get(id=event['client_id'])
for stream_id in event['stream_ids']:
# Since the user just unsubscribed, we don't require
# an active Subscription object (otherwise, private
# streams would never be accessible)
(stream, recipient,
sub) = access_stream_by_id(user_profile,
stream_id,
require_active=False)
do_mark_stream_messages_as_read(user_profile, client, stream)
elif event['type'] == 'realm_export':
realm = Realm.objects.get(id=event['realm_id'])
output_dir = tempfile.mkdtemp(prefix="zulip-export-")
public_url = export_realm_wrapper(realm=realm,
output_dir=output_dir,
threads=6,
upload=True,
public_only=True,
delete_after_upload=True)
assert public_url is not None
# Store the relative URL of the export.
export_event = RealmAuditLog.objects.get(id=event['id'])
export_event.extra_data = ujson.dumps({
'export_path':
urllib.parse.urlparse(public_url).path,
'deleted_timestamp':
None
})
export_event.save(update_fields=['extra_data'])
# Send a private message notification letting the user who
# triggered the export know the export finished.
user_profile = get_user_profile_by_id(event['user_profile_id'])
content = "Your data export is complete and has been uploaded here:\n\n%s" % (
public_url, )
internal_send_private_message(realm=user_profile.realm,
sender=get_system_bot(
settings.NOTIFICATION_BOT),
recipient_user=user_profile,
content=content)
# For future frontend use, also notify administrator
# clients that the export happened.
notify_realm_export(user_profile)
def consume(self, event: Dict[str, Any]) -> None:
start = time.time()
if event["type"] == "mark_stream_messages_as_read":
user_profile = get_user_profile_by_id(event["user_profile_id"])
for recipient_id in event["stream_recipient_ids"]:
count = do_mark_stream_messages_as_read(
user_profile, recipient_id)
logger.info(
"Marked %s messages as read for user %s, stream_recipient_id %s",
count,
user_profile.id,
recipient_id,
)
elif event["type"] == "mark_stream_messages_as_read_for_everyone":
# This event is generated by the stream deactivation code path.
batch_size = 100
offset = 0
while True:
messages = Message.objects.filter(
recipient_id=event["stream_recipient_id"]).order_by(
"id")[offset:offset + batch_size]
UserMessage.objects.filter(message__in=messages).extra(
where=[UserMessage.where_unread()]).update(
flags=F("flags").bitor(UserMessage.flags.read))
offset += len(messages)
if len(messages) < batch_size:
break
logger.info(
"Marked %s messages as read for all users, stream_recipient_id %s",
offset,
event["stream_recipient_id"],
)
elif event["type"] == "clear_push_device_tokens":
try:
clear_push_device_tokens(event["user_profile_id"])
except PushNotificationBouncerRetryLaterError:
def failure_processor(event: Dict[str, Any]) -> None:
logger.warning(
"Maximum retries exceeded for trigger:%s event:clear_push_device_tokens",
event["user_profile_id"],
)
retry_event(self.queue_name, event, failure_processor)
elif event["type"] == "realm_export":
realm = Realm.objects.get(id=event["realm_id"])
output_dir = tempfile.mkdtemp(prefix="zulip-export-")
export_event = RealmAuditLog.objects.get(id=event["id"])
user_profile = get_user_profile_by_id(event["user_profile_id"])
try:
public_url = export_realm_wrapper(
realm=realm,
output_dir=output_dir,
threads=6,
upload=True,
public_only=True,
delete_after_upload=True,
)
except Exception:
export_event.extra_data = orjson.dumps(
dict(failed_timestamp=timezone_now().timestamp(),
)).decode()
export_event.save(update_fields=["extra_data"])
logging.error(
"Data export for %s failed after %s",
user_profile.realm.string_id,
time.time() - start,
)
notify_realm_export(user_profile)
return
assert public_url is not None
# Update the extra_data field now that the export is complete.
export_event.extra_data = orjson.dumps(
dict(export_path=urllib.parse.urlparse(public_url).path,
)).decode()
export_event.save(update_fields=["extra_data"])
# Send a private message notification letting the user who
# triggered the export know the export finished.
with override_language(user_profile.default_language):
content = _(
"Your data export is complete and has been uploaded here:\n\n{public_url}"
).format(public_url=public_url)
internal_send_private_message(
sender=get_system_bot(settings.NOTIFICATION_BOT, realm.id),
recipient_user=user_profile,
content=content,
)
# For future frontend use, also notify administrator
# clients that the export happened.
notify_realm_export(user_profile)
logging.info(
"Completed data export for %s in %s",
user_profile.realm.string_id,
time.time() - start,
)
end = time.time()
logger.info("deferred_work processed %s event (%dms)", event["type"],
(end - start) * 1000)
def consume(self, event: Dict[str, Any]) -> None:
if event['type'] == 'mark_stream_messages_as_read':
user_profile = get_user_profile_by_id(event['user_profile_id'])
client = Client.objects.get(id=event['client_id'])
for stream_id in event['stream_ids']:
# Since the user just unsubscribed, we don't require
# an active Subscription object (otherwise, private
# streams would never be accessible)
(stream, recipient, sub) = access_stream_by_id(user_profile, stream_id,
require_active=False)
do_mark_stream_messages_as_read(user_profile, client, stream)
elif event["type"] == 'mark_stream_messages_as_read_for_everyone':
# This event is generated by the stream deactivation code path.
batch_size = 100
offset = 0
while True:
messages = Message.objects.filter(recipient_id=event["stream_recipient_id"]) \
.order_by("id")[offset:offset + batch_size]
UserMessage.objects.filter(message__in=messages).extra(where=[UserMessage.where_unread()]) \
.update(flags=F('flags').bitor(UserMessage.flags.read))
offset += len(messages)
if len(messages) < batch_size:
break
elif event['type'] == 'clear_push_device_tokens':
try:
clear_push_device_tokens(event["user_profile_id"])
except PushNotificationBouncerRetryLaterError:
def failure_processor(event: Dict[str, Any]) -> None:
logger.warning(
"Maximum retries exceeded for trigger:%s event:clear_push_device_tokens",
event['user_profile_id'])
retry_event(self.queue_name, event, failure_processor)
elif event['type'] == 'realm_export':
start = time.time()
realm = Realm.objects.get(id=event['realm_id'])
output_dir = tempfile.mkdtemp(prefix="zulip-export-")
export_event = RealmAuditLog.objects.get(id=event['id'])
user_profile = get_user_profile_by_id(event['user_profile_id'])
try:
public_url = export_realm_wrapper(realm=realm, output_dir=output_dir,
threads=6, upload=True, public_only=True,
delete_after_upload=True)
except Exception:
export_event.extra_data = orjson.dumps(dict(
failed_timestamp=timezone_now().timestamp(),
)).decode()
export_event.save(update_fields=['extra_data'])
logging.error(
"Data export for %s failed after %s",
user_profile.realm.string_id, time.time() - start,
)
notify_realm_export(user_profile)
return
assert public_url is not None
# Update the extra_data field now that the export is complete.
export_event.extra_data = orjson.dumps(dict(
export_path=urllib.parse.urlparse(public_url).path,
)).decode()
export_event.save(update_fields=['extra_data'])
# Send a private message notification letting the user who
# triggered the export know the export finished.
with override_language(user_profile.default_language):
content = _("Your data export is complete and has been uploaded here:\n\n{public_url}").format(public_url=public_url)
internal_send_private_message(
realm=user_profile.realm,
sender=get_system_bot(settings.NOTIFICATION_BOT),
recipient_user=user_profile,
content=content,
)
# For future frontend use, also notify administrator
# clients that the export happened.
notify_realm_export(user_profile)
logging.info(
"Completed data export for %s in %s",
user_profile.realm.string_id, time.time() - start,
)
