def do_events_register(user_profile: UserProfile, user_client: Client,
apply_markdown: bool = True,
client_gravatar: bool = False,
slim_presence: bool = False,
event_types: Optional[Iterable[str]] = None,
queue_lifespan_secs: int = 0,
all_public_streams: bool = False,
include_subscribers: bool = True,
notification_settings_null: bool = False,
narrow: Iterable[Sequence[str]] = [],
fetch_event_types: Optional[Iterable[str]] = None) -> Dict[str, Any]:
# Technically we don't need to check this here because
# build_narrow_filter will check it, but it's nicer from an error
# handling perspective to do it before contacting Tornado
check_supported_events_narrow_filter(narrow)
if user_profile.realm.email_address_visibility != Realm.EMAIL_ADDRESS_VISIBILITY_EVERYONE:
# If real email addresses are not available to the user, their
# clients cannot compute gravatars, so we force-set it to false.
client_gravatar = False
# Note that we pass event_types, not fetch_event_types here, since
# that's what controls which future events are sent.
queue_id = request_event_queue(user_profile, user_client,
apply_markdown, client_gravatar, slim_presence,
queue_lifespan_secs, event_types, all_public_streams,
narrow=narrow)
if queue_id is None:
raise JsonableError(_("Could not allocate event queue"))
if fetch_event_types is not None:
event_types_set = set(fetch_event_types) # type: Optional[Set[str]]
elif event_types is not None:
event_types_set = set(event_types)
else:
event_types_set = None
# Fill up the UserMessage rows if a soft-deactivated user has returned
reactivate_user_if_soft_deactivated(user_profile)
ret = fetch_initial_state_data(user_profile, event_types_set, queue_id,
client_gravatar=client_gravatar,
slim_presence=slim_presence,
include_subscribers=include_subscribers)
# Apply events that came in while we were fetching initial data
events = get_user_events(user_profile, queue_id, -1)
apply_events(ret, events, user_profile, include_subscribers=include_subscribers,
client_gravatar=client_gravatar, slim_presence=slim_presence,
fetch_event_types=fetch_event_types)
post_process_state(user_profile, ret, notification_settings_null)
if len(events) > 0:
ret['last_event_id'] = events[-1]['id']
else:
ret['last_event_id'] = -1
return ret
def test_reactivate_user_if_soft_deactivated(self) -> None:
recipient_list = [
self.example_user("hamlet"),
self.example_user("iago")
]
for user_profile in recipient_list:
self.subscribe(user_profile, "Denmark")
sender = self.example_user("iago")
stream_name = "Denmark"
topic_name = "foo"
def last_realm_audit_log_entry(event_type: int) -> RealmAuditLog:
return RealmAuditLog.objects.filter(
event_type=event_type).order_by("-event_time")[0]
long_term_idle_user = self.example_user("hamlet")
# We are sending this message to ensure that long_term_idle_user has
# at least one UserMessage row.
self.send_stream_message(long_term_idle_user, stream_name)
with self.assertLogs(logger_string, level="INFO") as info_logs:
do_soft_deactivate_users([long_term_idle_user])
self.assertEqual(
info_logs.output,
[
f"INFO:{logger_string}:Soft deactivated user {long_term_idle_user.id}",
f"INFO:{logger_string}:Soft-deactivated batch of 1 users; 0 remain to process",
],
)
message = "Test message 1"
message_id = self.send_stream_message(sender, stream_name, message,
topic_name)
idle_user_msg_list = get_user_messages(long_term_idle_user)
idle_user_msg_count = len(idle_user_msg_list)
self.assertNotEqual(idle_user_msg_list[-1].content, message)
with queries_captured() as queries:
reactivate_user_if_soft_deactivated(long_term_idle_user)
self.assert_length(queries, 8)
self.assertFalse(long_term_idle_user.long_term_idle)
self.assertEqual(
last_realm_audit_log_entry(
RealmAuditLog.USER_SOFT_ACTIVATED).modified_user,
long_term_idle_user,
)
idle_user_msg_list = get_user_messages(long_term_idle_user)
self.assert_length(idle_user_msg_list, idle_user_msg_count + 1)
self.assertEqual(idle_user_msg_list[-1].content, message)
long_term_idle_user.refresh_from_db()
self.assertEqual(long_term_idle_user.last_active_message_id,
message_id)
def test_reactivate_user_if_soft_deactivated(self) -> None:
recipient_list = [
self.example_user("hamlet"),
self.example_user("iago")
]
for user_profile in recipient_list:
self.subscribe(user_profile, "Denmark")
sender = self.example_user('iago')
stream_name = 'Denmark'
topic_name = 'foo'
def last_realm_audit_log_entry(event_type: int) -> RealmAuditLog:
return RealmAuditLog.objects.filter(
event_type=event_type, ).order_by('-event_time')[0]
long_term_idle_user = self.example_user('hamlet')
# We are sending this message to ensure that long_term_idle_user has
# at least one UserMessage row.
self.send_stream_message(long_term_idle_user, stream_name)
do_soft_deactivate_users([long_term_idle_user])
message = 'Test Message 1'
message_id = self.send_stream_message(sender, stream_name, message,
topic_name)
idle_user_msg_list = get_user_messages(long_term_idle_user)
idle_user_msg_count = len(idle_user_msg_list)
self.assertNotEqual(idle_user_msg_list[-1].content, message)
with queries_captured() as queries:
reactivate_user_if_soft_deactivated(long_term_idle_user)
self.assert_length(queries, 8)
self.assertFalse(long_term_idle_user.long_term_idle)
self.assertEqual(
last_realm_audit_log_entry(
RealmAuditLog.USER_SOFT_ACTIVATED).modified_user,
long_term_idle_user)
idle_user_msg_list = get_user_messages(long_term_idle_user)
self.assertEqual(len(idle_user_msg_list), idle_user_msg_count + 1)
self.assertEqual(idle_user_msg_list[-1].content, message)
long_term_idle_user.refresh_from_db()
self.assertEqual(long_term_idle_user.last_active_message_id,
message_id)
def consume(self, event: Dict[str, Any]) -> None:
start = time.time()
if event["type"] == "mark_stream_messages_as_read":
user_profile = get_user_profile_by_id(event["user_profile_id"])
for recipient_id in event["stream_recipient_ids"]:
count = do_mark_stream_messages_as_read(
user_profile, recipient_id)
logger.info(
"Marked %s messages as read for user %s, stream_recipient_id %s",
count,
user_profile.id,
recipient_id,
)
elif event["type"] == "mark_stream_messages_as_read_for_everyone":
# This event is generated by the stream deactivation code path.
batch_size = 100
offset = 0
while True:
messages = Message.objects.filter(
recipient_id=event["stream_recipient_id"]).order_by(
"id")[offset:offset + batch_size]
UserMessage.objects.filter(message__in=messages).extra(
where=[UserMessage.where_unread()]).update(
flags=F("flags").bitor(UserMessage.flags.read))
offset += len(messages)
if len(messages) < batch_size:
break
logger.info(
"Marked %s messages as read for all users, stream_recipient_id %s",
offset,
event["stream_recipient_id"],
)
elif event["type"] == "clear_push_device_tokens":
try:
clear_push_device_tokens(event["user_profile_id"])
except PushNotificationBouncerRetryLaterError:
def failure_processor(event: Dict[str, Any]) -> None:
logger.warning(
"Maximum retries exceeded for trigger:%s event:clear_push_device_tokens",
event["user_profile_id"],
)
retry_event(self.queue_name, event, failure_processor)
elif event["type"] == "realm_export":
realm = Realm.objects.get(id=event["realm_id"])
output_dir = tempfile.mkdtemp(prefix="zulip-export-")
export_event = RealmAuditLog.objects.get(id=event["id"])
user_profile = get_user_profile_by_id(event["user_profile_id"])
try:
public_url = export_realm_wrapper(
realm=realm,
output_dir=output_dir,
threads=6,
upload=True,
public_only=True,
)
except Exception:
export_event.extra_data = orjson.dumps(
dict(failed_timestamp=timezone_now().timestamp(),
)).decode()
export_event.save(update_fields=["extra_data"])
logging.exception(
"Data export for %s failed after %s",
user_profile.realm.string_id,
time.time() - start,
stack_info=True,
)
notify_realm_export(user_profile)
return
assert public_url is not None
# Update the extra_data field now that the export is complete.
export_event.extra_data = orjson.dumps(
dict(export_path=urllib.parse.urlparse(public_url).path,
)).decode()
export_event.save(update_fields=["extra_data"])
# Send a private message notification letting the user who
# triggered the export know the export finished.
with override_language(user_profile.default_language):
content = _(
"Your data export is complete and has been uploaded here:\n\n{public_url}"
).format(public_url=public_url)
internal_send_private_message(
sender=get_system_bot(settings.NOTIFICATION_BOT, realm.id),
recipient_user=user_profile,
content=content,
)
# For future frontend use, also notify administrator
# clients that the export happened.
notify_realm_export(user_profile)
logging.info(
"Completed data export for %s in %s",
user_profile.realm.string_id,
time.time() - start,
)
elif event["type"] == "reupload_realm_emoji":
# This is a special event queued by the migration for reuploading emojis.
# We don't want to run the necessary code in the actual migration, so it simply
# queues the necessary event, and the actual work is done here in the queue worker.
realm = Realm.objects.get(id=event["realm_id"])
logger.info("Processing reupload_realm_emoji event for realm %s",
realm.id)
handle_reupload_emojis_event(realm, logger)
elif event["type"] == "soft_reactivate":
user_profile = get_user_profile_by_id(event["user_profile_id"])
reactivate_user_if_soft_deactivated(user_profile)
end = time.time()
logger.info("deferred_work processed %s event (%dms)", event["type"],
(end - start) * 1000)
