def handle(self, *args, **options):
# type: (*Any, **Any) -> None
realm = self.get_realm(options)
user_profile = self.get_user(options['email'], realm)
print("Deactivating %s (%s) - %s" % (user_profile.full_name,
user_profile.email,
user_profile.realm.string_id))
print("%s has the following active sessions:" % (user_profile.email,))
for session in user_sessions(user_profile):
print(session.expire_date, session.get_decoded())
print("")
print("%s has %s active bots that will also be deactivated." % (
user_profile.email,
UserProfile.objects.filter(
is_bot=True, is_active=True, bot_owner=user_profile
).count()
))
if not options["for_real"]:
print("This was a dry run. Pass -f to actually deactivate.")
exit(1)
do_deactivate_user(user_profile)
print("Sessions deleted, user deactivated.")
def test_clear_scheduled_jobs(self) -> None:
user = self.example_user('hamlet')
send_future_email('zerver/emails/followup_day1', user.realm,
to_user_ids=[user.id], delay=datetime.timedelta(hours=1))
self.assertEqual(ScheduledEmail.objects.count(), 1)
do_deactivate_user(user)
self.assertEqual(ScheduledEmail.objects.count(), 0)
def test_access_user_by_id(self) -> None:
iago = self.example_user("iago")
# Must be a valid user ID in the realm
with self.assertRaises(JsonableError):
access_user_by_id(iago, 1234)
with self.assertRaises(JsonableError):
access_user_by_id(iago, self.mit_user("sipbtest").id)
# Can only access bot users if allow_deactivated is passed
bot = self.example_user("welcome_bot")
access_user_by_id(iago, bot.id, allow_bots=True)
with self.assertRaises(JsonableError):
access_user_by_id(iago, bot.id)
# Can only access deactivated users if allow_deactivated is passed
hamlet = self.example_user("hamlet")
do_deactivate_user(hamlet)
with self.assertRaises(JsonableError):
access_user_by_id(iago, hamlet.id)
access_user_by_id(iago, hamlet.id, allow_deactivated=True)
# Non-admin user can't admin another user
with self.assertRaises(JsonableError):
access_user_by_id(self.example_user("cordelia"),
self.example_user("aaron").id)
def test_access_user_by_id(self) -> None:
iago = self.example_user("iago")
# Must be a valid user ID in the realm
with self.assertRaises(JsonableError):
access_user_by_id(iago, 1234)
with self.assertRaises(JsonableError):
access_user_by_id(iago, self.mit_user("sipbtest").id)
# Can only access bot users if allow_deactivated is passed
bot = self.example_user("welcome_bot")
access_user_by_id(iago, bot.id, allow_bots=True)
with self.assertRaises(JsonableError):
access_user_by_id(iago, bot.id)
# Can only access deactivated users if allow_deactivated is passed
hamlet = self.example_user("hamlet")
do_deactivate_user(hamlet)
with self.assertRaises(JsonableError):
access_user_by_id(iago, hamlet.id)
access_user_by_id(iago, hamlet.id, allow_deactivated=True)
# Non-admin user can't admin another user
with self.assertRaises(JsonableError):
access_user_by_id(self.example_user("cordelia"), self.example_user("aaron").id)
def test_user_activation(self) -> None:
realm = get_realm('zulip')
now = timezone_now()
user = do_create_user('email', 'password', realm, 'full_name',
'short_name')
do_deactivate_user(user)
do_activate_user(user)
do_deactivate_user(user)
do_reactivate_user(user)
self.assertEqual(
RealmAuditLog.objects.filter(event_time__gte=now).count(), 5)
event_types = list(
RealmAuditLog.objects.filter(
realm=realm,
acting_user=None,
modified_user=user,
modified_stream=None,
event_time__gte=now,
event_time__lte=now +
timedelta(minutes=60)).order_by('event_time').values_list(
'event_type', flat=True))
self.assertEqual(event_types, [
RealmAuditLog.USER_CREATED, RealmAuditLog.USER_DEACTIVATED,
RealmAuditLog.USER_ACTIVATED, RealmAuditLog.USER_DEACTIVATED,
RealmAuditLog.USER_REACTIVATED
])
def test_inactive_user(self):
# type: () -> None
do_deactivate_user(self.user_profile)
result = self.client_post("/api/v1/fetch_api_key",
dict(username=self.email,
password=initial_password(self.email)))
self.assert_json_error_contains(result, "Your account has been disabled", 403)
def deactivate_user_own_backend(request, user_profile):
# type: (HttpRequest, UserProfile) -> HttpResponse
if user_profile.is_realm_admin and check_last_admin(user_profile):
return json_error(_('Cannot deactivate the only organization administrator'))
do_deactivate_user(user_profile, acting_user=user_profile)
return json_success()
def _deactivate_user_profile_backend(request, user_profile, target):
# type: (HttpRequest, UserProfile, UserProfile) -> HttpResponse
if not user_profile.can_admin_user(target):
return json_error(_('Insufficient permission'))
do_deactivate_user(target, acting_user=user_profile)
return json_success()
def get_or_build_user(self, username: str,
ldap_user: _LDAPUser) -> Tuple[UserProfile, bool]:
"""This is used only in non-authentication contexts such as:
./manage.py sync_ldap_user_data
In authentication contexts, this is overriden in ZulipLDAPAuthBackend.
"""
(user, built) = super().get_or_build_user(username, ldap_user)
self.sync_avatar_from_ldap(user, ldap_user)
self.sync_full_name_from_ldap(user, ldap_user)
self.sync_custom_profile_fields_from_ldap(user, ldap_user)
if 'userAccountControl' in settings.AUTH_LDAP_USER_ATTR_MAP:
user_disabled_in_ldap = self.is_account_control_disabled_user(
ldap_user)
if user_disabled_in_ldap and user.is_active:
logging.info(
"Deactivating user %s because they are disabled in LDAP." %
(user.email, ))
do_deactivate_user(user)
return (user, built)
if not user_disabled_in_ldap and not user.is_active:
logging.info(
"Reactivating user %s because they are not disabled in LDAP."
% (user.email, ))
do_reactivate_user(user)
return (user, built)
def test_clear_scheduled_jobs(self) -> None:
user = self.example_user('hamlet')
send_future_email('zerver/emails/followup_day1', user.realm,
to_user_id=user.id, delay=datetime.timedelta(hours=1))
self.assertEqual(ScheduledEmail.objects.count(), 1)
do_deactivate_user(user)
self.assertEqual(ScheduledEmail.objects.count(), 0)
def test_active_users_log_by_is_bot(self):
# type: () -> None
property = 'active_users_log:is_bot:day'
user = do_create_user('email', 'password', self.default_realm,
'full_name', 'short_name')
self.assertEqual(
1,
RealmCount.objects.filter(property=property,
subgroup=False).aggregate(
Sum('value'))['value__sum'])
do_deactivate_user(user)
self.assertEqual(
0,
RealmCount.objects.filter(property=property,
subgroup=False).aggregate(
Sum('value'))['value__sum'])
do_activate_user(user)
self.assertEqual(
1,
RealmCount.objects.filter(property=property,
subgroup=False).aggregate(
Sum('value'))['value__sum'])
do_deactivate_user(user)
self.assertEqual(
0,
RealmCount.objects.filter(property=property,
subgroup=False).aggregate(
Sum('value'))['value__sum'])
do_reactivate_user(user)
self.assertEqual(
1,
RealmCount.objects.filter(property=property,
subgroup=False).aggregate(
Sum('value'))['value__sum'])
def test_basics(self):
# type: () -> None
user = self.example_user('hamlet')
do_deactivate_user(user)
self.assertFalse(user.is_active)
do_reactivate_user(user)
self.assertTrue(user.is_active)
def handle(self, *args: Any, **options: Any) -> None:
realm = self.get_realm(options)
user_profile = self.get_user(options['email'], realm)
print(
f"Deactivating {user_profile.full_name} ({user_profile.delivery_email}) - {user_profile.realm.string_id}"
)
print(
f"{user_profile.delivery_email} has the following active sessions:"
)
for session in user_sessions(user_profile):
print(session.expire_date, session.get_decoded())
print("")
print("{} has {} active bots that will also be deactivated.".format(
user_profile.delivery_email,
UserProfile.objects.filter(
is_bot=True,
is_active=True,
bot_owner=user_profile,
).count(),
))
if not options["for_real"]:
raise CommandError(
"This was a dry run. Pass -f to actually deactivate.")
do_deactivate_user(user_profile)
print("Sessions deleted, user deactivated.")
def deactivate_user_own_backend(request, user_profile):
# type: (HttpRequest, UserProfile) -> HttpResponse
if user_profile.is_realm_admin and check_last_admin(user_profile):
return json_error(_('Cannot deactivate the only organization administrator'))
do_deactivate_user(user_profile)
return json_success()
def _deactivate_user_profile_backend(request, user_profile, target):
# type: (HttpRequest, UserProfile, UserProfile) -> HttpResponse
if not user_profile.can_admin_user(target):
return json_error(_('Insufficient permission'))
do_deactivate_user(target)
return json_success()
def test_change_email_deactivated_user_realm(self) -> None:
data = {"email": "*****@*****.**"}
user_profile = self.example_user("hamlet")
self.login_user(user_profile)
url = "/json/settings"
self.assert_length(mail.outbox, 0)
result = self.client_patch(url, data)
self.assert_length(mail.outbox, 1)
self.assert_json_success(result)
email_message = mail.outbox[0]
self.assertEqual(
email_message.subject,
"Verify your new email address",
)
body = email_message.body
self.assertIn("We received a request to change the email", body)
activation_url = [s for s in body.split("\n") if s][2]
do_deactivate_user(user_profile, acting_user=None)
response = self.client_get(activation_url)
self.assertEqual(response.status_code, 401)
do_deactivate_realm(user_profile.realm, acting_user=None)
response = self.client_get(activation_url)
self.assertEqual(response.status_code, 302)
self.assertTrue(response["Location"].endswith("/accounts/deactivated/"))
def test_send_deactivated_user(self):
"""
rest_dispatch rejects requests from deactivated users, both /json and api
"""
email = "*****@*****.**"
user_profile = get_user_profile_by_email(email)
self.login(email)
do_deactivate_user(user_profile)
result = self.client_post("/json/messages", {"type": "private",
"content": "Test message",
"client": "test suite",
"to": "*****@*****.**"})
self.assert_json_error_contains(result, "Not logged in", status_code=401)
# Even if a logged-in session was leaked, it still wouldn't work
do_reactivate_user(user_profile)
self.login(email)
user_profile.is_active = False
user_profile.save()
result = self.client_post("/json/messages", {"type": "private",
"content": "Test message",
"client": "test suite",
"to": "*****@*****.**"})
self.assert_json_error_contains(result, "Account not active", status_code=400)
result = self.client_post("/api/v1/messages", {"type": "private",
"content": "Test message",
"client": "test suite",
"to": "*****@*****.**"},
**self.api_auth("*****@*****.**"))
self.assert_json_error_contains(result, "Account not active", status_code=401)
def sync_user_from_ldap(user_profile: UserProfile) -> bool:
backend = ZulipLDAPUserPopulator()
updated_user = backend.populate_user(backend.django_to_ldap_username(user_profile.email))
if not updated_user:
if settings.LDAP_DEACTIVATE_NON_MATCHING_USERS:
do_deactivate_user(user_profile)
return False
return True
def deactivate_user_own_backend(request: HttpRequest, user_profile: UserProfile) -> HttpResponse:
if UserProfile.objects.filter(realm=user_profile.realm, is_active=True).count() == 1:
raise CannotDeactivateLastUserError(is_last_admin=False)
if user_profile.is_realm_admin and check_last_admin(user_profile):
raise CannotDeactivateLastUserError(is_last_admin=True)
do_deactivate_user(user_profile, acting_user=user_profile)
return json_success()
def deactivate_user_own_backend(request, user_profile):
# type: (HttpRequest, UserProfile) -> HttpResponse
admins = set(user_profile.realm.get_admin_users())
if user_profile.is_realm_admin and len(admins) == 1:
return json_error(_('Cannot deactivate the only admin'))
do_deactivate_user(user_profile)
return json_success()
def deactivate_user_own_backend(request, user_profile):
# type: (HttpRequest, UserProfile) -> HttpResponse
admins = set(user_profile.realm.get_admin_users())
if user_profile.is_realm_admin and len(admins) == 1:
return json_error(_('Cannot deactivate the only admin'))
do_deactivate_user(user_profile)
return json_success()
def deactivate_user_own_backend(request: HttpRequest, user_profile: UserProfile) -> HttpResponse:
if UserProfile.objects.filter(realm=user_profile.realm, is_active=True).count() == 1:
raise CannotDeactivateLastUserError(is_last_owner=False)
if user_profile.is_realm_owner and check_last_owner(user_profile):
raise CannotDeactivateLastUserError(is_last_owner=True)
do_deactivate_user(user_profile, acting_user=user_profile)
return json_success()
def test_login_deactivated_user(self):
"""
logging in fails with an inactive user
"""
email = "*****@*****.**"
user_profile = get_user_profile_by_email(email)
do_deactivate_user(user_profile)
result = self.login_with_return("*****@*****.**")
self.assert_in_response("Please enter a correct email and password", result)
def test_login_deactivated_user(self):
# type: () -> None
"""
logging in fails with an inactive user
"""
user_profile = self.example_user('hamlet')
do_deactivate_user(user_profile)
result = self.login_with_return(self.example_email("hamlet"))
self.assert_in_response(
"Sorry for the trouble, but your account has been deactivated",
result)
def test_login_deactivated_user(self):
# type: () -> None
"""
logging in fails with an inactive user
"""
user_profile = self.example_user('hamlet')
do_deactivate_user(user_profile)
result = self.login_with_return(self.example_email("hamlet"))
self.assert_in_response(
"Sorry for the trouble, but your account has been deactivated",
result)
def test_get_seat_count(self) -> None:
initial_count = get_seat_count(self.realm)
user1 = UserProfile.objects.create(realm=self.realm, email='*****@*****.**', pointer=-1)
user2 = UserProfile.objects.create(realm=self.realm, email='*****@*****.**', pointer=-1)
self.assertEqual(get_seat_count(self.realm), initial_count + 2)
# Test that bots aren't counted
user1.is_bot = True
user1.save(update_fields=['is_bot'])
self.assertEqual(get_seat_count(self.realm), initial_count + 1)
# Test that inactive users aren't counted
do_deactivate_user(user2)
self.assertEqual(get_seat_count(self.realm), initial_count)
def _test_remove_muted_user_valid_data(self,
deactivate_user: bool = False
) -> None:
hamlet = self.example_user("hamlet")
self.login_user(hamlet)
cordelia = self.example_user("cordelia")
mute_time = datetime(2021, 1, 1, tzinfo=timezone.utc)
if deactivate_user:
do_deactivate_user(cordelia, acting_user=None)
with mock.patch("zerver.views.muting.timezone_now",
return_value=mute_time):
url = f"/api/v1/users/me/muted_users/{cordelia.id}"
result = self.api_post(hamlet, url)
self.assert_json_success(result)
with mock.patch("zerver.lib.actions.timezone_now",
return_value=mute_time):
# To test that `RealmAuditLog` entry has correct `event_time`.
url = f"/api/v1/users/me/muted_users/{cordelia.id}"
result = self.api_delete(hamlet, url)
self.assert_json_success(result)
self.assertNotIn(
{
"id": cordelia.id,
"timestamp": datetime_to_timestamp(mute_time),
},
get_user_mutes(hamlet),
)
self.assertIsNone(get_mute_object(hamlet, cordelia))
audit_log_entries = list(
RealmAuditLog.objects.filter(acting_user=hamlet,
modified_user=hamlet).values_list(
"event_type", "event_time",
"extra_data").order_by("id"))
self.assert_length(audit_log_entries, 2)
audit_log_entry = audit_log_entries[1]
self.assertEqual(
audit_log_entry,
(
RealmAuditLog.USER_UNMUTED,
mute_time,
orjson.dumps({
"unmuted_user_id": cordelia.id
}).decode(),
),
)
def sync_user_from_ldap(user_profile: UserProfile, logger: logging.Logger) -> bool:
backend = ZulipLDAPUserPopulator()
updated_user = backend.populate_user(backend.django_to_ldap_username(user_profile.email))
if updated_user:
logger.info("Updated %s." % (user_profile.email,))
return True
if settings.LDAP_DEACTIVATE_NON_MATCHING_USERS:
do_deactivate_user(user_profile)
logger.info("Deactivated non-matching user: %s" % (user_profile.email,))
return True
elif user_profile.is_active:
logger.warning("Did not find %s in LDAP." % (user_profile.email,))
return False
def test_single_user_get(self) -> None:
reset_emails_in_zulip_realm()
# First, we setup the test with some data
user = self.example_user("othello")
self.login_user(user)
result = self.client_post("/json/users/me/presence",
{"status": "active"})
result = self.client_post("/json/users/me/presence",
{"status": "active"},
HTTP_USER_AGENT="ZulipDesktop/1.0")
result = self.api_post(
user,
"/api/v1/users/me/presence",
{"status": "idle"},
HTTP_USER_AGENT="ZulipAndroid/1.0",
)
self.assert_json_success(result)
# Check some error conditions
result = self.client_get("/json/users/[email protected]/presence")
self.assert_json_error(result, "No such user")
result = self.client_get("/json/users/[email protected]/presence")
self.assert_json_error(result,
"No presence data for [email protected]")
do_deactivate_user(self.example_user("cordelia"))
result = self.client_get("/json/users/[email protected]/presence")
self.assert_json_error(result, "No such user")
result = self.client_get("/json/users/[email protected]/presence")
self.assert_json_error(result,
"Presence is not supported for bot users.")
sipbtest = self.mit_user("sipbtest")
self.login_user(sipbtest)
result = self.client_get("/json/users/[email protected]/presence",
subdomain="zephyr")
self.assert_json_error(result, "No such user")
# Then, we check everything works
self.login("hamlet")
result = self.client_get("/json/users/[email protected]/presence")
result_dict = result.json()
self.assertEqual(set(result_dict["presence"].keys()),
{"ZulipAndroid", "website", "aggregated"})
self.assertEqual(set(result_dict["presence"]["website"].keys()),
{"status", "timestamp"})
def test_webhook_deactivated_user(self):
"""
Deactivated users can't use webhooks
"""
email = "*****@*****.**"
user_profile = get_user_profile_by_email(email)
do_deactivate_user(user_profile)
api_key = self.get_api_key(email)
url = "/api/v1/external/jira?api_key=%s&stream=jira_custom" % (api_key,)
data = self.fixture_data('jira', "created")
result = self.client_post(url, data,
content_type="application/json")
self.assert_json_error_contains(result, "Account not active", status_code=400)
def test_webhook_deactivated_user(self):
# type: () -> None
"""
Deactivated users can't use webhooks
"""
user_profile = self.example_user('hamlet')
do_deactivate_user(user_profile)
url = "/api/v1/external/jira?api_key=%s&stream=jira_custom" % (
user_profile.api_key,)
data = self.fixture_data('jira', "created_v2")
result = self.client_post(url, data,
content_type="application/json")
self.assert_json_error_contains(result, "Account not active", status_code=400)
def test_user_activation(self) -> None:
realm = get_realm('zulip')
now = timezone_now()
user = do_create_user('email', 'password', realm, 'full_name', 'short_name')
do_deactivate_user(user)
do_activate_user(user)
do_deactivate_user(user)
do_reactivate_user(user)
self.assertEqual(RealmAuditLog.objects.filter(event_time__gte=now).count(), 5)
event_types = list(RealmAuditLog.objects.filter(
realm=realm, acting_user=None, modified_user=user, modified_stream=None,
event_time__gte=now, event_time__lte=now+timedelta(minutes=60))
.order_by('event_time').values_list('event_type', flat=True))
self.assertEqual(event_types, [RealmAuditLog.USER_CREATED, RealmAuditLog.USER_DEACTIVATED, RealmAuditLog.USER_ACTIVATED,
RealmAuditLog.USER_DEACTIVATED, RealmAuditLog.USER_REACTIVATED])
def test_get_seat_count(self) -> None:
realm = get_realm("zulip")
initial_count = get_seat_count(realm)
user1 = UserProfile.objects.create(realm=realm, email='*****@*****.**', pointer=-1)
user2 = UserProfile.objects.create(realm=realm, email='*****@*****.**', pointer=-1)
self.assertEqual(get_seat_count(realm), initial_count + 2)
# Test that bots aren't counted
user1.is_bot = True
user1.save(update_fields=['is_bot'])
self.assertEqual(get_seat_count(realm), initial_count + 1)
# Test that inactive users aren't counted
do_deactivate_user(user2)
self.assertEqual(get_seat_count(realm), initial_count)
def test_webhook_deactivated_user(self):
# type: () -> None
"""
Deactivated users can't use webhooks
"""
user_profile = self.example_user('hamlet')
do_deactivate_user(user_profile)
url = "/api/v1/external/jira?api_key=%s&stream=jira_custom" % (
user_profile.api_key,)
data = self.fixture_data('jira', "created_v2")
result = self.client_post(url, data,
content_type="application/json")
self.assert_json_error_contains(result, "Account not active", status_code=400)
def get_or_build_user(self, username: str,
ldap_user: _LDAPUser) -> Tuple[UserProfile, bool]: # nocoverage
(user, built) = super().get_or_build_user(username, ldap_user)
self.sync_avatar_from_ldap(user, ldap_user)
if 'userAccountControl' in settings.AUTH_LDAP_USER_ATTR_MAP:
user_disabled_in_ldap = self.is_account_control_disabled_user(ldap_user)
if user_disabled_in_ldap and user.is_active:
logging.info("Deactivating user %s because they are disabled in LDAP." %
(user.email,))
do_deactivate_user(user)
return (user, built)
if not user_disabled_in_ldap and not user.is_active:
logging.info("Reactivating user %s because they are not disabled in LDAP." %
(user.email,))
do_reactivate_user(user)
return (user, built)
def test_requires_billing_update_for_is_active_changes(self) -> None:
count = RealmAuditLog.objects.count()
user1 = do_create_user('*****@*****.**', 'password', self.realm, 'full name', 'short name')
do_deactivate_user(user1)
do_reactivate_user(user1)
# Not a proper use of do_activate_user, but it's fine to call it like this for this test
do_activate_user(user1)
self.assertEqual(count + 4,
RealmAuditLog.objects.filter(requires_billing_update=False).count())
self.realm.has_seat_based_plan = True
self.realm.save(update_fields=['has_seat_based_plan'])
user2 = do_create_user('*****@*****.**', 'password', self.realm, 'full name', 'short name')
do_deactivate_user(user2)
do_reactivate_user(user2)
do_activate_user(user2)
self.assertEqual(4, RealmAuditLog.objects.filter(requires_billing_update=True).count())
def test_patch_bot_owner_deactivated(self) -> None:
self.login(self.example_email('hamlet'))
self.create_bot()
self.assert_num_bots_equal(1)
target_user_profile = self.example_user("othello")
do_deactivate_user(target_user_profile)
target_user_profile = self.example_user('othello')
self.assertFalse(target_user_profile.is_active)
bot_info = {
'bot_owner': self.example_email('othello'),
}
result = self.client_patch("/json/bots/[email protected]", bot_info)
self.assert_json_error(result, "Failed to change owner, user is deactivated")
profile = get_user('*****@*****.**', get_realm('zulip'))
self.assertEqual(profile.bot_owner, self.example_user("hamlet"))
def get_or_build_user(self, username: str,
ldap_user: _LDAPUser) -> Tuple[UserProfile, bool]:
(user, built) = super().get_or_build_user(username, ldap_user)
self.sync_avatar_from_ldap(user, ldap_user)
self.sync_full_name_from_ldap(user, ldap_user)
if 'userAccountControl' in settings.AUTH_LDAP_USER_ATTR_MAP:
user_disabled_in_ldap = self.is_account_control_disabled_user(ldap_user)
if user_disabled_in_ldap and user.is_active:
logging.info("Deactivating user %s because they are disabled in LDAP." %
(user.email,))
do_deactivate_user(user)
return (user, built)
if not user_disabled_in_ldap and not user.is_active:
logging.info("Reactivating user %s because they are not disabled in LDAP." %
(user.email,))
do_reactivate_user(user)
return (user, built)
def test_single_user_get(self):
# type: () -> None
# First, we setup the test with some data
email = self.example_email("othello")
self.login(self.example_email("othello"))
result = self.client_post("/json/users/me/presence",
{'status': 'active'})
result = self.client_post("/json/users/me/presence",
{'status': 'active'},
HTTP_USER_AGENT="ZulipDesktop/1.0")
result = self.client_post("/api/v1/users/me/presence",
{'status': 'idle'},
HTTP_USER_AGENT="ZulipAndroid/1.0",
**self.api_auth(email))
self.assert_json_success(result)
# Check some error conditions
result = self.client_get("/json/users/[email protected]/presence")
self.assert_json_error(result, "No such user")
result = self.client_get("/json/users/[email protected]/presence")
self.assert_json_error(result,
"No presence data for [email protected]")
do_deactivate_user(self.example_user('cordelia'))
result = self.client_get("/json/users/[email protected]/presence")
self.assert_json_error(result, "No such user")
result = self.client_get("/json/users/[email protected]/presence")
self.assert_json_error(result,
"Presence is not supported for bot users.")
self.login(self.mit_email("sipbtest"))
result = self.client_get("/json/users/[email protected]/presence",
subdomain="zephyr")
self.assert_json_error(result, "No such user")
# Then, we check everything works
self.login(self.example_email("hamlet"))
result = self.client_get("/json/users/[email protected]/presence")
result_dict = result.json()
self.assertEqual(set(result_dict['presence'].keys()),
{"ZulipAndroid", "website", "aggregated"})
self.assertEqual(set(result_dict['presence']['website'].keys()),
{"status", "timestamp"})
def test_requires_billing_update_for_is_active_changes(self) -> None:
count = RealmAuditLog.objects.count()
realm = get_realm("zulip")
user1 = do_create_user('*****@*****.**', 'password', realm, 'full name', 'short name')
do_deactivate_user(user1)
do_reactivate_user(user1)
# Not a proper use of do_activate_user, but it's fine to call it like this for this test
do_activate_user(user1)
self.assertEqual(count + 4,
RealmAuditLog.objects.filter(requires_billing_update=False).count())
realm.has_seat_based_plan = True
realm.save(update_fields=['has_seat_based_plan'])
user2 = do_create_user('*****@*****.**', 'password', realm, 'full name', 'short name')
do_deactivate_user(user2)
do_reactivate_user(user2)
do_activate_user(user2)
self.assertEqual(4, RealmAuditLog.objects.filter(requires_billing_update=True).count())
def test_active_users_log_by_is_bot(self):
# type: () -> None
property = 'active_users_log:is_bot:day'
user = do_create_user('email', 'password', self.default_realm, 'full_name', 'short_name')
self.assertEqual(1, RealmCount.objects.filter(property=property, subgroup=False)
.aggregate(Sum('value'))['value__sum'])
do_deactivate_user(user)
self.assertEqual(0, RealmCount.objects.filter(property=property, subgroup=False)
.aggregate(Sum('value'))['value__sum'])
do_activate_user(user)
self.assertEqual(1, RealmCount.objects.filter(property=property, subgroup=False)
.aggregate(Sum('value'))['value__sum'])
do_deactivate_user(user)
self.assertEqual(0, RealmCount.objects.filter(property=property, subgroup=False)
.aggregate(Sum('value'))['value__sum'])
do_reactivate_user(user)
self.assertEqual(1, RealmCount.objects.filter(property=property, subgroup=False)
.aggregate(Sum('value'))['value__sum'])
def test_billing_quantity_changes_end_to_end(
self, mock_customer_with_subscription: Mock, mock_create_subscription: Mock,
mock_create_customer: Mock) -> None:
self.login(self.example_email("hamlet"))
processor = BillingProcessor.objects.create(
log_row=RealmAuditLog.objects.order_by('id').first(), state=BillingProcessor.DONE)
def check_billing_processor_update(event_type: str, quantity: int) -> None:
def check_subscription_save(subscription: stripe.Subscription, idempotency_key: str) -> None:
self.assertEqual(subscription.quantity, quantity)
log_row = RealmAuditLog.objects.filter(
event_type=event_type, requires_billing_update=True).order_by('-id').first()
self.assertEqual(idempotency_key, 'process_billing_log_entry:%s' % (log_row.id,))
self.assertEqual(subscription.proration_date, datetime_to_timestamp(log_row.event_time))
with patch.object(stripe.Subscription, 'save', autospec=True,
side_effect=check_subscription_save):
run_billing_processor_one_step(processor)
# Test STRIPE_PLAN_QUANTITY_RESET
new_seat_count = 123
# change the seat count while the user is going through the upgrade flow
with patch('corporate.lib.stripe.get_seat_count', return_value=new_seat_count):
self.client_post("/upgrade/", {'stripeToken': self.token,
'signed_seat_count': self.signed_seat_count,
'salt': self.salt,
'plan': Plan.CLOUD_ANNUAL})
check_billing_processor_update(RealmAuditLog.STRIPE_PLAN_QUANTITY_RESET, new_seat_count)
# Test USER_CREATED
user = do_create_user('*****@*****.**', 'password', get_realm('zulip'), 'full name', 'short name')
check_billing_processor_update(RealmAuditLog.USER_CREATED, self.quantity + 1)
# Test USER_DEACTIVATED
do_deactivate_user(user)
check_billing_processor_update(RealmAuditLog.USER_DEACTIVATED, self.quantity - 1)
# Test USER_REACTIVATED
do_reactivate_user(user)
check_billing_processor_update(RealmAuditLog.USER_REACTIVATED, self.quantity + 1)
# Test USER_ACTIVATED
# Not a proper use of do_activate_user, but it's fine to call it like this for this test
do_activate_user(user)
check_billing_processor_update(RealmAuditLog.USER_ACTIVATED, self.quantity + 1)
def test_user_activation(self) -> None:
realm = get_realm("zulip")
now = timezone_now()
user = do_create_user("email",
"password",
realm,
"full_name",
acting_user=None)
do_deactivate_user(user, acting_user=user)
do_activate_user(user, acting_user=user)
do_deactivate_user(user, acting_user=user)
do_reactivate_user(user, acting_user=user)
self.assertEqual(
RealmAuditLog.objects.filter(event_time__gte=now).count(), 5)
event_types = list(
RealmAuditLog.objects.filter(
realm=realm,
acting_user=user,
modified_user=user,
modified_stream=None,
event_time__gte=now,
event_time__lte=now + timedelta(minutes=60),
).order_by("event_time").values_list("event_type", flat=True))
self.assertEqual(
event_types,
[
RealmAuditLog.USER_CREATED,
RealmAuditLog.USER_DEACTIVATED,
RealmAuditLog.USER_ACTIVATED,
RealmAuditLog.USER_DEACTIVATED,
RealmAuditLog.USER_REACTIVATED,
],
)
for event in RealmAuditLog.objects.filter(
realm=realm,
acting_user=user,
modified_user=user,
modified_stream=None,
event_time__gte=now,
event_time__lte=now + timedelta(minutes=60),
):
extra_data = orjson.loads(event.extra_data)
self.check_role_count_schema(extra_data[RealmAuditLog.ROLE_COUNT])
self.assertNotIn(RealmAuditLog.OLD_VALUE, extra_data)
def test_billing_quantity_changes_end_to_end(
self, mock_customer_with_subscription: Mock, mock_create_subscription: Mock,
mock_create_customer: Mock) -> None:
self.login(self.example_email("hamlet"))
processor = BillingProcessor.objects.create(
log_row=RealmAuditLog.objects.order_by('id').first(), state=BillingProcessor.DONE)
def check_billing_processor_update(event_type: str, quantity: int) -> None:
def check_subscription_save(subscription: stripe.Subscription, idempotency_key: str) -> None:
self.assertEqual(subscription.quantity, quantity)
log_row = RealmAuditLog.objects.filter(
event_type=event_type, requires_billing_update=True).order_by('-id').first()
self.assertEqual(idempotency_key, 'process_billing_log_entry:%s' % (log_row.id,))
self.assertEqual(subscription.proration_date, datetime_to_timestamp(log_row.event_time))
with patch.object(stripe.Subscription, 'save', autospec=True,
side_effect=check_subscription_save):
run_billing_processor_one_step(processor)
# Test STRIPE_PLAN_QUANTITY_RESET
new_seat_count = 123
# change the seat count while the user is going through the upgrade flow
with patch('corporate.lib.stripe.get_seat_count', return_value=new_seat_count):
self.client_post("/upgrade/", {'stripeToken': self.token,
'signed_seat_count': self.signed_seat_count,
'salt': self.salt,
'plan': Plan.CLOUD_ANNUAL})
check_billing_processor_update(RealmAuditLog.STRIPE_PLAN_QUANTITY_RESET, new_seat_count)
# Test USER_CREATED
user = do_create_user('*****@*****.**', 'password', get_realm('zulip'), 'full name', 'short name')
check_billing_processor_update(RealmAuditLog.USER_CREATED, self.quantity + 1)
# Test USER_DEACTIVATED
do_deactivate_user(user)
check_billing_processor_update(RealmAuditLog.USER_DEACTIVATED, self.quantity - 1)
# Test USER_REACTIVATED
do_reactivate_user(user)
check_billing_processor_update(RealmAuditLog.USER_REACTIVATED, self.quantity + 1)
# Test USER_ACTIVATED
# Not a proper use of do_activate_user, but it's fine to call it like this for this test
do_activate_user(user)
check_billing_processor_update(RealmAuditLog.USER_ACTIVATED, self.quantity + 1)
def get_or_build_user(
self, username: str,
ldap_user: _LDAPUser) -> Tuple[UserProfile, bool]: # nocoverage
(user, built) = super().get_or_build_user(username, ldap_user)
self.sync_avatar_from_ldap(user, ldap_user)
user_disabled_in_ldap = self.is_account_control_disabled_user(
ldap_user)
if user_disabled_in_ldap and user.is_active:
logging.info(
"Deactivating user %s because they are disabled in LDAP." %
(user.email, ))
do_deactivate_user(user)
return (user, built)
if not user_disabled_in_ldap and not user.is_active:
logging.info(
"Reactivating user %s because they are not disabled in LDAP." %
(user.email, ))
do_reactivate_user(user)
return (user, built)
def test_login_deactivated_mirror_dummy(self):
# type: () -> None
"""
logging in fails with an inactive user
"""
user_profile = self.example_user('hamlet')
user_profile.is_mirror_dummy = True
user_profile.save()
password = initial_password(user_profile.email)
request = mock.MagicMock()
request.get_host.return_value = 'zulip.testserver'
# Test a mirror-dummy active user.
form = OurAuthenticationForm(request,
data={'username': user_profile.email,
'password': password})
with self.settings(AUTHENTICATION_BACKENDS=('zproject.backends.EmailAuthBackend',)):
self.assertTrue(form.is_valid())
# Test a mirror-dummy deactivated user.
do_deactivate_user(user_profile)
user_profile.save()
form = OurAuthenticationForm(request,
data={'username': user_profile.email,
'password': password})
with self.settings(AUTHENTICATION_BACKENDS=('zproject.backends.EmailAuthBackend',)):
self.assertFalse(form.is_valid())
self.assertIn("Please enter a correct email", str(form.errors))
# Test a non-mirror-dummy deactivated user.
user_profile.is_mirror_dummy = False
user_profile.save()
form = OurAuthenticationForm(request,
data={'username': user_profile.email,
'password': password})
with self.settings(AUTHENTICATION_BACKENDS=('zproject.backends.EmailAuthBackend',)):
self.assertFalse(form.is_valid())
self.assertIn("your account has been deactivated", str(form.errors))
def test_login_deactivated_mirror_dummy(self):
# type: () -> None
"""
logging in fails with an inactive user
"""
user_profile = self.example_user('hamlet')
user_profile.is_mirror_dummy = True
user_profile.save()
password = initial_password(user_profile.email)
request = mock.MagicMock()
request.get_host.return_value = 'zulip.testserver'
# Test a mirror-dummy active user.
form = OurAuthenticationForm(request,
data={'username': user_profile.email,
'password': password})
with self.settings(AUTHENTICATION_BACKENDS=('zproject.backends.EmailAuthBackend',)):
self.assertTrue(form.is_valid())
# Test a mirror-dummy deactivated user.
do_deactivate_user(user_profile)
user_profile.save()
form = OurAuthenticationForm(request,
data={'username': user_profile.email,
'password': password})
with self.settings(AUTHENTICATION_BACKENDS=('zproject.backends.EmailAuthBackend',)):
self.assertFalse(form.is_valid())
self.assertIn("Please enter a correct email", str(form.errors))
# Test a non-mirror-dummy deactivated user.
user_profile.is_mirror_dummy = False
user_profile.save()
form = OurAuthenticationForm(request,
data={'username': user_profile.email,
'password': password})
with self.settings(AUTHENTICATION_BACKENDS=('zproject.backends.EmailAuthBackend',)):
self.assertFalse(form.is_valid())
self.assertIn("your account has been deactivated", str(form.errors))
def test_single_user_get(self):
# type: () -> None
# First, we setup the test with some data
email = self.example_email("othello")
self.login(self.example_email("othello"))
result = self.client_post("/json/users/me/presence", {'status': 'active'})
result = self.client_post("/json/users/me/presence", {'status': 'active'},
HTTP_USER_AGENT="ZulipDesktop/1.0")
result = self.client_post("/api/v1/users/me/presence", {'status': 'idle'},
HTTP_USER_AGENT="ZulipAndroid/1.0",
**self.api_auth(email))
self.assert_json_success(result)
# Check some error conditions
result = self.client_get("/json/users/[email protected]/presence")
self.assert_json_error(result, "No such user")
result = self.client_get("/json/users/[email protected]/presence")
self.assert_json_error(result, "No presence data for [email protected]")
do_deactivate_user(self.example_user('cordelia'))
result = self.client_get("/json/users/[email protected]/presence")
self.assert_json_error(result, "No such user")
result = self.client_get("/json/users/[email protected]/presence")
self.assert_json_error(result, "Presence is not supported for bot users.")
self.login(self.mit_email("sipbtest"))
result = self.client_get("/json/users/[email protected]/presence",
subdomain="zephyr")
self.assert_json_error(result, "No such user")
# Then, we check everything works
self.login(self.example_email("hamlet"))
result = self.client_get("/json/users/[email protected]/presence")
result_dict = result.json()
self.assertEqual(
set(result_dict['presence'].keys()),
{"ZulipAndroid", "website", "aggregated"})
self.assertEqual(set(result_dict['presence']['website'].keys()), {"status", "timestamp"})
def handle(self, *args, **options):
user_profile = get_user_profile_by_email(options['email'])
print "Deactivating %s (%s) - %s" % (user_profile.full_name,
user_profile.email,
user_profile.realm.domain)
print "%s has the following active sessions:" % (user_profile.email, )
for session in user_sessions(user_profile):
print session.expire_date, session.get_decoded()
print ""
print "%s has %s active bots that will also be deactivated." % (
user_profile.email,
UserProfile.objects.filter(
is_bot=True, is_active=True, bot_owner=user_profile).count())
if not options["for_real"]:
print "This was a dry run. Pass -f to actually deactivate."
exit(1)
do_deactivate_user(user_profile)
print "Sessions deleted, user deactivated."
def verify_backend(self, backend, good_args=None,
good_kwargs=None, bad_kwargs=None,
email_to_username=None):
# type: (Any, List[Any], Dict[str, Any], Dict[str, Any], Callable[[text_type], text_type]) -> None
if good_args is None:
good_args = []
if good_kwargs is None:
good_kwargs = {}
email = u"*****@*****.**"
user_profile = get_user_profile_by_email(email)
username = email
if email_to_username is not None:
username = email_to_username(email)
# If bad_kwargs was specified, verify auth fails in that case
if bad_kwargs is not None:
self.assertIsNone(backend.authenticate(username, **bad_kwargs))
# Verify auth works
result = backend.authenticate(username, *good_args, **good_kwargs)
self.assertEqual(user_profile, result)
# Verify auth fails with a deactivated user
do_deactivate_user(user_profile)
self.assertIsNone(backend.authenticate(username, *good_args, **good_kwargs))
# Reactivate the user and verify auth works again
do_reactivate_user(user_profile)
result = backend.authenticate(username, *good_args, **good_kwargs)
self.assertEqual(user_profile, result)
# Verify auth fails with a deactivated realm
do_deactivate_realm(user_profile.realm)
self.assertIsNone(backend.authenticate(username, *good_args, **good_kwargs))
# Verify auth works again after reactivating the realm
do_reactivate_realm(user_profile.realm)
result = backend.authenticate(username, *good_args, **good_kwargs)
self.assertEqual(user_profile, result)
def get_or_build_user(self, username: str,
ldap_user: _LDAPUser) -> Tuple[UserProfile, bool]:
"""This is used only in non-authentication contexts such as:
./manage.py sync_ldap_user_data
In authentication contexts, this is overriden in ZulipLDAPAuthBackend.
"""
(user, built) = super().get_or_build_user(username, ldap_user)
self.sync_avatar_from_ldap(user, ldap_user)
self.sync_full_name_from_ldap(user, ldap_user)
self.sync_custom_profile_fields_from_ldap(user, ldap_user)
if 'userAccountControl' in settings.AUTH_LDAP_USER_ATTR_MAP:
user_disabled_in_ldap = self.is_account_control_disabled_user(ldap_user)
if user_disabled_in_ldap and user.is_active:
logging.info("Deactivating user %s because they are disabled in LDAP." %
(user.email,))
do_deactivate_user(user)
return (user, built)
if not user_disabled_in_ldap and not user.is_active:
logging.info("Reactivating user %s because they are not disabled in LDAP." %
(user.email,))
do_reactivate_user(user)
return (user, built)
def test_do_deactivate_user(self):
bot_deactivate_checker = check_dict([
('type', equals('realm_bot')),
('op', equals('remove')),
('bot', check_dict([
('email', check_string),
('full_name', check_string),
])),
])
bot = self.create_bot('*****@*****.**')
action = lambda: do_deactivate_user(bot)
events = self.do_test(action)
error = bot_deactivate_checker('events[1]', events[1])
self.assert_on_error(error)
def handle(self, *args: Any, **options: Any) -> None:
realm = self.get_realm(options)
user_profile = self.get_user(options['email'], realm)
print("Deactivating %s (%s) - %s" %
(user_profile.full_name, user_profile.email,
user_profile.realm.string_id))
print("%s has the following active sessions:" % (user_profile.email, ))
for session in user_sessions(user_profile):
print(session.expire_date, session.get_decoded())
print("")
print(
"%s has %s active bots that will also be deactivated." %
(user_profile.email,
UserProfile.objects.filter(
is_bot=True, is_active=True, bot_owner=user_profile).count()))
if not options["for_real"]:
print("This was a dry run. Pass -f to actually deactivate.")
exit(1)
do_deactivate_user(user_profile)
print("Sessions deleted, user deactivated.")
def handle(self, *args, **options):
user_profile = get_user_profile_by_email(options['email'])
print "Deactivating %s (%s) - %s" % (user_profile.full_name,
user_profile.email,
user_profile.realm.domain)
print "%s has the following active sessions:" % (user_profile.email,)
for session in user_sessions(user_profile):
print session.expire_date, session.get_decoded()
print ""
print "%s has %s active bots that will also be deactivated." % (
user_profile.email,
UserProfile.objects.filter(
is_bot=True, is_active=True, bot_owner=user_profile
).count()
)
if not options["for_real"]:
print "This was a dry run. Pass -f to actually deactivate."
exit(1)
do_deactivate_user(user_profile)
print "Sessions deleted, user deactivated."
def test_do_deactivate_user(self):
bot_deactivate_checker = check_dict([
('type', equals('realm_bot')),
('op', equals('remove')),
('bot', check_dict([
('email', check_string),
('full_name', check_string),
])),
])
bot = self.create_bot('*****@*****.**')
action = lambda: do_deactivate_user(bot)
events = self.do_test(action)
error = bot_deactivate_checker('events[1]', events[1])
self.assert_on_error(error)
def test_basics(self) -> None:
user = self.example_user('hamlet')
do_deactivate_user(user)
self.assertFalse(user.is_active)
do_reactivate_user(user)
self.assertTrue(user.is_active)
def _deactivate_user_profile_backend(request: HttpRequest, user_profile: UserProfile,
target: UserProfile) -> HttpResponse:
do_deactivate_user(target, acting_user=user_profile)
return json_success()
def restore_saved_messages():
# type: () -> None
old_messages = [] # type: List[Dict[str, Any]]
duplicate_suppression_hash = {} # type: Dict[str, bool]
stream_dict = {} # type: Dict[Tuple[text_type, text_type], Tuple[text_type, text_type]]
user_set = set() # type: Set[Tuple[text_type, text_type, text_type, bool]]
email_set = set([u.email for u in UserProfile.objects.all()]) # type: Set[text_type]
realm_set = set() # type: Set[text_type]
# Initial client_set is nonempty temporarily because we don't have
# clients in logs at all right now -- later we can start with nothing.
client_set = set(["populate_db", "website", "zephyr_mirror"])
huddle_user_set = set() # type: Set[Tuple[text_type, ...]]
# First, determine all the objects our messages will need.
print(datetime.datetime.now(), "Creating realms/streams/etc...")
def process_line(line):
# type: (str) -> None
old_message_json = line.strip()
# Due to populate_db's shakespeare mode, we have a lot of
# duplicate messages in our log that only differ in their
# logged ID numbers (same timestamp, content, etc.). With
# sqlite, bulk creating those messages won't work properly: in
# particular, the first 100 messages will actually only result
# in 20 rows ending up in the target table, which screws up
# the below accounting where for handling changing
# subscriptions, we assume that the Nth row populate_db
# created goes with the Nth non-subscription row of the input
# So suppress the duplicates when using sqlite.
if "sqlite" in settings.DATABASES["default"]["ENGINE"]:
tmp_message = ujson.loads(old_message_json)
tmp_message['id'] = '1'
duplicate_suppression_key = ujson.dumps(tmp_message)
if duplicate_suppression_key in duplicate_suppression_hash:
return
duplicate_suppression_hash[duplicate_suppression_key] = True
old_message = ujson.loads(old_message_json)
message_type = old_message["type"]
# Lower case emails and domains; it will screw up
# deduplication if we don't
def fix_email(email):
# type: (text_type) -> text_type
return email.strip().lower()
if message_type in ["stream", "huddle", "personal"]:
old_message["sender_email"] = fix_email(old_message["sender_email"])
# Fix the length on too-long messages before we start processing them
if len(old_message["content"]) > MAX_MESSAGE_LENGTH:
old_message["content"] = "[ This message was deleted because it was too long ]"
if message_type in ["subscription_added", "subscription_removed"]:
old_message["domain"] = old_message["domain"].lower()
old_message["user"] = fix_email(old_message["user"])
elif message_type == "subscription_property":
old_message["user"] = fix_email(old_message["user"])
elif message_type == "user_email_changed":
old_message["old_email"] = fix_email(old_message["old_email"])
old_message["new_email"] = fix_email(old_message["new_email"])
elif message_type.startswith("user_"):
old_message["user"] = fix_email(old_message["user"])
elif message_type.startswith("enable_"):
old_message["user"] = fix_email(old_message["user"])
if message_type == 'personal':
old_message["recipient"][0]["email"] = fix_email(old_message["recipient"][0]["email"])
elif message_type == "huddle":
for i in range(len(old_message["recipient"])):
old_message["recipient"][i]["email"] = fix_email(old_message["recipient"][i]["email"])
old_messages.append(old_message)
if message_type in ["subscription_added", "subscription_removed"]:
stream_name = old_message["name"].strip() # type: text_type
canon_stream_name = stream_name.lower()
if canon_stream_name not in stream_dict:
stream_dict[(old_message["domain"], canon_stream_name)] = \
(old_message["domain"], stream_name)
elif message_type == "user_created":
user_set.add((old_message["user"], old_message["full_name"], old_message["short_name"], False))
elif message_type == "realm_created":
realm_set.add(old_message["domain"])
if message_type not in ["stream", "huddle", "personal"]:
return
sender_email = old_message["sender_email"]
domain = split_email_to_domain(sender_email)
realm_set.add(domain)
if old_message["sender_email"] not in email_set:
user_set.add((old_message["sender_email"],
old_message["sender_full_name"],
old_message["sender_short_name"],
False))
if 'sending_client' in old_message:
client_set.add(old_message['sending_client'])
if message_type == 'stream':
stream_name = old_message["recipient"].strip()
canon_stream_name = stream_name.lower()
if canon_stream_name not in stream_dict:
stream_dict[(domain, canon_stream_name)] = (domain, stream_name)
elif message_type == 'personal':
u = old_message["recipient"][0]
if u["email"] not in email_set:
user_set.add((u["email"], u["full_name"], u["short_name"], False))
email_set.add(u["email"])
elif message_type == 'huddle':
for u in old_message["recipient"]:
user_set.add((u["email"], u["full_name"], u["short_name"], False))
if u["email"] not in email_set:
user_set.add((u["email"], u["full_name"], u["short_name"], False))
email_set.add(u["email"])
huddle_user_set.add(tuple(sorted(set(u["email"] for u in old_message["recipient"]))))
else:
raise ValueError('Bad message type')
event_glob = os.path.join(settings.EVENT_LOG_DIR, 'events.*')
for filename in sorted(glob.glob(event_glob)):
with open(filename, "r") as message_log:
for line in message_log.readlines():
process_line(line)
stream_recipients = {} # type: Dict[Tuple[int, text_type], Recipient]
user_recipients = {} # type: Dict[text_type, Recipient]
huddle_recipients = {} # type: Dict[text_type, Recipient]
# Then, create the objects our messages need.
print(datetime.datetime.now(), "Creating realms...")
bulk_create_realms(realm_set)
realms = {} # type: Dict[text_type, Realm]
for realm in Realm.objects.all():
realms[realm.domain] = realm
print(datetime.datetime.now(), "Creating clients...")
bulk_create_clients(client_set)
clients = {} # type: Dict[text_type, Client]
for client in Client.objects.all():
clients[client.name] = client
print(datetime.datetime.now(), "Creating streams...")
bulk_create_streams(realms, list(stream_dict.values()))
streams = {} # type: Dict[int, Stream]
for stream in Stream.objects.all():
streams[stream.id] = stream
for recipient in Recipient.objects.filter(type=Recipient.STREAM):
stream_recipients[(streams[recipient.type_id].realm_id,
streams[recipient.type_id].name.lower())] = recipient
print(datetime.datetime.now(), "Creating users...")
bulk_create_users(realms, user_set, tos_version=settings.TOS_VERSION)
users = {} # type: Dict[text_type, UserProfile]
users_by_id = {} # type: Dict[int, UserProfile]
for user_profile in UserProfile.objects.select_related().all():
users[user_profile.email] = user_profile
users_by_id[user_profile.id] = user_profile
for recipient in Recipient.objects.filter(type=Recipient.PERSONAL):
user_recipients[users_by_id[recipient.type_id].email] = recipient
print(datetime.datetime.now(), "Creating huddles...")
bulk_create_huddles(users, huddle_user_set)
huddles_by_id = {} # type: Dict[int, Huddle]
for huddle in Huddle.objects.all():
huddles_by_id[huddle.id] = huddle
for recipient in Recipient.objects.filter(type=Recipient.HUDDLE):
huddle_recipients[huddles_by_id[recipient.type_id].huddle_hash] = recipient
# TODO: Add a special entry type in the log that is a subscription
# change and import those as we go to make subscription changes
# take effect!
print(datetime.datetime.now(), "Importing subscriptions...")
subscribers = {} # type: Dict[int, Set[int]]
for s in Subscription.objects.select_related().all():
if s.active:
subscribers.setdefault(s.recipient.id, set()).add(s.user_profile.id)
# Then create all the messages, without talking to the DB!
print(datetime.datetime.now(), "Importing messages, part 1...")
first_message_id = None
if Message.objects.exists():
first_message_id = Message.objects.all().order_by("-id")[0].id + 1
messages_to_create = [] # type: List[Message]
for idx, old_message in enumerate(old_messages):
message_type = old_message["type"]
if message_type not in ["stream", "huddle", "personal"]:
continue
message = Message()
sender_email = old_message["sender_email"]
domain = split_email_to_domain(sender_email)
realm = realms[domain]
message.sender = users[sender_email]
type_hash = {"stream": Recipient.STREAM,
"huddle": Recipient.HUDDLE,
"personal": Recipient.PERSONAL}
if 'sending_client' in old_message:
message.sending_client = clients[old_message['sending_client']]
elif sender_email in ["*****@*****.**", "*****@*****.**", "*****@*****.**",
"*****@*****.**", "*****@*****.**"]:
message.sending_client = clients['populate_db']
elif realm.domain == "zulip.com":
message.sending_client = clients["website"]
elif realm.domain == "mit.edu":
message.sending_client = clients['zephyr_mirror']
else:
message.sending_client = clients['populate_db']
message.type = type_hash[message_type]
message.content = old_message["content"]
message.subject = old_message["subject"]
message.pub_date = timestamp_to_datetime(old_message["timestamp"])
if message.type == Recipient.PERSONAL:
message.recipient = user_recipients[old_message["recipient"][0]["email"]]
elif message.type == Recipient.STREAM:
message.recipient = stream_recipients[(realm.id,
old_message["recipient"].lower())]
elif message.type == Recipient.HUDDLE:
huddle_hash = get_huddle_hash([users[u["email"]].id
for u in old_message["recipient"]])
message.recipient = huddle_recipients[huddle_hash]
else:
raise ValueError('Bad message type')
messages_to_create.append(message)
print(datetime.datetime.now(), "Importing messages, part 2...")
Message.objects.bulk_create(messages_to_create)
messages_to_create = []
# Finally, create all the UserMessage objects
print(datetime.datetime.now(), "Importing usermessages, part 1...")
personal_recipients = {} # type: Dict[int, bool]
for r in Recipient.objects.filter(type = Recipient.PERSONAL):
personal_recipients[r.id] = True
all_messages = Message.objects.all() # type: Sequence[Message]
user_messages_to_create = [] # type: List[UserMessage]
messages_by_id = {} # type: Dict[int, Message]
for message in all_messages:
messages_by_id[message.id] = message
if len(messages_by_id) == 0:
print(datetime.datetime.now(), "No old messages to replay")
return
if first_message_id is None:
first_message_id = min(messages_by_id.keys())
tot_user_messages = 0
pending_subs = {} # type: Dict[Tuple[int, int], bool]
current_message_id = first_message_id
pending_colors = {} # type: Dict[Tuple[text_type, text_type], text_type]
for old_message in old_messages:
message_type = old_message["type"]
if message_type == 'subscription_added':
stream_key = (realms[old_message["domain"]].id, old_message["name"].strip().lower())
subscribers.setdefault(stream_recipients[stream_key].id,
set()).add(users[old_message["user"]].id)
pending_subs[(stream_recipients[stream_key].id,
users[old_message["user"]].id)] = True
continue
elif message_type == "subscription_removed":
stream_key = (realms[old_message["domain"]].id, old_message["name"].strip().lower())
user_id = users[old_message["user"]].id
subscribers.setdefault(stream_recipients[stream_key].id, set())
try:
subscribers[stream_recipients[stream_key].id].remove(user_id)
except KeyError:
print("Error unsubscribing %s from %s: not subscribed" % (
old_message["user"], old_message["name"]))
pending_subs[(stream_recipients[stream_key].id,
users[old_message["user"]].id)] = False
continue
elif message_type == "user_activated" or message_type == "user_created":
# These are rare, so just handle them the slow way
user_profile = users[old_message["user"]]
join_date = timestamp_to_datetime(old_message['timestamp'])
do_activate_user(user_profile, log=False, join_date=join_date)
# Update the cache of users to show this user as activated
users_by_id[user_profile.id] = user_profile
users[old_message["user"]] = user_profile
continue
elif message_type == "user_deactivated":
user_profile = users[old_message["user"]]
do_deactivate_user(user_profile, log=False)
continue
elif message_type == "user_change_password":
# Just handle these the slow way
user_profile = users[old_message["user"]]
do_change_password(user_profile, old_message["pwhash"], log=False,
hashed_password=True)
continue
elif message_type == "user_change_full_name":
# Just handle these the slow way
user_profile = users[old_message["user"]]
user_profile.full_name = old_message["full_name"]
user_profile.save(update_fields=["full_name"])
continue
elif message_type == "enable_desktop_notifications_changed":
# Just handle these the slow way
user_profile = users[old_message["user"]]
user_profile.enable_desktop_notifications = (old_message["enable_desktop_notifications"] != "false")
user_profile.save(update_fields=["enable_desktop_notifications"])
continue
elif message_type == "enable_sounds_changed":
user_profile = users[old_message["user"]]
user_profile.enable_sounds = (old_message["enable_sounds"] != "false")
user_profile.save(update_fields=["enable_sounds"])
elif message_type == "enable_offline_email_notifications_changed":
user_profile = users[old_message["user"]]
user_profile.enable_offline_email_notifications = (
old_message["enable_offline_email_notifications"] != "false")
user_profile.save(update_fields=["enable_offline_email_notifications"])
continue
elif message_type == "enable_offline_push_notifications_changed":
user_profile = users[old_message["user"]]
user_profile.enable_offline_push_notifications = (
old_message["enable_offline_push_notifications"] != "false")
user_profile.save(update_fields=["enable_offline_push_notifications"])
continue
elif message_type == "default_streams":
set_default_streams(get_realm(old_message["domain"]),
old_message["streams"])
continue
elif message_type == "subscription_property":
property_name = old_message.get("property")
if property_name == "stream_color" or property_name == "color":
color = old_message.get("color", old_message.get("value"))
pending_colors[(old_message["user"],
old_message["stream_name"].lower())] = color
elif property_name in ["in_home_view", "notifications"]:
# TODO: Handle this
continue
else:
raise RuntimeError("Unknown property %s" % (property_name,))
continue
elif message_type == "realm_created":
# No action required
continue
elif message_type in ["user_email_changed", "update_onboarding", "update_message"]:
# TODO: Handle these
continue
if message_type not in ["stream", "huddle", "personal"]:
raise RuntimeError("Unexpected message type %s" % (message_type,))
message = messages_by_id[current_message_id]
current_message_id += 1
if message.recipient_id not in subscribers:
# Nobody received this message -- probably due to our
# subscriptions being out-of-date.
continue
recipient_user_ids = set() # type: Set[int]
for user_profile_id in subscribers[message.recipient_id]:
recipient_user_ids.add(user_profile_id)
if message.recipient_id in personal_recipients:
# Include the sender in huddle recipients
recipient_user_ids.add(message.sender_id)
for user_profile_id in recipient_user_ids:
if users_by_id[user_profile_id].is_active:
um = UserMessage(user_profile_id=user_profile_id,
message=message)
user_messages_to_create.append(um)
if len(user_messages_to_create) > 100000:
tot_user_messages += len(user_messages_to_create)
UserMessage.objects.bulk_create(user_messages_to_create)
user_messages_to_create = []
print(datetime.datetime.now(), "Importing usermessages, part 2...")
tot_user_messages += len(user_messages_to_create)
UserMessage.objects.bulk_create(user_messages_to_create)
print(datetime.datetime.now(), "Finalizing subscriptions...")
current_subs = {} # type: Dict[Tuple[int, int], bool]
current_subs_obj = {} # type: Dict[Tuple[int, int], Subscription]
for s in Subscription.objects.select_related().all():
current_subs[(s.recipient_id, s.user_profile_id)] = s.active
current_subs_obj[(s.recipient_id, s.user_profile_id)] = s
subscriptions_to_add = [] # type: List[Subscription]
subscriptions_to_change = [] # type: List[Tuple[Tuple[int, int], bool]]
for pending_sub in pending_subs.keys():
(recipient_id, user_profile_id) = pending_sub
current_state = current_subs.get(pending_sub)
if pending_subs[pending_sub] == current_state:
# Already correct in the database
continue
elif current_state is not None:
subscriptions_to_change.append((pending_sub, pending_subs[pending_sub]))
continue
s = Subscription(recipient_id=recipient_id,
user_profile_id=user_profile_id,
active=pending_subs[pending_sub])
subscriptions_to_add.append(s)
Subscription.objects.bulk_create(subscriptions_to_add)
for (sub_tuple, active) in subscriptions_to_change:
current_subs_obj[sub_tuple].active = active
current_subs_obj[sub_tuple].save(update_fields=["active"])
subs = {} # type: Dict[Tuple[int, int], Subscription]
for sub in Subscription.objects.all():
subs[(sub.user_profile_id, sub.recipient_id)] = sub
# TODO: do restore of subscription colors -- we're currently not
# logging changes so there's little point in having the code :(
print(datetime.datetime.now(), "Finished importing %s messages (%s usermessages)" % \
(len(all_messages), tot_user_messages))
site = Site.objects.get_current()
site.domain = 'zulip.com'
site.save()
print(datetime.datetime.now(), "Filling in user pointers...")
# Set restored pointers to the very latest messages
for user_profile in UserProfile.objects.all():
try:
top = UserMessage.objects.filter(
user_profile_id=user_profile.id).order_by("-message")[0]
user_profile.pointer = top.message_id
except IndexError:
user_profile.pointer = -1
user_profile.save(update_fields=["pointer"])
print(datetime.datetime.now(), "Done replaying old messages")
def _deactivate_user_profile_backend(request, user_profile, target):
if not user_profile.can_admin_user(target):
return json_error('Insufficient permission')
do_deactivate_user(target)
return json_success({})