def build_user_profile( avatar_source: str, date_joined: Any, delivery_email: str, email: str, full_name: str, id: int, is_active: bool, role: int, is_mirror_dummy: bool, realm_id: int, short_name: str, timezone: Optional[str], is_bot: bool = False, bot_type: Optional[int] = None, ) -> ZerverFieldsT: obj = UserProfile( avatar_source=avatar_source, date_joined=date_joined, delivery_email=delivery_email, email=email, full_name=full_name, id=id, is_mirror_dummy=is_mirror_dummy, is_active=is_active, role=role, realm_id=realm_id, timezone=timezone, is_bot=is_bot, bot_type=bot_type, ) dct = model_to_dict(obj) """ Even though short_name is no longer in the Zulip UserProfile, it's helpful to have it in our import dictionaries for legacy reasons. """ dct["short_name"] = short_name return dct
def build_user(avatar_source: str, date_joined: Any, delivery_email: str, email: str, full_name: str, id: int, is_active: bool, is_realm_admin: bool, is_guest: bool, realm_id: int, short_name: str, timezone: str) -> ZerverFieldsT: pointer = -1 obj = UserProfile( avatar_source=avatar_source, date_joined=date_joined, delivery_email=delivery_email, email=email, full_name=full_name, id=id, is_active=is_active, is_realm_admin=is_realm_admin, is_guest=is_guest, pointer=pointer, realm_id=realm_id, short_name=short_name, timezone=timezone, ) dct = model_to_dict(obj) return dct
def create_user_profile(realm, email, password, active, bot_type, full_name, short_name, bot_owner, is_mirror_dummy, tos_version): # type: (Realm, Text, Optional[Text], bool, Optional[int], Text, Text, Optional[UserProfile], bool, Optional[Text]) -> UserProfile now = timezone.now() email = UserManager.normalize_email(email) user_profile = UserProfile(email=email, is_staff=False, is_active=active, full_name=full_name, short_name=short_name, last_login=now, date_joined=now, realm=realm, pointer=-1, is_bot=bool(bot_type), bot_type=bot_type, bot_owner=bot_owner, is_mirror_dummy=is_mirror_dummy, tos_version=tos_version, onboarding_steps=ujson.dumps([]), default_language=realm.default_language) if bot_type or not active: password = None user_profile.set_password(password) user_profile.api_key = random_api_key() return user_profile
def create_user_profile(realm, email, password, active, bot_type, full_name, short_name, bot_owner, is_mirror_dummy): now = timezone.now() email = UserManager.normalize_email(email) enable_stream_desktop_notifications = (realm.domain != 'zulip.com') user_profile = UserProfile(email=email, is_staff=False, is_active=active, full_name=full_name, short_name=short_name, last_login=now, date_joined=now, realm=realm, pointer=-1, is_bot=bool(bot_type), bot_type=bot_type, bot_owner=bot_owner, is_mirror_dummy=is_mirror_dummy, enable_stream_desktop_notifications=enable_stream_desktop_notifications, onboarding_steps=ujson.dumps([])) if bot_type or not active: password = None user_profile.set_password(password) user_profile.api_key = random_api_key() return user_profile
def build_user_profile(avatar_source: str, date_joined: Any, delivery_email: str, email: str, full_name: str, id: int, is_active: bool, role: int, is_mirror_dummy: bool, realm_id: int, short_name: str, timezone: Optional[str]) -> ZerverFieldsT: pointer = -1 obj = UserProfile( avatar_source=avatar_source, date_joined=date_joined, delivery_email=delivery_email, email=email, full_name=full_name, id=id, is_mirror_dummy=is_mirror_dummy, is_active=is_active, role=role, pointer=pointer, realm_id=realm_id, short_name=short_name, timezone=timezone, ) dct = model_to_dict(obj) return dct
def create_user_profile(realm, email, password, active, bot_type, full_name, short_name, bot_owner, is_mirror_dummy, tos_version): # type: (Realm, text_type, text_type, bool, Optional[int], text_type, text_type, Optional[UserProfile], bool, Optional[text_type]) -> UserProfile now = timezone.now() email = UserManager.normalize_email(email) enable_stream_desktop_notifications = (realm.domain != 'zulip.com') user_profile = UserProfile( email=email, is_staff=False, is_active=active, full_name=full_name, short_name=short_name, last_login=now, date_joined=now, realm=realm, pointer=-1, is_bot=bool(bot_type), bot_type=bot_type, is_mirror_dummy=is_mirror_dummy, tos_version=tos_version, enable_stream_desktop_notifications=enable_stream_desktop_notifications, onboarding_steps=ujson.dumps([]), default_language=realm.default_language) if bot_owner is not None: # `user_profile.bot_owner = bot_owner` doesn't work on python 3.4 user_profile.bot_owner_id = bot_owner.id if bot_type or not active: password = None user_profile.set_password(password) user_profile.api_key = random_api_key() return user_profile
def users_to_zerver_userprofile(slack_data_dir: str, users: List[ZerverFieldsT], realm_id: int, timestamp: Any, domain_name: str) -> Tuple[List[ZerverFieldsT], List[ZerverFieldsT], AddedUsersT, List[ZerverFieldsT], List[ZerverFieldsT]]: """ Returns: 1. zerver_userprofile, which is a list of user profile 2. avatar_list, which is list to map avatars to zulip avatard records.json 3. added_users, which is a dictionary to map from slack user id to zulip user id 4. zerver_customprofilefield, which is a list of all custom profile fields 5. zerver_customprofilefield_values, which is a list of user profile fields """ logging.info('######### IMPORTING USERS STARTED #########\n') zerver_userprofile = [] zerver_customprofilefield = [] # type: List[ZerverFieldsT] zerver_customprofilefield_values = [] # type: List[ZerverFieldsT] avatar_list = [] # type: List[ZerverFieldsT] added_users = {} # The user data we get from the slack api does not contain custom profile data # Hence we get it from the slack zip file slack_data_file_user_list = get_data_file(slack_data_dir + '/users.json') # To map user id with the custom profile fields of the corresponding user slack_user_custom_field_map = {} # type: ZerverFieldsT # To store custom fields corresponding to their ids custom_field_map = {} # type: ZerverFieldsT for user in slack_data_file_user_list: process_slack_custom_fields(user, slack_user_custom_field_map) # We have only one primary owner in slack, see link # https://get.slack.help/hc/en-us/articles/201912948-Owners-and-Administrators # This is to import the primary owner first from all the users user_id_count = custom_field_id_count = customprofilefield_id = 0 primary_owner_id = user_id_count user_id_count += 1 for user in users: slack_user_id = user['id'] if user.get('is_primary_owner', False): user_id = primary_owner_id else: user_id = user_id_count # email email = get_user_email(user, domain_name) # avatar # ref: https://chat.zulip.org/help/change-your-avatar avatar_url = build_avatar_url(slack_user_id, user['team_id'], user['profile']['avatar_hash']) build_avatar(user_id, realm_id, email, avatar_url, timestamp, avatar_list) # check if user is the admin realm_admin = get_admin(user) # timezone timezone = get_user_timezone(user) # Check for custom profile fields if slack_user_id in slack_user_custom_field_map: # For processing the fields custom_field_map, customprofilefield_id = build_customprofile_field( zerver_customprofilefield, slack_user_custom_field_map[slack_user_id], customprofilefield_id, realm_id, custom_field_map) # Store the custom field values for the corresponding user custom_field_id_count = build_customprofilefields_values( custom_field_map, slack_user_custom_field_map[slack_user_id], user_id, custom_field_id_count, zerver_customprofilefield_values) userprofile = UserProfile( full_name=get_user_full_name(user), short_name=user['name'], is_active=not user['deleted'], id=user_id, email=email, delivery_email=email, avatar_source='U', is_bot=user.get('is_bot', False), pointer=-1, is_realm_admin=realm_admin, bot_type=1 if user.get('is_bot', False) else None, date_joined=timestamp, timezone=timezone, last_login=timestamp) userprofile_dict = model_to_dict(userprofile) # Set realm id separately as the corresponding realm is not yet a Realm model instance userprofile_dict['realm'] = realm_id zerver_userprofile.append(userprofile_dict) added_users[slack_user_id] = user_id if not user.get('is_primary_owner', False): user_id_count += 1 logging.info(u"{} -> {}".format(user['name'], userprofile_dict['email'])) process_customprofilefields(zerver_customprofilefield, zerver_customprofilefield_values) logging.info('######### IMPORTING USERS FINISHED #########\n') return zerver_userprofile, avatar_list, added_users, zerver_customprofilefield, \ zerver_customprofilefield_values
def do_import_realm(import_dir: Path, subdomain: str) -> Realm: logging.info("Importing realm dump %s" % (import_dir, )) if not os.path.exists(import_dir): raise Exception("Missing import directory!") realm_data_filename = os.path.join(import_dir, "realm.json") if not os.path.exists(realm_data_filename): raise Exception("Missing realm.json file!") logging.info("Importing realm data from %s" % (realm_data_filename, )) with open(realm_data_filename) as f: data = ujson.load(f) update_model_ids(Stream, data, 'zerver_stream', 'stream') re_map_foreign_keys(data, 'zerver_realm', 'notifications_stream', related_table="stream") fix_datetime_fields(data, 'zerver_realm') # Fix realm subdomain information data['zerver_realm'][0]['string_id'] = subdomain data['zerver_realm'][0]['name'] = subdomain fix_realm_authentication_bitfield(data, 'zerver_realm', 'authentication_methods') update_model_ids(Realm, data, 'zerver_realm', 'realm') realm = Realm(**data['zerver_realm'][0]) if realm.notifications_stream_id is not None: notifications_stream_id = int( realm.notifications_stream_id) # type: Optional[int] else: notifications_stream_id = None realm.notifications_stream_id = None realm.save() bulk_import_client(data, Client, 'zerver_client') # Email tokens will automatically be randomly generated when the # Stream objects are created by Django. fix_datetime_fields(data, 'zerver_stream') re_map_foreign_keys(data, 'zerver_stream', 'realm', related_table="realm") bulk_import_model(data, Stream, 'zerver_stream') realm.notifications_stream_id = notifications_stream_id realm.save() re_map_foreign_keys(data, 'zerver_defaultstream', 'stream', related_table="stream") re_map_foreign_keys(data, 'zerver_realmemoji', 'author', related_table="user_profile") for (table, model, related_table) in realm_tables: re_map_foreign_keys(data, table, 'realm', related_table="realm") update_model_ids(model, data, table, related_table) bulk_import_model(data, model, table) # Remap the user IDs for notification_bot and friends to their # appropriate IDs on this server for item in data['zerver_userprofile_crossrealm']: logging.info("Adding to ID map: %s %s" % (item['id'], get_system_bot(item['email']).id)) new_user_id = get_system_bot(item['email']).id update_id_map(table='user_profile', old_id=item['id'], new_id=new_user_id) # Merge in zerver_userprofile_mirrordummy data['zerver_userprofile'] = data['zerver_userprofile'] + data[ 'zerver_userprofile_mirrordummy'] del data['zerver_userprofile_mirrordummy'] data['zerver_userprofile'].sort(key=lambda r: r['id']) # To remap foreign key for UserProfile.last_active_message_id update_message_foreign_keys(import_dir) fix_datetime_fields(data, 'zerver_userprofile') update_model_ids(UserProfile, data, 'zerver_userprofile', 'user_profile') re_map_foreign_keys(data, 'zerver_userprofile', 'realm', related_table="realm") re_map_foreign_keys(data, 'zerver_userprofile', 'bot_owner', related_table="user_profile") re_map_foreign_keys(data, 'zerver_userprofile', 'default_sending_stream', related_table="stream") re_map_foreign_keys(data, 'zerver_userprofile', 'default_events_register_stream', related_table="stream") re_map_foreign_keys(data, 'zerver_userprofile', 'last_active_message_id', related_table="message", id_field=True) for user_profile_dict in data['zerver_userprofile']: user_profile_dict['password'] = None user_profile_dict['api_key'] = random_api_key() # Since Zulip doesn't use these permissions, drop them del user_profile_dict['user_permissions'] del user_profile_dict['groups'] user_profiles = [ UserProfile(**item) for item in data['zerver_userprofile'] ] for user_profile in user_profiles: user_profile.set_unusable_password() UserProfile.objects.bulk_create(user_profiles) if 'zerver_huddle' in data: bulk_import_model(data, Huddle, 'zerver_huddle') re_map_foreign_keys(data, 'zerver_recipient', 'type_id', related_table="stream", recipient_field=True, id_field=True) re_map_foreign_keys(data, 'zerver_recipient', 'type_id', related_table="user_profile", recipient_field=True, id_field=True) update_model_ids(Recipient, data, 'zerver_recipient', 'recipient') bulk_import_model(data, Recipient, 'zerver_recipient') re_map_foreign_keys(data, 'zerver_subscription', 'user_profile', related_table="user_profile") re_map_foreign_keys(data, 'zerver_subscription', 'recipient', related_table="recipient") update_model_ids(Subscription, data, 'zerver_subscription', 'subscription') bulk_import_model(data, Subscription, 'zerver_subscription') fix_datetime_fields(data, 'zerver_userpresence') re_map_foreign_keys(data, 'zerver_userpresence', 'user_profile', related_table="user_profile") re_map_foreign_keys(data, 'zerver_userpresence', 'client', related_table='client') update_model_ids(UserPresence, data, 'zerver_userpresence', 'user_presence') bulk_import_model(data, UserPresence, 'zerver_userpresence') fix_datetime_fields(data, 'zerver_useractivity') re_map_foreign_keys(data, 'zerver_useractivity', 'user_profile', related_table="user_profile") re_map_foreign_keys(data, 'zerver_useractivity', 'client', related_table='client') update_model_ids(UserActivity, data, 'zerver_useractivity', 'useractivity') bulk_import_model(data, UserActivity, 'zerver_useractivity') fix_datetime_fields(data, 'zerver_useractivityinterval') re_map_foreign_keys(data, 'zerver_useractivityinterval', 'user_profile', related_table="user_profile") update_model_ids(UserActivityInterval, data, 'zerver_useractivityinterval', 'useractivityinterval') bulk_import_model(data, UserActivityInterval, 'zerver_useractivityinterval') if 'zerver_customprofilefield' in data: # As the export of Custom Profile fields is not supported, Zulip exported # data would not contain this field. # However this is supported in slack importer script re_map_foreign_keys(data, 'zerver_customprofilefield', 'realm', related_table="realm") update_model_ids(CustomProfileField, data, 'zerver_customprofilefield', related_table="customprofilefield") bulk_import_model(data, CustomProfileField, 'zerver_customprofilefield') re_map_foreign_keys(data, 'zerver_customprofilefield_value', 'user_profile', related_table="user_profile") re_map_foreign_keys(data, 'zerver_customprofilefield_value', 'field', related_table="customprofilefield") update_model_ids(CustomProfileFieldValue, data, 'zerver_customprofilefield_value', related_table="customprofilefield_value") bulk_import_model(data, CustomProfileFieldValue, 'zerver_customprofilefield_value') # Import uploaded files and avatars import_uploads(os.path.join(import_dir, "avatars"), processing_avatars=True) import_uploads(os.path.join(import_dir, "uploads")) # We need to have this check as the emoji files are only present in the data # importer from slack # For Zulip export, this doesn't exist if os.path.exists(os.path.join(import_dir, "emoji")): import_uploads(os.path.join(import_dir, "emoji"), processing_emojis=True) # Import zerver_message and zerver_usermessage import_message_data(import_dir) # Do attachments AFTER message data is loaded. # TODO: de-dup how we read these json files. fn = os.path.join(import_dir, "attachment.json") if not os.path.exists(fn): raise Exception("Missing attachment.json file!") logging.info("Importing attachment data from %s" % (fn, )) with open(fn) as f: data = ujson.load(f) import_attachments(data) return realm
def users_to_zerver_userprofile( slack_data_dir: str, users: List[ZerverFieldsT], realm_id: int, timestamp: Any, domain_name: str ) -> Tuple[List[ZerverFieldsT], List[ZerverFieldsT], SlackToZulipUserIDT, List[ZerverFieldsT], List[ZerverFieldsT]]: """ Returns: 1. zerver_userprofile, which is a list of user profile 2. avatar_list, which is list to map avatars to zulip avatard records.json 3. slack_user_id_to_zulip_user_id, which is a dictionary to map from slack user id to zulip user id 4. zerver_customprofilefield, which is a list of all custom profile fields 5. zerver_customprofilefield_values, which is a list of user profile fields """ logging.info('######### IMPORTING USERS STARTED #########\n') zerver_userprofile = [] zerver_customprofilefield = [] # type: List[ZerverFieldsT] zerver_customprofilefield_values = [] # type: List[ZerverFieldsT] avatar_list = [] # type: List[ZerverFieldsT] slack_user_id_to_zulip_user_id = {} # The user data we get from the slack api does not contain custom profile data # Hence we get it from the slack zip file slack_data_file_user_list = get_data_file(slack_data_dir + '/users.json') slack_user_id_to_custom_profile_fields = {} # type: ZerverFieldsT slack_custom_field_name_to_zulip_custom_field_id = { } # type: ZerverFieldsT for user in slack_data_file_user_list: process_slack_custom_fields(user, slack_user_id_to_custom_profile_fields) # We have only one primary owner in slack, see link # https://get.slack.help/hc/en-us/articles/201912948-Owners-and-Administrators # This is to import the primary owner first from all the users user_id_count = custom_profile_field_value_id_count = custom_profile_field_id_count = 0 primary_owner_id = user_id_count user_id_count += 1 for user in users: slack_user_id = user['id'] if user.get('is_primary_owner', False): user_id = primary_owner_id else: user_id = user_id_count email = get_user_email(user, domain_name) # ref: https://chat.zulip.org/help/set-your-profile-picture avatar_url = build_avatar_url(slack_user_id, user['team_id'], user['profile']['avatar_hash']) build_avatar(user_id, realm_id, email, avatar_url, timestamp, avatar_list) role = UserProfile.ROLE_MEMBER if get_admin(user): role = UserProfile.ROLE_REALM_ADMINISTRATOR timezone = get_user_timezone(user) if slack_user_id in slack_user_id_to_custom_profile_fields: slack_custom_field_name_to_zulip_custom_field_id, custom_profile_field_id_count = \ build_customprofile_field(zerver_customprofilefield, slack_user_id_to_custom_profile_fields[slack_user_id], custom_profile_field_id_count, realm_id, slack_custom_field_name_to_zulip_custom_field_id) custom_profile_field_value_id_count = build_customprofilefields_values( slack_custom_field_name_to_zulip_custom_field_id, slack_user_id_to_custom_profile_fields[slack_user_id], user_id, custom_profile_field_value_id_count, zerver_customprofilefield_values) userprofile = UserProfile( full_name=get_user_full_name(user), short_name=user['name'], is_active=not user.get('deleted', False) and not user["is_mirror_dummy"], is_mirror_dummy=user["is_mirror_dummy"], id=user_id, email=email, delivery_email=email, avatar_source='U', is_bot=user.get('is_bot', False), pointer=-1, role=role, bot_type=1 if user.get('is_bot', False) else None, date_joined=timestamp, timezone=timezone, last_login=timestamp) userprofile_dict = model_to_dict(userprofile) # Set realm id separately as the corresponding realm is not yet a Realm model instance userprofile_dict['realm'] = realm_id zerver_userprofile.append(userprofile_dict) slack_user_id_to_zulip_user_id[slack_user_id] = user_id if not user.get('is_primary_owner', False): user_id_count += 1 logging.info(u"{} -> {}".format(user['name'], userprofile_dict['email'])) process_customprofilefields(zerver_customprofilefield, zerver_customprofilefield_values) logging.info('######### IMPORTING USERS FINISHED #########\n') return zerver_userprofile, avatar_list, slack_user_id_to_zulip_user_id, zerver_customprofilefield, \ zerver_customprofilefield_values
def do_import_realm(import_dir: Path, subdomain: str) -> Realm: logging.info("Importing realm dump %s" % (import_dir,)) if not os.path.exists(import_dir): raise Exception("Missing import directory!") realm_data_filename = os.path.join(import_dir, "realm.json") if not os.path.exists(realm_data_filename): raise Exception("Missing realm.json file!") logging.info("Importing realm data from %s" % (realm_data_filename,)) with open(realm_data_filename) as f: data = ujson.load(f) bulk_import_client(data, Client, 'zerver_client') # We don't import the Stream model yet, since it depends on Realm, # which isn't imported yet. But we need the Stream model IDs for # notifications_stream. update_model_ids(Stream, data, 'stream') re_map_foreign_keys(data, 'zerver_realm', 'notifications_stream', related_table="stream") re_map_foreign_keys(data, 'zerver_realm', 'signup_notifications_stream', related_table="stream") fix_datetime_fields(data, 'zerver_realm') # Fix realm subdomain information data['zerver_realm'][0]['string_id'] = subdomain data['zerver_realm'][0]['name'] = subdomain fix_realm_authentication_bitfield(data, 'zerver_realm', 'authentication_methods') update_model_ids(Realm, data, 'realm') realm = Realm(**data['zerver_realm'][0]) if settings.BILLING_ENABLED: realm.plan_type = Realm.LIMITED else: realm.plan_type = Realm.SELF_HOSTED if realm.notifications_stream_id is not None: notifications_stream_id = int(realm.notifications_stream_id) # type: Optional[int] else: notifications_stream_id = None realm.notifications_stream_id = None if realm.signup_notifications_stream_id is not None: signup_notifications_stream_id = int(realm.signup_notifications_stream_id) # type: Optional[int] else: signup_notifications_stream_id = None realm.signup_notifications_stream_id = None realm.save() # Email tokens will automatically be randomly generated when the # Stream objects are created by Django. fix_datetime_fields(data, 'zerver_stream') re_map_foreign_keys(data, 'zerver_stream', 'realm', related_table="realm") bulk_import_model(data, Stream) realm.notifications_stream_id = notifications_stream_id realm.signup_notifications_stream_id = signup_notifications_stream_id realm.save() # Remap the user IDs for notification_bot and friends to their # appropriate IDs on this server for item in data['zerver_userprofile_crossrealm']: logging.info("Adding to ID map: %s %s" % (item['id'], get_system_bot(item['email']).id)) new_user_id = get_system_bot(item['email']).id update_id_map(table='user_profile', old_id=item['id'], new_id=new_user_id) new_recipient_id = Recipient.objects.get(type=Recipient.PERSONAL, type_id=new_user_id).id update_id_map(table='recipient', old_id=item['recipient_id'], new_id=new_recipient_id) # Merge in zerver_userprofile_mirrordummy data['zerver_userprofile'] = data['zerver_userprofile'] + data['zerver_userprofile_mirrordummy'] del data['zerver_userprofile_mirrordummy'] data['zerver_userprofile'].sort(key=lambda r: r['id']) # To remap foreign key for UserProfile.last_active_message_id update_message_foreign_keys(import_dir) fix_datetime_fields(data, 'zerver_userprofile') update_model_ids(UserProfile, data, 'user_profile') re_map_foreign_keys(data, 'zerver_userprofile', 'realm', related_table="realm") re_map_foreign_keys(data, 'zerver_userprofile', 'bot_owner', related_table="user_profile") re_map_foreign_keys(data, 'zerver_userprofile', 'default_sending_stream', related_table="stream") re_map_foreign_keys(data, 'zerver_userprofile', 'default_events_register_stream', related_table="stream") re_map_foreign_keys(data, 'zerver_userprofile', 'last_active_message_id', related_table="message", id_field=True) for user_profile_dict in data['zerver_userprofile']: user_profile_dict['password'] = None user_profile_dict['api_key'] = generate_api_key() # Since Zulip doesn't use these permissions, drop them del user_profile_dict['user_permissions'] del user_profile_dict['groups'] user_profiles = [UserProfile(**item) for item in data['zerver_userprofile']] for user_profile in user_profiles: user_profile.set_unusable_password() UserProfile.objects.bulk_create(user_profiles) re_map_foreign_keys(data, 'zerver_defaultstream', 'stream', related_table="stream") re_map_foreign_keys(data, 'zerver_realmemoji', 'author', related_table="user_profile") for (table, model, related_table) in realm_tables: re_map_foreign_keys(data, table, 'realm', related_table="realm") update_model_ids(model, data, related_table) bulk_import_model(data, model) if 'zerver_huddle' in data: update_model_ids(Huddle, data, 'huddle') # We don't import Huddle yet, since we don't have the data to # compute huddle hashes until we've imported some of the # tables below. # TODO: double-check this. re_map_foreign_keys(data, 'zerver_recipient', 'type_id', related_table="stream", recipient_field=True, id_field=True) re_map_foreign_keys(data, 'zerver_recipient', 'type_id', related_table="user_profile", recipient_field=True, id_field=True) re_map_foreign_keys(data, 'zerver_recipient', 'type_id', related_table="huddle", recipient_field=True, id_field=True) update_model_ids(Recipient, data, 'recipient') bulk_import_model(data, Recipient) re_map_foreign_keys(data, 'zerver_subscription', 'user_profile', related_table="user_profile") get_huddles_from_subscription(data, 'zerver_subscription') re_map_foreign_keys(data, 'zerver_subscription', 'recipient', related_table="recipient") update_model_ids(Subscription, data, 'subscription') bulk_import_model(data, Subscription) if 'zerver_realmauditlog' in data: fix_datetime_fields(data, 'zerver_realmauditlog') re_map_foreign_keys(data, 'zerver_realmauditlog', 'realm', related_table="realm") re_map_foreign_keys(data, 'zerver_realmauditlog', 'modified_user', related_table='user_profile') re_map_foreign_keys(data, 'zerver_realmauditlog', 'acting_user', related_table='user_profile') re_map_foreign_keys(data, 'zerver_realmauditlog', 'modified_stream', related_table="stream") update_model_ids(RealmAuditLog, data, related_table="realmauditlog") bulk_import_model(data, RealmAuditLog) else: logging.info('about to call create_subscription_events') create_subscription_events( data=data, realm_id=realm.id, ) logging.info('done with create_subscription_events') if 'zerver_huddle' in data: process_huddle_hash(data, 'zerver_huddle') bulk_import_model(data, Huddle) if 'zerver_userhotspot' in data: fix_datetime_fields(data, 'zerver_userhotspot') re_map_foreign_keys(data, 'zerver_userhotspot', 'user', related_table='user_profile') update_model_ids(UserHotspot, data, 'userhotspot') bulk_import_model(data, UserHotspot) if 'zerver_mutedtopic' in data: re_map_foreign_keys(data, 'zerver_mutedtopic', 'user_profile', related_table='user_profile') re_map_foreign_keys(data, 'zerver_mutedtopic', 'stream', related_table='stream') re_map_foreign_keys(data, 'zerver_mutedtopic', 'recipient', related_table='recipient') update_model_ids(MutedTopic, data, 'mutedtopic') bulk_import_model(data, MutedTopic) if 'zerver_service' in data: re_map_foreign_keys(data, 'zerver_service', 'user_profile', related_table='user_profile') fix_service_tokens(data, 'zerver_service') update_model_ids(Service, data, 'service') bulk_import_model(data, Service) if 'zerver_usergroup' in data: re_map_foreign_keys(data, 'zerver_usergroup', 'realm', related_table='realm') re_map_foreign_keys_many_to_many(data, 'zerver_usergroup', 'members', related_table='user_profile') update_model_ids(UserGroup, data, 'usergroup') bulk_import_model(data, UserGroup) re_map_foreign_keys(data, 'zerver_usergroupmembership', 'user_group', related_table='usergroup') re_map_foreign_keys(data, 'zerver_usergroupmembership', 'user_profile', related_table='user_profile') update_model_ids(UserGroupMembership, data, 'usergroupmembership') bulk_import_model(data, UserGroupMembership) if 'zerver_botstoragedata' in data: re_map_foreign_keys(data, 'zerver_botstoragedata', 'bot_profile', related_table='user_profile') update_model_ids(BotStorageData, data, 'botstoragedata') bulk_import_model(data, BotStorageData) if 'zerver_botconfigdata' in data: re_map_foreign_keys(data, 'zerver_botconfigdata', 'bot_profile', related_table='user_profile') update_model_ids(BotConfigData, data, 'botconfigdata') bulk_import_model(data, BotConfigData) fix_datetime_fields(data, 'zerver_userpresence') re_map_foreign_keys(data, 'zerver_userpresence', 'user_profile', related_table="user_profile") re_map_foreign_keys(data, 'zerver_userpresence', 'client', related_table='client') update_model_ids(UserPresence, data, 'user_presence') bulk_import_model(data, UserPresence) fix_datetime_fields(data, 'zerver_useractivity') re_map_foreign_keys(data, 'zerver_useractivity', 'user_profile', related_table="user_profile") re_map_foreign_keys(data, 'zerver_useractivity', 'client', related_table='client') update_model_ids(UserActivity, data, 'useractivity') bulk_import_model(data, UserActivity) fix_datetime_fields(data, 'zerver_useractivityinterval') re_map_foreign_keys(data, 'zerver_useractivityinterval', 'user_profile', related_table="user_profile") update_model_ids(UserActivityInterval, data, 'useractivityinterval') bulk_import_model(data, UserActivityInterval) re_map_foreign_keys(data, 'zerver_customprofilefield', 'realm', related_table="realm") update_model_ids(CustomProfileField, data, related_table="customprofilefield") bulk_import_model(data, CustomProfileField) re_map_foreign_keys(data, 'zerver_customprofilefieldvalue', 'user_profile', related_table="user_profile") re_map_foreign_keys(data, 'zerver_customprofilefieldvalue', 'field', related_table="customprofilefield") fix_customprofilefield(data) update_model_ids(CustomProfileFieldValue, data, related_table="customprofilefieldvalue") bulk_import_model(data, CustomProfileFieldValue) # Import uploaded files and avatars import_uploads(os.path.join(import_dir, "avatars"), processing_avatars=True) import_uploads(os.path.join(import_dir, "uploads")) # We need to have this check as the emoji files are only present in the data # importer from slack # For Zulip export, this doesn't exist if os.path.exists(os.path.join(import_dir, "emoji")): import_uploads(os.path.join(import_dir, "emoji"), processing_emojis=True) # Import zerver_message and zerver_usermessage import_message_data(import_dir) re_map_foreign_keys(data, 'zerver_reaction', 'message', related_table="message") re_map_foreign_keys(data, 'zerver_reaction', 'user_profile', related_table="user_profile") re_map_foreign_keys(data, 'zerver_reaction', 'emoji_code', related_table="realmemoji", id_field=True, reaction_field=True) update_model_ids(Reaction, data, 'reaction') bulk_import_model(data, Reaction) # Do attachments AFTER message data is loaded. # TODO: de-dup how we read these json files. fn = os.path.join(import_dir, "attachment.json") if not os.path.exists(fn): raise Exception("Missing attachment.json file!") logging.info("Importing attachment data from %s" % (fn,)) with open(fn) as f: data = ujson.load(f) import_attachments(data) return realm
def users_to_zerver_userprofile( slack_data_dir: str, users: List[ZerverFieldsT], realm_id: int, timestamp: Any, domain_name: str ) -> Tuple[ List[ZerverFieldsT], List[ZerverFieldsT], SlackToZulipUserIDT, List[ZerverFieldsT], List[ZerverFieldsT], ]: """ Returns: 1. zerver_userprofile, which is a list of user profile 2. avatar_list, which is list to map avatars to Zulip avatard records.json 3. slack_user_id_to_zulip_user_id, which is a dictionary to map from Slack user ID to Zulip user id 4. zerver_customprofilefield, which is a list of all custom profile fields 5. zerver_customprofilefield_values, which is a list of user profile fields """ logging.info("######### IMPORTING USERS STARTED #########\n") zerver_userprofile = [] zerver_customprofilefield: List[ZerverFieldsT] = [] zerver_customprofilefield_values: List[ZerverFieldsT] = [] avatar_list: List[ZerverFieldsT] = [] slack_user_id_to_zulip_user_id = {} # The user data we get from the Slack API does not contain custom profile data # Hence we get it from the Slack zip file slack_data_file_user_list = get_data_file(slack_data_dir + "/users.json") slack_user_id_to_custom_profile_fields: ZerverFieldsT = {} slack_custom_field_name_to_zulip_custom_field_id: ZerverFieldsT = {} for user in slack_data_file_user_list: process_slack_custom_fields(user, slack_user_id_to_custom_profile_fields) # We have only one primary owner in Slack, see link # https://get.slack.help/hc/en-us/articles/201912948-Owners-and-Administrators # This is to import the primary owner first from all the users user_id_count = custom_profile_field_value_id_count = custom_profile_field_id_count = 0 primary_owner_id = user_id_count user_id_count += 1 for user in users: slack_user_id = user["id"] if user.get("is_primary_owner", False): user_id = primary_owner_id else: user_id = user_id_count email = get_user_email(user, domain_name) # ref: https://zulip.com/help/change-your-profile-picture avatar_url = build_avatar_url( slack_user_id, user["team_id"], user["profile"]["avatar_hash"] ) build_avatar(user_id, realm_id, email, avatar_url, timestamp, avatar_list) role = UserProfile.ROLE_MEMBER if get_owner(user): role = UserProfile.ROLE_REALM_OWNER elif get_admin(user): role = UserProfile.ROLE_REALM_ADMINISTRATOR if get_guest(user): role = UserProfile.ROLE_GUEST timezone = get_user_timezone(user) if slack_user_id in slack_user_id_to_custom_profile_fields: ( slack_custom_field_name_to_zulip_custom_field_id, custom_profile_field_id_count, ) = build_customprofile_field( zerver_customprofilefield, slack_user_id_to_custom_profile_fields[slack_user_id], custom_profile_field_id_count, realm_id, slack_custom_field_name_to_zulip_custom_field_id, ) custom_profile_field_value_id_count = build_customprofilefields_values( slack_custom_field_name_to_zulip_custom_field_id, slack_user_id_to_custom_profile_fields[slack_user_id], user_id, custom_profile_field_value_id_count, zerver_customprofilefield_values, ) userprofile = UserProfile( full_name=get_user_full_name(user), is_active=not user.get("deleted", False) and not user["is_mirror_dummy"], is_mirror_dummy=user["is_mirror_dummy"], id=user_id, email=email, delivery_email=email, avatar_source="U", is_bot=user.get("is_bot", False), role=role, bot_type=1 if user.get("is_bot", False) else None, date_joined=timestamp, timezone=timezone, last_login=timestamp, ) userprofile_dict = model_to_dict(userprofile) # Set realm id separately as the corresponding realm is not yet a Realm model instance userprofile_dict["realm"] = realm_id zerver_userprofile.append(userprofile_dict) slack_user_id_to_zulip_user_id[slack_user_id] = user_id if not user.get("is_primary_owner", False): user_id_count += 1 logging.info("%s -> %s", user["name"], userprofile_dict["email"]) process_customprofilefields(zerver_customprofilefield, zerver_customprofilefield_values) logging.info("######### IMPORTING USERS FINISHED #########\n") return ( zerver_userprofile, avatar_list, slack_user_id_to_zulip_user_id, zerver_customprofilefield, zerver_customprofilefield_values, )
def fetch_initial_state_data( user_profile: Optional[UserProfile], *, realm: Optional[Realm] = None, event_types: Optional[Iterable[str]] = None, queue_id: Optional[str] = "", client_gravatar: bool = False, user_avatar_url_field_optional: bool = False, slim_presence: bool = False, include_subscribers: bool = True, include_streams: bool = True, ) -> Dict[str, Any]: """When `event_types` is None, fetches the core data powering the webapp's `page_params` and `/api/v1/register` (for mobile/terminal apps). Can also fetch a subset as determined by `event_types`. The user_profile=None code path is used for logged-out public access to streams with is_web_public=True. Whenever you add new code to this function, you should also add corresponding events for changes in the data structures and new code to apply_events (and add a test in test_events.py). """ if realm is None: assert user_profile is not None realm = user_profile.realm state: Dict[str, Any] = {'queue_id': queue_id} if event_types is None: # return True always want: Callable[[str], bool] = always_want else: want = set(event_types).__contains__ # Show the version info unconditionally. state['zulip_version'] = ZULIP_VERSION state['zulip_feature_level'] = API_FEATURE_LEVEL if want('alert_words'): state['alert_words'] = [] if user_profile is None else user_alert_words(user_profile) if want('custom_profile_fields'): fields = custom_profile_fields_for_realm(realm.id) state['custom_profile_fields'] = [f.as_dict() for f in fields] state['custom_profile_field_types'] = { item[4]: {"id": item[0], "name": str(item[1])} for item in CustomProfileField.ALL_FIELD_TYPES } if want('hotspots'): # Even if we offered special hotspots for guests without an # account, we'd maybe need to store their state using cookies # or local storage, rather than in the database. state['hotspots'] = [] if user_profile is None else get_next_hotspots(user_profile) if want('message'): # Since the introduction of `anchor="latest"` in the API, # `max_message_id` is primarily used for generating `local_id` # values that are higher than this. We likely can eventually # remove this parameter from the API. user_messages = [] if user_profile is not None: user_messages = UserMessage.objects \ .filter(user_profile=user_profile) \ .order_by('-message_id') \ .values('message_id')[:1] if user_messages: state['max_message_id'] = user_messages[0]['message_id'] else: state['max_message_id'] = -1 if want('muted_topics'): state['muted_topics'] = [] if user_profile is None else get_topic_mutes(user_profile) if want('presence'): state['presences'] = {} if user_profile is None else get_presences_for_realm(realm, slim_presence) if want('realm'): for property_name in Realm.property_types: state['realm_' + property_name] = getattr(realm, property_name) # Most state is handled via the property_types framework; # these manual entries are for those realm settings that don't # fit into that framework. state['realm_authentication_methods'] = realm.authentication_methods_dict() # We pretend these features are disabled because guests can't # access them. In the future, we may want to move this logic # to the frontends, so that we can correctly display what # these fields are in the settings. state['realm_allow_message_editing'] = False if user_profile is None else realm.allow_message_editing state['realm_allow_community_topic_editing'] = False if user_profile is None else realm.allow_community_topic_editing state['realm_allow_message_deleting'] = False if user_profile is None else realm.allow_message_deleting state['realm_message_content_edit_limit_seconds'] = realm.message_content_edit_limit_seconds state['realm_message_content_delete_limit_seconds'] = realm.message_content_delete_limit_seconds state['realm_community_topic_editing_limit_seconds'] = \ Realm.DEFAULT_COMMUNITY_TOPIC_EDITING_LIMIT_SECONDS # This setting determines whether to send presence and also # whether to display of users list in the right sidebar; we # want both behaviors for logged-out users. We may in the # future choose to move this logic to the frontend. state['realm_presence_disabled'] = True if user_profile is None else realm.presence_disabled state['realm_icon_url'] = realm_icon_url(realm) state['realm_icon_source'] = realm.icon_source state['max_icon_file_size'] = settings.MAX_ICON_FILE_SIZE add_realm_logo_fields(state, realm) state['realm_bot_domain'] = realm.get_bot_domain() state['realm_uri'] = realm.uri state['realm_available_video_chat_providers'] = realm.VIDEO_CHAT_PROVIDERS state['settings_send_digest_emails'] = settings.SEND_DIGEST_EMAILS state['realm_digest_emails_enabled'] = realm.digest_emails_enabled and settings.SEND_DIGEST_EMAILS state['realm_is_zephyr_mirror_realm'] = realm.is_zephyr_mirror_realm state['realm_email_auth_enabled'] = email_auth_enabled(realm) state['realm_password_auth_enabled'] = password_auth_enabled(realm) state['realm_push_notifications_enabled'] = push_notifications_enabled() state['realm_upload_quota'] = realm.upload_quota_bytes() state['realm_plan_type'] = realm.plan_type state['zulip_plan_is_not_limited'] = realm.plan_type != Realm.LIMITED state['upgrade_text_for_wide_organization_logo'] = str(Realm.UPGRADE_TEXT_STANDARD) state['realm_default_external_accounts'] = DEFAULT_EXTERNAL_ACCOUNTS state['jitsi_server_url'] = settings.JITSI_SERVER_URL.rstrip('/') state['development_environment'] = settings.DEVELOPMENT state['server_generation'] = settings.SERVER_GENERATION state['password_min_length'] = settings.PASSWORD_MIN_LENGTH state['password_min_guesses'] = settings.PASSWORD_MIN_GUESSES state['max_file_upload_size_mib'] = settings.MAX_FILE_UPLOAD_SIZE state['max_avatar_file_size_mib'] = settings.MAX_AVATAR_FILE_SIZE state['server_inline_image_preview'] = settings.INLINE_IMAGE_PREVIEW state['server_inline_url_embed_preview'] = settings.INLINE_URL_EMBED_PREVIEW state['server_avatar_changes_disabled'] = settings.AVATAR_CHANGES_DISABLED state['server_name_changes_disabled'] = settings.NAME_CHANGES_DISABLED if realm.notifications_stream and not realm.notifications_stream.deactivated: notifications_stream = realm.notifications_stream state['realm_notifications_stream_id'] = notifications_stream.id else: state['realm_notifications_stream_id'] = -1 signup_notifications_stream = realm.get_signup_notifications_stream() if signup_notifications_stream: state['realm_signup_notifications_stream_id'] = signup_notifications_stream.id else: state['realm_signup_notifications_stream_id'] = -1 if want('realm_domains'): state['realm_domains'] = get_realm_domains(realm) if want('realm_emoji'): state['realm_emoji'] = realm.get_emoji() if want('realm_filters'): state['realm_filters'] = realm_filters_for_realm(realm.id) if want('realm_user_groups'): state['realm_user_groups'] = user_groups_in_realm_serialized(realm) if user_profile is not None: settings_user = user_profile else: # When UserProfile=None, we want to serve the values for various # settings as the defaults. Instead of copying the default values # from models.py here, we access these default values from a # temporary UserProfile object that will not be saved to the database. # # We also can set various fields to avoid duplicating code # unnecessarily. settings_user = UserProfile( full_name="Anonymous User", email="*****@*****.**", delivery_email="*****@*****.**", realm=realm, # We tag logged-out users as guests because most guest # restrictions apply to these users as well, and it lets # us avoid unnecessary conditionals. role=UserProfile.ROLE_GUEST, avatar_source=UserProfile.AVATAR_FROM_GRAVATAR, # ID=0 is not used in real Zulip databases, ensuring this is unique. id=0, ) if want('realm_user'): state['raw_users'] = get_raw_user_data(realm, user_profile, client_gravatar=client_gravatar, user_avatar_url_field_optional=user_avatar_url_field_optional) state['cross_realm_bots'] = list(get_cross_realm_dicts()) # For the user's own avatar URL, we force # client_gravatar=False, since that saves some unnecessary # client-side code for handing medium-size avatars. See #8253 # for details. state['avatar_source'] = settings_user.avatar_source state['avatar_url_medium'] = avatar_url( settings_user, medium=True, client_gravatar=False, ) state['avatar_url'] = avatar_url( settings_user, medium=False, client_gravatar=False, ) state['can_create_streams'] = settings_user.can_create_streams() state['can_subscribe_other_users'] = settings_user.can_subscribe_other_users() state['is_admin'] = settings_user.is_realm_admin state['is_owner'] = settings_user.is_realm_owner state['is_guest'] = settings_user.is_guest state['user_id'] = settings_user.id state['enter_sends'] = settings_user.enter_sends state['email'] = settings_user.email state['delivery_email'] = settings_user.delivery_email state['full_name'] = settings_user.full_name if want('realm_bot'): state['realm_bots'] = [] if user_profile is None else get_owned_bot_dicts(user_profile) # This does not yet have an apply_event counterpart, since currently, # new entries for EMBEDDED_BOTS can only be added directly in the codebase. if want('realm_embedded_bots'): realm_embedded_bots = [] for bot in EMBEDDED_BOTS: realm_embedded_bots.append({'name': bot.name, 'config': load_bot_config_template(bot.name)}) state['realm_embedded_bots'] = realm_embedded_bots # This does not have an apply_events counterpart either since # this data is mostly static. if want('realm_incoming_webhook_bots'): realm_incoming_webhook_bots = [] for integration in WEBHOOK_INTEGRATIONS: realm_incoming_webhook_bots.append({ 'name': integration.name, 'config': {c[1]: c[0] for c in integration.config_options}, }) state['realm_incoming_webhook_bots'] = realm_incoming_webhook_bots if want('recent_private_conversations'): # A data structure containing records of this form: # # [{'max_message_id': 700175, 'user_ids': [801]}] # # for all recent private message conversations, ordered by the # highest message ID in the conversation. The user_ids list # is the list of users other than the current user in the # private message conversation (so it is [] for PMs to self). # Note that raw_recent_private_conversations is an # intermediate form as a dictionary keyed by recipient_id, # which is more efficient to update, and is rewritten to the # final format in post_process_state. state['raw_recent_private_conversations'] = {} if user_profile is None else get_recent_private_conversations(user_profile) if want('subscription'): if user_profile is not None: sub_info = gather_subscriptions_helper( user_profile, include_subscribers=include_subscribers, ) else: sub_info = get_web_public_subs(realm) state['subscriptions'] = sub_info.subscriptions state['unsubscribed'] = sub_info.unsubscribed state['never_subscribed'] = sub_info.never_subscribed if want('update_message_flags') and want('message'): # Keeping unread_msgs updated requires both message flag updates and # message updates. This is due to the fact that new messages will not # generate a flag update so we need to use the flags field in the # message event. if user_profile is not None: state['raw_unread_msgs'] = get_raw_unread_data(user_profile) else: # For logged-out visitors, we treat all messages as read; # calling this helper lets us return empty objects in the # appropriate format. state['raw_unread_msgs'] = extract_unread_data_from_um_rows([], user_profile) if want('starred_messages'): state['starred_messages'] = [] if user_profile is None else get_starred_message_ids(user_profile) if want('stream'): if include_streams: # The webapp doesn't use the data from here; instead, # it uses data from state["subscriptions"] and other # places. if user_profile is not None: state['streams'] = do_get_streams(user_profile) else: # TODO: This line isn't used by the webapp because it # gets these data via the `subscriptions` key; it will # be used when the mobile apps support logged-out # access. state['streams'] = get_web_public_streams(realm) # nocoverage state['stream_name_max_length'] = Stream.MAX_NAME_LENGTH state['stream_description_max_length'] = Stream.MAX_DESCRIPTION_LENGTH if want('default_streams'): if settings_user.is_guest: # Guest users and logged-out users don't have access to # all default streams, so we pretend the organization # doesn't have any. state['realm_default_streams'] = [] else: state['realm_default_streams'] = streams_to_dicts_sorted( get_default_streams_for_realm(realm.id)) if want('default_stream_groups'): if settings_user.is_guest: state['realm_default_stream_groups'] = [] else: state['realm_default_stream_groups'] = default_stream_groups_to_dicts_sorted( get_default_stream_groups(realm)) if want('stop_words'): state['stop_words'] = read_stop_words() if want('update_display_settings'): for prop in UserProfile.property_types: state[prop] = getattr(settings_user, prop) state['emojiset_choices'] = UserProfile.emojiset_choices() if want('update_global_notifications'): for notification in UserProfile.notification_setting_types: state[notification] = getattr(settings_user, notification) state['available_notification_sounds'] = get_available_notification_sounds() if want('user_status'): # We require creating an account to access statuses. state['user_status'] = {} if user_profile is None else get_user_info_dict(realm_id=realm.id) if want('video_calls'): state['has_zoom_token'] = settings_user.zoom_token is not None return state
def get_or_create_user(self, username, ldap_user): try: return get_user_profile_by_email(username), False except UserProfile.DoesNotExist: return UserProfile(), False