def fix_service_tokens(data: TableData, table: TableName) -> None: """ The tokens in the services are created by 'random_api_key'. As the tokens are unique, they should be re-created for the imports. """ for item in data[table]: item['token'] = random_api_key()
def handle(self, *args, **options): # type: (*Any, **Any) -> None if None in (options["api"], options["web"], options["domain"]): print("\033[1;31mYou must provide a domain, an API URL, and a web URL.\033[0m\n", file=sys.stderr) self.print_help("python manage.py", "create_realm") exit(1) if not options["no_realm"]: CreateRealm().handle(*args, **options) print() # Newline realm = get_realm(options["domain"]) if realm is None: print("\033[1;31mRealm does not exist!\033[0m\n", file=sys.stderr) exit(2) dep = Deployment() dep.api_key = random_api_key() dep.save() old_dep = realm.deployment if old_dep is not None: old_dep.realms.remove(realm) old_dep.save() dep.realms = [realm] dep.base_api_url = options["api"] dep.base_site_url = options["web"] dep.save() print("Deployment %s created." % (dep.id,)) print("DEPLOYMENT_ROLE_NAME = %s" % (dep.name,)) print("DEPLOYMENT_ROLE_KEY = %s" % (dep.api_key,))
def do_import_realm(import_dir: Path, subdomain: str) -> Realm: logging.info("Importing realm dump %s" % (import_dir, )) if not os.path.exists(import_dir): raise Exception("Missing import directory!") realm_data_filename = os.path.join(import_dir, "realm.json") if not os.path.exists(realm_data_filename): raise Exception("Missing realm.json file!") logging.info("Importing realm data from %s" % (realm_data_filename, )) with open(realm_data_filename) as f: data = ujson.load(f) update_model_ids(Stream, data, 'zerver_stream', 'stream') re_map_foreign_keys(data, 'zerver_realm', 'notifications_stream', related_table="stream") fix_datetime_fields(data, 'zerver_realm') # Fix realm subdomain information data['zerver_realm'][0]['string_id'] = subdomain data['zerver_realm'][0]['name'] = subdomain fix_realm_authentication_bitfield(data, 'zerver_realm', 'authentication_methods') update_model_ids(Realm, data, 'zerver_realm', 'realm') realm = Realm(**data['zerver_realm'][0]) if realm.notifications_stream_id is not None: notifications_stream_id = int( realm.notifications_stream_id) # type: Optional[int] else: notifications_stream_id = None realm.notifications_stream_id = None realm.save() bulk_import_client(data, Client, 'zerver_client') # Email tokens will automatically be randomly generated when the # Stream objects are created by Django. fix_datetime_fields(data, 'zerver_stream') re_map_foreign_keys(data, 'zerver_stream', 'realm', related_table="realm") bulk_import_model(data, Stream, 'zerver_stream') realm.notifications_stream_id = notifications_stream_id realm.save() re_map_foreign_keys(data, 'zerver_defaultstream', 'stream', related_table="stream") re_map_foreign_keys(data, 'zerver_realmemoji', 'author', related_table="user_profile") for (table, model, related_table) in realm_tables: re_map_foreign_keys(data, table, 'realm', related_table="realm") update_model_ids(model, data, table, related_table) bulk_import_model(data, model, table) # Remap the user IDs for notification_bot and friends to their # appropriate IDs on this server for item in data['zerver_userprofile_crossrealm']: logging.info("Adding to ID map: %s %s" % (item['id'], get_system_bot(item['email']).id)) new_user_id = get_system_bot(item['email']).id update_id_map(table='user_profile', old_id=item['id'], new_id=new_user_id) # Merge in zerver_userprofile_mirrordummy data['zerver_userprofile'] = data['zerver_userprofile'] + data[ 'zerver_userprofile_mirrordummy'] del data['zerver_userprofile_mirrordummy'] data['zerver_userprofile'].sort(key=lambda r: r['id']) # To remap foreign key for UserProfile.last_active_message_id update_message_foreign_keys(import_dir) fix_datetime_fields(data, 'zerver_userprofile') update_model_ids(UserProfile, data, 'zerver_userprofile', 'user_profile') re_map_foreign_keys(data, 'zerver_userprofile', 'realm', related_table="realm") re_map_foreign_keys(data, 'zerver_userprofile', 'bot_owner', related_table="user_profile") re_map_foreign_keys(data, 'zerver_userprofile', 'default_sending_stream', related_table="stream") re_map_foreign_keys(data, 'zerver_userprofile', 'default_events_register_stream', related_table="stream") re_map_foreign_keys(data, 'zerver_userprofile', 'last_active_message_id', related_table="message", id_field=True) for user_profile_dict in data['zerver_userprofile']: user_profile_dict['password'] = None user_profile_dict['api_key'] = random_api_key() # Since Zulip doesn't use these permissions, drop them del user_profile_dict['user_permissions'] del user_profile_dict['groups'] user_profiles = [ UserProfile(**item) for item in data['zerver_userprofile'] ] for user_profile in user_profiles: user_profile.set_unusable_password() UserProfile.objects.bulk_create(user_profiles) if 'zerver_huddle' in data: bulk_import_model(data, Huddle, 'zerver_huddle') re_map_foreign_keys(data, 'zerver_recipient', 'type_id', related_table="stream", recipient_field=True, id_field=True) re_map_foreign_keys(data, 'zerver_recipient', 'type_id', related_table="user_profile", recipient_field=True, id_field=True) update_model_ids(Recipient, data, 'zerver_recipient', 'recipient') bulk_import_model(data, Recipient, 'zerver_recipient') re_map_foreign_keys(data, 'zerver_subscription', 'user_profile', related_table="user_profile") re_map_foreign_keys(data, 'zerver_subscription', 'recipient', related_table="recipient") update_model_ids(Subscription, data, 'zerver_subscription', 'subscription') bulk_import_model(data, Subscription, 'zerver_subscription') fix_datetime_fields(data, 'zerver_userpresence') re_map_foreign_keys(data, 'zerver_userpresence', 'user_profile', related_table="user_profile") re_map_foreign_keys(data, 'zerver_userpresence', 'client', related_table='client') update_model_ids(UserPresence, data, 'zerver_userpresence', 'user_presence') bulk_import_model(data, UserPresence, 'zerver_userpresence') fix_datetime_fields(data, 'zerver_useractivity') re_map_foreign_keys(data, 'zerver_useractivity', 'user_profile', related_table="user_profile") re_map_foreign_keys(data, 'zerver_useractivity', 'client', related_table='client') update_model_ids(UserActivity, data, 'zerver_useractivity', 'useractivity') bulk_import_model(data, UserActivity, 'zerver_useractivity') fix_datetime_fields(data, 'zerver_useractivityinterval') re_map_foreign_keys(data, 'zerver_useractivityinterval', 'user_profile', related_table="user_profile") update_model_ids(UserActivityInterval, data, 'zerver_useractivityinterval', 'useractivityinterval') bulk_import_model(data, UserActivityInterval, 'zerver_useractivityinterval') if 'zerver_customprofilefield' in data: # As the export of Custom Profile fields is not supported, Zulip exported # data would not contain this field. # However this is supported in slack importer script re_map_foreign_keys(data, 'zerver_customprofilefield', 'realm', related_table="realm") update_model_ids(CustomProfileField, data, 'zerver_customprofilefield', related_table="customprofilefield") bulk_import_model(data, CustomProfileField, 'zerver_customprofilefield') re_map_foreign_keys(data, 'zerver_customprofilefield_value', 'user_profile', related_table="user_profile") re_map_foreign_keys(data, 'zerver_customprofilefield_value', 'field', related_table="customprofilefield") update_model_ids(CustomProfileFieldValue, data, 'zerver_customprofilefield_value', related_table="customprofilefield_value") bulk_import_model(data, CustomProfileFieldValue, 'zerver_customprofilefield_value') # Import uploaded files and avatars import_uploads(os.path.join(import_dir, "avatars"), processing_avatars=True) import_uploads(os.path.join(import_dir, "uploads")) # We need to have this check as the emoji files are only present in the data # importer from slack # For Zulip export, this doesn't exist if os.path.exists(os.path.join(import_dir, "emoji")): import_uploads(os.path.join(import_dir, "emoji"), processing_emojis=True) # Import zerver_message and zerver_usermessage import_message_data(import_dir) # Do attachments AFTER message data is loaded. # TODO: de-dup how we read these json files. fn = os.path.join(import_dir, "attachment.json") if not os.path.exists(fn): raise Exception("Missing attachment.json file!") logging.info("Importing attachment data from %s" % (fn, )) with open(fn) as f: data = ujson.load(f) import_attachments(data) return realm
def do_import_realm(import_dir: Path, subdomain: str) -> Realm: logging.info("Importing realm dump %s" % (import_dir,)) if not os.path.exists(import_dir): raise Exception("Missing import directory!") realm_data_filename = os.path.join(import_dir, "realm.json") if not os.path.exists(realm_data_filename): raise Exception("Missing realm.json file!") logging.info("Importing realm data from %s" % (realm_data_filename,)) with open(realm_data_filename) as f: data = ujson.load(f) update_model_ids(Stream, data, 'zerver_stream', 'stream') re_map_foreign_keys(data, 'zerver_realm', 'notifications_stream', related_table="stream") fix_datetime_fields(data, 'zerver_realm') # Fix realm subdomain information data['zerver_realm'][0]['string_id'] = subdomain data['zerver_realm'][0]['name'] = subdomain fix_realm_authentication_bitfield(data, 'zerver_realm', 'authentication_methods') update_model_ids(Realm, data, 'zerver_realm', 'realm') realm = Realm(**data['zerver_realm'][0]) if realm.notifications_stream_id is not None: notifications_stream_id = int(realm.notifications_stream_id) # type: Optional[int] else: notifications_stream_id = None realm.notifications_stream_id = None realm.save() bulk_import_client(data, Client, 'zerver_client') # Email tokens will automatically be randomly generated when the # Stream objects are created by Django. fix_datetime_fields(data, 'zerver_stream') re_map_foreign_keys(data, 'zerver_stream', 'realm', related_table="realm") bulk_import_model(data, Stream, 'zerver_stream') realm.notifications_stream_id = notifications_stream_id realm.save() re_map_foreign_keys(data, 'zerver_defaultstream', 'stream', related_table="stream") re_map_foreign_keys(data, 'zerver_realmemoji', 'author', related_table="user_profile") for (table, model, related_table) in realm_tables: re_map_foreign_keys(data, table, 'realm', related_table="realm") update_model_ids(model, data, table, related_table) bulk_import_model(data, model, table) # Remap the user IDs for notification_bot and friends to their # appropriate IDs on this server for item in data['zerver_userprofile_crossrealm']: logging.info("Adding to ID map: %s %s" % (item['id'], get_system_bot(item['email']).id)) new_user_id = get_system_bot(item['email']).id update_id_map(table='user_profile', old_id=item['id'], new_id=new_user_id) # Merge in zerver_userprofile_mirrordummy data['zerver_userprofile'] = data['zerver_userprofile'] + data['zerver_userprofile_mirrordummy'] del data['zerver_userprofile_mirrordummy'] data['zerver_userprofile'].sort(key=lambda r: r['id']) # To remap foreign key for UserProfile.last_active_message_id update_message_foreign_keys(import_dir) fix_datetime_fields(data, 'zerver_userprofile') update_model_ids(UserProfile, data, 'zerver_userprofile', 'user_profile') re_map_foreign_keys(data, 'zerver_userprofile', 'realm', related_table="realm") re_map_foreign_keys(data, 'zerver_userprofile', 'bot_owner', related_table="user_profile") re_map_foreign_keys(data, 'zerver_userprofile', 'default_sending_stream', related_table="stream") re_map_foreign_keys(data, 'zerver_userprofile', 'default_events_register_stream', related_table="stream") re_map_foreign_keys(data, 'zerver_userprofile', 'last_active_message_id', related_table="message", id_field=True) for user_profile_dict in data['zerver_userprofile']: user_profile_dict['password'] = None user_profile_dict['api_key'] = random_api_key() # Since Zulip doesn't use these permissions, drop them del user_profile_dict['user_permissions'] del user_profile_dict['groups'] user_profiles = [UserProfile(**item) for item in data['zerver_userprofile']] for user_profile in user_profiles: user_profile.set_unusable_password() UserProfile.objects.bulk_create(user_profiles) if 'zerver_huddle' in data: bulk_import_model(data, Huddle, 'zerver_huddle') re_map_foreign_keys(data, 'zerver_recipient', 'type_id', related_table="stream", recipient_field=True, id_field=True) re_map_foreign_keys(data, 'zerver_recipient', 'type_id', related_table="user_profile", recipient_field=True, id_field=True) update_model_ids(Recipient, data, 'zerver_recipient', 'recipient') bulk_import_model(data, Recipient, 'zerver_recipient') re_map_foreign_keys(data, 'zerver_subscription', 'user_profile', related_table="user_profile") re_map_foreign_keys(data, 'zerver_subscription', 'recipient', related_table="recipient") update_model_ids(Subscription, data, 'zerver_subscription', 'subscription') bulk_import_model(data, Subscription, 'zerver_subscription') fix_datetime_fields(data, 'zerver_userpresence') re_map_foreign_keys(data, 'zerver_userpresence', 'user_profile', related_table="user_profile") re_map_foreign_keys(data, 'zerver_userpresence', 'client', related_table='client') update_model_ids(UserPresence, data, 'zerver_userpresence', 'user_presence') bulk_import_model(data, UserPresence, 'zerver_userpresence') fix_datetime_fields(data, 'zerver_useractivity') re_map_foreign_keys(data, 'zerver_useractivity', 'user_profile', related_table="user_profile") re_map_foreign_keys(data, 'zerver_useractivity', 'client', related_table='client') update_model_ids(UserActivity, data, 'zerver_useractivity', 'useractivity') bulk_import_model(data, UserActivity, 'zerver_useractivity') fix_datetime_fields(data, 'zerver_useractivityinterval') re_map_foreign_keys(data, 'zerver_useractivityinterval', 'user_profile', related_table="user_profile") update_model_ids(UserActivityInterval, data, 'zerver_useractivityinterval', 'useractivityinterval') bulk_import_model(data, UserActivityInterval, 'zerver_useractivityinterval') if 'zerver_customprofilefield' in data: # As the export of Custom Profile fields is not supported, Zulip exported # data would not contain this field. # However this is supported in slack importer script re_map_foreign_keys(data, 'zerver_customprofilefield', 'realm', related_table="realm") update_model_ids(CustomProfileField, data, 'zerver_customprofilefield', related_table="customprofilefield") bulk_import_model(data, CustomProfileField, 'zerver_customprofilefield') re_map_foreign_keys(data, 'zerver_customprofilefield_value', 'user_profile', related_table="user_profile") re_map_foreign_keys(data, 'zerver_customprofilefield_value', 'field', related_table="customprofilefield") update_model_ids(CustomProfileFieldValue, data, 'zerver_customprofilefield_value', related_table="customprofilefield_value") bulk_import_model(data, CustomProfileFieldValue, 'zerver_customprofilefield_value') # Import uploaded files and avatars import_uploads(os.path.join(import_dir, "avatars"), processing_avatars=True) import_uploads(os.path.join(import_dir, "uploads")) # We need to have this check as the emoji files are only present in the data # importer from slack # For Zulip export, this doesn't exist if os.path.exists(os.path.join(import_dir, "emoji")): import_uploads(os.path.join(import_dir, "emoji"), processing_emojis=True) # Import zerver_message and zerver_usermessage import_message_data(import_dir) # Do attachments AFTER message data is loaded. # TODO: de-dup how we read these json files. fn = os.path.join(import_dir, "attachment.json") if not os.path.exists(fn): raise Exception("Missing attachment.json file!") logging.info("Importing attachment data from %s" % (fn,)) with open(fn) as f: data = ujson.load(f) import_attachments(data) return realm
def add_bot_backend(request, user_profile, full_name_raw=REQ("full_name"), short_name_raw=REQ("short_name"), bot_type=REQ(validator=check_int, default=UserProfile.DEFAULT_BOT), payload_url=REQ(validator=check_url, default=""), service_name=REQ(default=None), interface_type=REQ(validator=check_int, default=Service.GENERIC), default_sending_stream_name=REQ('default_sending_stream', default=None), default_events_register_stream_name=REQ( 'default_events_register_stream', default=None), default_all_public_streams=REQ(validator=check_bool, default=None)): # type: (HttpRequest, UserProfile, Text, Text, int, Optional[Text], Optional[Text], int, Optional[Text], Optional[Text], Optional[bool]) -> HttpResponse short_name = check_short_name(short_name_raw) service_name = service_name or short_name short_name += "-bot" full_name = check_full_name(full_name_raw) email = '%s@%s' % (short_name, user_profile.realm.get_bot_domain()) form = CreateUserForm({'full_name': full_name, 'email': email}) if bot_type == UserProfile.EMBEDDED_BOT: if not settings.EMBEDDED_BOTS_ENABLED: return json_error(_("Embedded bots are not enabled.")) if service_name not in [bot.name for bot in EMBEDDED_BOTS]: return json_error(_("Invalid embedded bot name.")) if not form.is_valid(): # We validate client-side as well return json_error(_('Bad name or username')) try: get_user(email, user_profile.realm) return json_error(_("Username already in use")) except UserProfile.DoesNotExist: pass check_valid_bot_type(bot_type) check_valid_interface_type(interface_type) if len(request.FILES) == 0: avatar_source = UserProfile.AVATAR_FROM_GRAVATAR elif len(request.FILES) != 1: return json_error(_("You may only upload one file at a time")) else: avatar_source = UserProfile.AVATAR_FROM_USER default_sending_stream = None if default_sending_stream_name is not None: (default_sending_stream, ignored_rec, ignored_sub) = access_stream_by_name(user_profile, default_sending_stream_name) default_events_register_stream = None if default_events_register_stream_name is not None: (default_events_register_stream, ignored_rec, ignored_sub) = access_stream_by_name( user_profile, default_events_register_stream_name) bot_profile = do_create_user( email=email, password='', realm=user_profile.realm, full_name=full_name, short_name=short_name, active=True, bot_type=bot_type, bot_owner=user_profile, avatar_source=avatar_source, default_sending_stream=default_sending_stream, default_events_register_stream=default_events_register_stream, default_all_public_streams=default_all_public_streams) if len(request.FILES) == 1: user_file = list(request.FILES.values())[0] upload_avatar_image(user_file, user_profile, bot_profile) if bot_type in (UserProfile.OUTGOING_WEBHOOK_BOT, UserProfile.EMBEDDED_BOT): add_service(name=service_name, user_profile=bot_profile, base_url=payload_url, interface=interface_type, token=random_api_key()) json_result = dict( api_key=bot_profile.api_key, avatar_url=avatar_url(bot_profile), default_sending_stream=get_stream_name( bot_profile.default_sending_stream), default_events_register_stream=get_stream_name( bot_profile.default_events_register_stream), default_all_public_streams=bot_profile.default_all_public_streams, ) return json_success(json_result)
def add_bot_backend( request: HttpRequest, user_profile: UserProfile, full_name_raw: Text=REQ("full_name"), short_name_raw: Text=REQ("short_name"), bot_type: int=REQ(validator=check_int, default=UserProfile.DEFAULT_BOT), payload_url: Optional[Text]=REQ(validator=check_url, default=""), service_name: Optional[Text]=REQ(default=None), config_data: Optional[Dict[Text, Text]]=REQ(default=None, validator=check_dict(value_validator=check_string)), interface_type: int=REQ(validator=check_int, default=Service.GENERIC), default_sending_stream_name: Optional[Text]=REQ('default_sending_stream', default=None), default_events_register_stream_name: Optional[Text]=REQ('default_events_register_stream', default=None), default_all_public_streams: Optional[bool]=REQ(validator=check_bool, default=None) ) -> HttpResponse: short_name = check_short_name(short_name_raw) service_name = service_name or short_name short_name += "-bot" full_name = check_full_name(full_name_raw) email = '%s@%s' % (short_name, user_profile.realm.get_bot_domain()) form = CreateUserForm({'full_name': full_name, 'email': email}) if bot_type == UserProfile.EMBEDDED_BOT: if not settings.EMBEDDED_BOTS_ENABLED: return json_error(_("Embedded bots are not enabled.")) if service_name not in [bot.name for bot in EMBEDDED_BOTS]: return json_error(_("Invalid embedded bot name.")) if not form.is_valid(): # We validate client-side as well return json_error(_('Bad name or username')) try: get_user(email, user_profile.realm) return json_error(_("Username already in use")) except UserProfile.DoesNotExist: pass check_valid_bot_type(user_profile, bot_type) check_valid_interface_type(interface_type) if len(request.FILES) == 0: avatar_source = UserProfile.AVATAR_FROM_GRAVATAR elif len(request.FILES) != 1: return json_error(_("You may only upload one file at a time")) else: avatar_source = UserProfile.AVATAR_FROM_USER default_sending_stream = None if default_sending_stream_name is not None: (default_sending_stream, ignored_rec, ignored_sub) = access_stream_by_name( user_profile, default_sending_stream_name) default_events_register_stream = None if default_events_register_stream_name is not None: (default_events_register_stream, ignored_rec, ignored_sub) = access_stream_by_name( user_profile, default_events_register_stream_name) bot_profile = do_create_user(email=email, password='', realm=user_profile.realm, full_name=full_name, short_name=short_name, bot_type=bot_type, bot_owner=user_profile, avatar_source=avatar_source, default_sending_stream=default_sending_stream, default_events_register_stream=default_events_register_stream, default_all_public_streams=default_all_public_streams) if len(request.FILES) == 1: user_file = list(request.FILES.values())[0] upload_avatar_image(user_file, user_profile, bot_profile) if bot_type in (UserProfile.OUTGOING_WEBHOOK_BOT, UserProfile.EMBEDDED_BOT): add_service(name=service_name, user_profile=bot_profile, base_url=payload_url, interface=interface_type, token=random_api_key()) if bot_type == UserProfile.EMBEDDED_BOT: for key, value in config_data.items(): set_bot_config(bot_profile, key, value) json_result = dict( api_key=bot_profile.api_key, avatar_url=avatar_url(bot_profile), default_sending_stream=get_stream_name(bot_profile.default_sending_stream), default_events_register_stream=get_stream_name(bot_profile.default_events_register_stream), default_all_public_streams=bot_profile.default_all_public_streams, ) return json_success(json_result)
def do_import_realm(import_dir): # type: (Path) -> None logging.info("Importing realm dump %s" % (import_dir,)) if not os.path.exists(import_dir): raise Exception("Missing import directory!") realm_data_filename = os.path.join(import_dir, "realm.json") if not os.path.exists(realm_data_filename): raise Exception("Missing realm.json file!") logging.info("Importing realm data from %s" % (realm_data_filename,)) with open(realm_data_filename) as f: data = ujson.load(f) convert_to_id_fields(data, 'zerver_realm', 'notifications_stream') fix_datetime_fields(data, 'zerver_realm') realm = Realm(**data['zerver_realm'][0]) if realm.notifications_stream_id is not None: notifications_stream_id = int(realm.notifications_stream_id) else: notifications_stream_id = None realm.notifications_stream_id = None realm.save() bulk_import_client(data, Client, 'zerver_client') # Email tokens will automatically be randomly generated when the # Stream objects are created by Django. fix_datetime_fields(data, 'zerver_stream') convert_to_id_fields(data, 'zerver_stream', 'realm') bulk_import_model(data, Stream, 'zerver_stream') realm.notifications_stream_id = notifications_stream_id realm.save() convert_to_id_fields(data, "zerver_defaultstream", 'stream') for (table, model) in realm_tables: convert_to_id_fields(data, table, 'realm') bulk_import_model(data, model, table) # Remap the user IDs for notification_bot and friends to their # appropriate IDs on this server for item in data['zerver_userprofile_crossrealm']: logging.info("Adding to ID map: %s %s" % (item['id'], get_user_profile_by_email(item['email']).id)) new_user_id = get_user_profile_by_email(item['email']).id update_id_map(table='user_profile', old_id=item['id'], new_id=new_user_id) fix_datetime_fields(data, 'zerver_userprofile') convert_to_id_fields(data, 'zerver_userprofile', 'realm') re_map_foreign_keys(data, 'zerver_userprofile', 'bot_owner', related_table="user_profile") convert_to_id_fields(data, 'zerver_userprofile', 'default_sending_stream') convert_to_id_fields(data, 'zerver_userprofile', 'default_events_register_stream') for user_profile_dict in data['zerver_userprofile']: user_profile_dict['password'] = None user_profile_dict['api_key'] = random_api_key() # Since Zulip doesn't use these permissions, drop them del user_profile_dict['user_permissions'] del user_profile_dict['groups'] user_profiles = [UserProfile(**item) for item in data['zerver_userprofile']] for user_profile in user_profiles: user_profile.set_unusable_password() UserProfile.objects.bulk_create(user_profiles) if 'zerver_huddle' in data: bulk_import_model(data, Huddle, 'zerver_huddle') bulk_import_model(data, Recipient, 'zerver_recipient') re_map_foreign_keys(data, 'zerver_subscription', 'user_profile', related_table="user_profile") convert_to_id_fields(data, 'zerver_subscription', 'recipient') bulk_import_model(data, Subscription, 'zerver_subscription') fix_datetime_fields(data, 'zerver_userpresence') re_map_foreign_keys(data, 'zerver_userpresence', 'user_profile', related_table="user_profile") re_map_foreign_keys(data, 'zerver_userpresence', 'client', related_table='client') bulk_import_model(data, UserPresence, 'zerver_userpresence') fix_datetime_fields(data, 'zerver_useractivity') re_map_foreign_keys(data, 'zerver_useractivity', 'user_profile', related_table="user_profile") re_map_foreign_keys(data, 'zerver_useractivity', 'client', related_table='client') bulk_import_model(data, UserActivity, 'zerver_useractivity') fix_datetime_fields(data, 'zerver_useractivityinterval') re_map_foreign_keys(data, 'zerver_useractivityinterval', 'user_profile', related_table="user_profile") bulk_import_model(data, UserActivityInterval, 'zerver_useractivityinterval') # Import uploaded files and avatars import_uploads(os.path.join(import_dir, "avatars"), avatar_bucket=True) import_uploads(os.path.join(import_dir, "uploads")) # Import zerver_message and zerver_usermessage import_message_data(import_dir) # Do attachments AFTER message data is loaded. import_attachments(data)
def do_import_realm(import_dir): logging.info("Importing realm dump %s" % (import_dir,)) if not os.path.exists(import_dir): raise Exception("Missing import directory!") realm_data_filename = os.path.join(import_dir, "realm.json") if not os.path.exists(realm_data_filename): raise Exception("Missing realm.json file!") logging.info("Importing realm data from %s" % (realm_data_filename,)) with open(realm_data_filename) as f: data = ujson.load(f) fix_foreign_keys(data, 'zerver_realm', 'notifications_stream') fix_datetime_fields(data, 'zerver_realm', 'date_created') realm = Realm(**data['zerver_realm'][0]) if realm.notifications_stream_id is not None: notifications_stream_id = int(realm.notifications_stream_id) else: notifications_stream_id = None realm.notifications_stream_id = None realm.save() bulk_import_client(data, Client, 'zerver_client') # Email tokens will automatically be randomly generated when the # Stream objects are created by Django. fix_datetime_fields(data, 'zerver_stream', 'date_created') fix_foreign_keys(data, 'zerver_stream', 'realm') bulk_import_model(data, Stream, 'zerver_stream') realm.notifications_stream_id = notifications_stream_id realm.save() fix_foreign_keys(data, "zerver_defaultstream", 'stream') for (table, model) in realm_tables: fix_foreign_keys(data, table, 'realm') bulk_import_model(data, model, table) # Remap the user IDs for notification_bot and friends to their # appropriate IDs on this server for item in data['zerver_userprofile_crossrealm']: logging.info("Adding to ID map: %s %s" % (item['id'], get_user_profile_by_email(item['email']).id)) id_maps["user_profile"][item['id']] = get_user_profile_by_email(item['email']).id fix_datetime_fields(data, 'zerver_userprofile', 'date_joined') fix_datetime_fields(data, 'zerver_userprofile', 'last_login') fix_datetime_fields(data, 'zerver_userprofile', 'last_reminder') fix_foreign_keys(data, 'zerver_userprofile', 'realm') fix_foreign_keys(data, 'zerver_userprofile', 'bot_owner', id_map_table="user_profile") fix_foreign_keys(data, 'zerver_userprofile', 'default_sending_stream') fix_foreign_keys(data, 'zerver_userprofile', 'default_events_register_stream') for user_profile_dict in data['zerver_userprofile']: user_profile_dict['password'] = None user_profile_dict['api_key'] = random_api_key() # Since Zulip doesn't use these permissions, drop them del user_profile_dict['user_permissions'] del user_profile_dict['groups'] user_profiles = [UserProfile(**item) for item in data['zerver_userprofile']] for user_profile in user_profiles: user_profile.set_unusable_password() UserProfile.objects.bulk_create(user_profiles) if 'zerver_huddle' in data: bulk_import_model(data, Huddle, 'zerver_huddle') bulk_import_model(data, Recipient, 'zerver_recipient') fix_foreign_keys(data, 'zerver_subscription', 'user_profile', id_map_table="user_profile") fix_foreign_keys(data, 'zerver_subscription', 'recipient') bulk_import_model(data, Subscription, 'zerver_subscription') fix_datetime_fields(data, 'zerver_userpresence', 'timestamp') fix_foreign_keys(data, 'zerver_userpresence', 'user_profile', id_map_table="user_profile") fix_foreign_keys(data, 'zerver_userpresence', 'client', id_map_table='client') bulk_import_model(data, UserPresence, 'zerver_userpresence') fix_datetime_fields(data, 'zerver_useractivity', 'last_visit') fix_foreign_keys(data, 'zerver_useractivity', 'user_profile', id_map_table="user_profile") fix_foreign_keys(data, 'zerver_useractivity', 'client', id_map_table='client') bulk_import_model(data, UserActivity, 'zerver_useractivity') fix_datetime_fields(data, 'zerver_useractivityinterval', 'start') fix_datetime_fields(data, 'zerver_useractivityinterval', 'end') fix_foreign_keys(data, 'zerver_useractivityinterval', 'user_profile', id_map_table="user_profile") bulk_import_model(data, UserActivityInterval, 'zerver_useractivityinterval') # Import uploaded files and avatars import_uploads(os.path.join(import_dir, "avatars"), avatar_bucket=True) import_uploads(os.path.join(import_dir, "uploads")) dump_file_id = 1 while True: message_filename = os.path.join(import_dir, "messages-%06d.json" % (dump_file_id,)) if not os.path.exists(message_filename): break with open(message_filename) as f: data = ujson.load(f) logging.info("Importing message dump %s" % (message_filename,)) fix_foreign_keys(data, 'zerver_message', 'sender', id_map_table="user_profile") fix_foreign_keys(data, 'zerver_message', 'recipient') fix_foreign_keys(data, 'zerver_message', 'sending_client', id_map_table='client') fix_datetime_fields(data, 'zerver_message', 'pub_date') fix_datetime_fields(data, 'zerver_message', 'last_edit_time') bulk_import_model(data, Message, 'zerver_message') # Due to the structure of these message chunks, we're # guaranteed to have already imported all the Message objects # for this batch of UserMessage objects. fix_foreign_keys(data, 'zerver_usermessage', 'message') fix_foreign_keys(data, 'zerver_usermessage', 'user_profile', id_map_table="user_profile") fix_bitfield_keys(data, 'zerver_usermessage', 'flags') bulk_import_model(data, UserMessage, 'zerver_usermessage') dump_file_id += 1 fix_datetime_fields(data, 'zerver_attachment', 'create_time') fix_foreign_keys(data, 'zerver_attachment', 'owner', id_map_table="user_profile") fix_foreign_keys(data, 'zerver_attachment', 'realm') # TODO: Handle the `messages` keys. # fix_foreign_keys(data, 'zerver_attachment', 'messages') bulk_import_model(data, Attachment, 'zerver_attachment')
def handle(self, **options: Any) -> None: if options["percent_huddles"] + options["percent_personals"] > 100: self.stderr.write( "Error! More than 100% of messages allocated.\n") return # Get consistent data for backend tests. if options["test_suite"]: random.seed(0) if options["delete"]: # Start by clearing all the data in our database clear_database() # Create our two default realms # Could in theory be done via zerver.lib.actions.do_create_realm, but # welcome-bot (needed for do_create_realm) hasn't been created yet zulip_realm = Realm.objects.create( string_id="zulip", name="Zulip Dev", restricted_to_domain=True, description= "The Zulip development environment default organization." " It's great for testing!", invite_required=False, org_type=Realm.CORPORATE) RealmDomain.objects.create(realm=zulip_realm, domain="zulip.com") if options["test_suite"]: mit_realm = Realm.objects.create(string_id="zephyr", name="MIT", restricted_to_domain=True, invite_required=False, org_type=Realm.CORPORATE) RealmDomain.objects.create(realm=mit_realm, domain="mit.edu") lear_realm = Realm.objects.create(string_id="lear", name="Lear & Co.", restricted_to_domain=False, invite_required=False, org_type=Realm.CORPORATE) # Create test Users (UserProfiles are automatically created, # as are subscriptions to the ability to receive personals). names = [ ("Zoe", "*****@*****.**"), ("Othello, the Moor of Venice", "*****@*****.**"), ("Iago", "*****@*****.**"), ("Prospero from The Tempest", "*****@*****.**"), ("Cordelia Lear", "*****@*****.**"), ("King Hamlet", "*****@*****.**"), ("aaron", "*****@*****.**"), ("Polonius", "*****@*****.**"), ] for i in range(options["extra_users"]): names.append( ('Extra User %d' % (i, ), '*****@*****.**' % (i, ))) create_users(zulip_realm, names) iago = get_user("*****@*****.**", zulip_realm) do_change_is_admin(iago, True) iago.is_staff = True iago.save(update_fields=['is_staff']) guest_user = get_user("*****@*****.**", zulip_realm) guest_user.is_guest = True guest_user.save(update_fields=['is_guest']) # These bots are directly referenced from code and thus # are needed for the test suite. all_realm_bots = [ (bot['name'], bot['email_template'] % (settings.INTERNAL_BOT_DOMAIN, )) for bot in settings.INTERNAL_BOTS ] zulip_realm_bots = [ ("Zulip New User Bot", "*****@*****.**"), ("Zulip Error Bot", "*****@*****.**"), ("Zulip Default Bot", "*****@*****.**"), ("Welcome Bot", "*****@*****.**"), ] for i in range(options["extra_bots"]): zulip_realm_bots.append( ('Extra Bot %d' % (i, ), '*****@*****.**' % (i, ))) zulip_realm_bots.extend(all_realm_bots) create_users(zulip_realm, zulip_realm_bots, bot_type=UserProfile.DEFAULT_BOT) # Initialize the email gateway bot as an API Super User email_gateway_bot = get_system_bot(settings.EMAIL_GATEWAY_BOT) email_gateway_bot.is_api_super_user = True email_gateway_bot.save() zoe = get_user("*****@*****.**", zulip_realm) zulip_webhook_bots = [ ("Zulip Webhook Bot", "*****@*****.**"), ] # If a stream is not supplied in the webhook URL, the webhook # will (in some cases) send the notification as a PM to the # owner of the webhook bot, so bot_owner can't be None create_users(zulip_realm, zulip_webhook_bots, bot_type=UserProfile.INCOMING_WEBHOOK_BOT, bot_owner=zoe) aaron = get_user("*****@*****.**", zulip_realm) zulip_outgoing_bots = [("Outgoing Webhook", "*****@*****.**")] create_users(zulip_realm, zulip_outgoing_bots, bot_type=UserProfile.OUTGOING_WEBHOOK_BOT, bot_owner=aaron) outgoing_webhook = get_user("*****@*****.**", zulip_realm) add_service("outgoing-webhook", user_profile=outgoing_webhook, interface=Service.GENERIC, base_url="http://127.0.0.1:5002", token=random_api_key()) # Add the realm internl bots to each realm. create_if_missing_realm_internal_bots() # Create public streams. stream_list = ["Verona", "Denmark", "Scotland", "Venice", "Rome"] stream_dict = { "Verona": { "description": "A city in Italy" }, "Denmark": { "description": "A Scandinavian country" }, "Scotland": { "description": "Located in the United Kingdom" }, "Venice": { "description": "A northeastern Italian city" }, "Rome": { "description": "Yet another Italian city", "is_web_public": True } } # type: Dict[str, Dict[str, Any]] bulk_create_streams(zulip_realm, stream_dict) recipient_streams = [ Stream.objects.get(name=name, realm=zulip_realm).id for name in stream_list ] # type: List[int] # Create subscriptions to streams. The following # algorithm will give each of the users a different but # deterministic subset of the streams (given a fixed list # of users). For the test suite, we have a fixed list of # subscriptions to make sure test data is consistent # across platforms. subscriptions_list = [ ] # type: List[Tuple[UserProfile, Recipient]] profiles = UserProfile.objects.select_related().filter( is_bot=False).order_by("email") # type: Sequence[UserProfile] if options["test_suite"]: subscriptions_map = { '*****@*****.**': ['Verona'], '*****@*****.**': ['Verona'], '*****@*****.**': ['Verona', 'Denmark'], '*****@*****.**': ['Verona', 'Denmark', 'Scotland'], '*****@*****.**': ['Verona', 'Denmark', 'Scotland'], '*****@*****.**': ['Verona', 'Denmark', 'Scotland', 'Venice'], '*****@*****.**': ['Verona', 'Denmark', 'Scotland', 'Venice', 'Rome'], '*****@*****.**': ['Verona'], } for profile in profiles: if profile.email not in subscriptions_map: raise Exception( 'Subscriptions not listed for user %s' % (profile.email, )) for stream_name in subscriptions_map[profile.email]: stream = Stream.objects.get(name=stream_name) r = Recipient.objects.get(type=Recipient.STREAM, type_id=stream.id) subscriptions_list.append((profile, r)) else: for i, profile in enumerate(profiles): # Subscribe to some streams. for type_id in recipient_streams[:int( len(recipient_streams) * float(i) / len(profiles)) + 1]: r = Recipient.objects.get(type=Recipient.STREAM, type_id=type_id) subscriptions_list.append((profile, r)) subscriptions_to_add = [] # type: List[Subscription] event_time = timezone_now() all_subscription_logs = [] # type: (List[RealmAuditLog]) i = 0 for profile, recipient in subscriptions_list: i += 1 color = STREAM_ASSIGNMENT_COLORS[i % len(STREAM_ASSIGNMENT_COLORS)] s = Subscription(recipient=recipient, user_profile=profile, color=color) subscriptions_to_add.append(s) log = RealmAuditLog( realm=profile.realm, modified_user=profile, modified_stream_id=recipient.type_id, event_last_message_id=0, event_type=RealmAuditLog.SUBSCRIPTION_CREATED, event_time=event_time) all_subscription_logs.append(log) Subscription.objects.bulk_create(subscriptions_to_add) RealmAuditLog.objects.bulk_create(all_subscription_logs) # Create custom profile field data phone_number = try_add_realm_custom_profile_field( zulip_realm, "Phone number", CustomProfileField.SHORT_TEXT, hint='') biography = try_add_realm_custom_profile_field( zulip_realm, "Biography", CustomProfileField.LONG_TEXT, hint='What are you known for?') favorite_food = try_add_realm_custom_profile_field( zulip_realm, "Favorite food", CustomProfileField.SHORT_TEXT, hint="Or drink, if you'd prefer") field_data = { 'vim': { 'text': 'Vim', 'order': '1' }, 'emacs': { 'text': 'Emacs', 'order': '2' }, } favorite_editor = try_add_realm_custom_profile_field( zulip_realm, "Favorite editor", CustomProfileField.CHOICE, field_data=field_data) birthday = try_add_realm_custom_profile_field( zulip_realm, "Birthday", CustomProfileField.DATE) favorite_website = try_add_realm_custom_profile_field( zulip_realm, "GitHub profile", CustomProfileField.URL, hint="Or your personal blog's URL") mentor = try_add_realm_custom_profile_field( zulip_realm, "Mentor", CustomProfileField.USER) # Fill in values for Iago and Hamlet hamlet = get_user("*****@*****.**", zulip_realm) do_update_user_custom_profile_data(iago, [ { "id": phone_number.id, "value": "+1-234-567-8901" }, { "id": biography.id, "value": "Betrayer of Othello." }, { "id": favorite_food.id, "value": "Apples" }, { "id": favorite_editor.id, "value": "emacs" }, { "id": birthday.id, "value": "2000-1-1" }, { "id": favorite_website.id, "value": "https://github.com/zulip/zulip" }, { "id": mentor.id, "value": [hamlet.id] }, ]) do_update_user_custom_profile_data(hamlet, [ { "id": phone_number.id, "value": "+0-11-23-456-7890" }, { "id": biography.id, "value": "Prince of Denmark, and other things!" }, { "id": favorite_food.id, "value": "Dark chocolate" }, { "id": favorite_editor.id, "value": "vim" }, { "id": birthday.id, "value": "1900-1-1" }, { "id": favorite_website.id, "value": "https://blog.zulig.org" }, { "id": mentor.id, "value": [iago.id] }, ]) else: zulip_realm = get_realm("zulip") recipient_streams = [ klass.type_id for klass in Recipient.objects.filter(type=Recipient.STREAM) ] # Extract a list of all users user_profiles = list(UserProfile.objects.filter( is_bot=False)) # type: List[UserProfile] # Create a test realm emoji. IMAGE_FILE_PATH = os.path.join(settings.STATIC_ROOT, 'images', 'test-images', 'checkbox.png') with open(IMAGE_FILE_PATH, 'rb') as fp: check_add_realm_emoji(zulip_realm, 'green_tick', iago, fp) if not options["test_suite"]: # Populate users with some bar data for user in user_profiles: status = UserPresence.ACTIVE # type: int date = timezone_now() client = get_client("website") if user.full_name[0] <= 'H': client = get_client("ZulipAndroid") UserPresence.objects.get_or_create(user_profile=user, client=client, timestamp=date, status=status) user_profiles_ids = [user_profile.id for user_profile in user_profiles] # Create several initial huddles for i in range(options["num_huddles"]): get_huddle(random.sample(user_profiles_ids, random.randint(3, 4))) # Create several initial pairs for personals personals_pairs = [ random.sample(user_profiles_ids, 2) for i in range(options["num_personals"]) ] # Generate a new set of test data. create_test_data() # prepopulate the URL preview/embed data for the links present # in the config.generate_data.json data set. This makes it # possible for populate_db to run happily without Internet # access. with open("zerver/tests/fixtures/docs_url_preview_data.json", "r") as f: urls_with_preview_data = ujson.load(f) for url in urls_with_preview_data: cache_set(url, urls_with_preview_data[url], PREVIEW_CACHE_NAME) threads = options["threads"] jobs = [ ] # type: List[Tuple[int, List[List[int]], Dict[str, Any], Callable[[str], int], int]] for i in range(threads): count = options["num_messages"] // threads if i < options["num_messages"] % threads: count += 1 jobs.append((count, personals_pairs, options, self.stdout.write, random.randint(0, 10**10))) for job in jobs: send_messages(job) if options["delete"]: # Create the "website" and "API" clients; if we don't, the # default values in zerver/decorators.py will not work # with the Django test suite. get_client("website") get_client("API") if options["test_suite"]: # Create test users; the MIT ones are needed to test # the Zephyr mirroring codepaths. testsuite_mit_users = [ ("Fred Sipb (MIT)", "*****@*****.**"), ("Athena Consulting Exchange User (MIT)", "*****@*****.**"), ("Esp Classroom (MIT)", "*****@*****.**"), ] create_users(mit_realm, testsuite_mit_users) testsuite_lear_users = [ ("King Lear", "*****@*****.**"), ("Cordelia Lear", "*****@*****.**"), ] create_users(lear_realm, testsuite_lear_users) if not options["test_suite"]: # To keep the messages.json fixtures file for the test # suite fast, don't add these users and subscriptions # when running populate_db for the test suite zulip_stream_dict = { "devel": { "description": "For developing" }, "all": { "description": "For everything" }, "announce": { "description": "For announcements", 'is_announcement_only': True }, "design": { "description": "For design" }, "support": { "description": "For support" }, "social": { "description": "For socializing" }, "test": { "description": "For testing" }, "errors": { "description": "For errors" }, "sales": { "description": "For sales discussion" } } # type: Dict[str, Dict[str, Any]] # Calculate the maximum number of digits in any extra stream's # number, since a stream with name "Extra Stream 3" could show # up after "Extra Stream 29". (Used later to pad numbers with # 0s). maximum_digits = len(str(options['extra_streams'] - 1)) for i in range(options['extra_streams']): # Pad the number with 0s based on `maximum_digits`. number_str = str(i).zfill(maximum_digits) extra_stream_name = 'Extra Stream ' + number_str zulip_stream_dict[extra_stream_name] = { "description": "Auto-generated extra stream.", } bulk_create_streams(zulip_realm, zulip_stream_dict) # Now that we've created the notifications stream, configure it properly. zulip_realm.notifications_stream = get_stream( "announce", zulip_realm) zulip_realm.save(update_fields=['notifications_stream']) # Add a few default streams for default_stream_name in [ "design", "devel", "social", "support" ]: DefaultStream.objects.create(realm=zulip_realm, stream=get_stream( default_stream_name, zulip_realm)) # Now subscribe everyone to these streams subscriptions_to_add = [] event_time = timezone_now() all_subscription_logs = [] profiles = UserProfile.objects.select_related().filter( realm=zulip_realm) for i, stream_name in enumerate(zulip_stream_dict): stream = Stream.objects.get(name=stream_name, realm=zulip_realm) recipient = Recipient.objects.get(type=Recipient.STREAM, type_id=stream.id) for profile in profiles: # Subscribe to some streams. s = Subscription( recipient=recipient, user_profile=profile, color=STREAM_ASSIGNMENT_COLORS[ i % len(STREAM_ASSIGNMENT_COLORS)]) subscriptions_to_add.append(s) log = RealmAuditLog( realm=profile.realm, modified_user=profile, modified_stream=stream, event_last_message_id=0, event_type=RealmAuditLog.SUBSCRIPTION_CREATED, event_time=event_time) all_subscription_logs.append(log) Subscription.objects.bulk_create(subscriptions_to_add) RealmAuditLog.objects.bulk_create(all_subscription_logs) # These bots are not needed by the test suite internal_zulip_users_nosubs = [ ("Zulip Commit Bot", "*****@*****.**"), ("Zulip Trac Bot", "*****@*****.**"), ("Zulip Nagios Bot", "*****@*****.**"), ] create_users(zulip_realm, internal_zulip_users_nosubs, bot_type=UserProfile.DEFAULT_BOT) zulip_cross_realm_bots = [ ("Zulip Feedback Bot", "*****@*****.**"), ] create_users(zulip_realm, zulip_cross_realm_bots, bot_type=UserProfile.DEFAULT_BOT) # Mark all messages as read UserMessage.objects.all().update(flags=UserMessage.flags.read) if not options["test_suite"]: # Update pointer of each user to point to the last message in their # UserMessage rows with sender_id=user_profile_id. users = list( UserMessage.objects.filter(message__sender_id=F( 'user_profile_id')).values('user_profile_id').annotate( pointer=Max('message_id'))) for user in users: UserProfile.objects.filter( id=user['user_profile_id']).update( pointer=user['pointer']) create_user_groups() if not options["test_suite"]: # We populate the analytics database here for # development purpose only call_command('populate_analytics_db') self.stdout.write("Successfully populated test database.\n")
def add_bot_backend(request, user_profile, full_name_raw=REQ("full_name"), short_name_raw=REQ("short_name"), bot_type=REQ(validator=check_int, default=UserProfile.DEFAULT_BOT), payload_url=REQ(validator=check_url, default=None), default_sending_stream_name=REQ('default_sending_stream', default=None), default_events_register_stream_name=REQ('default_events_register_stream', default=None), default_all_public_streams=REQ(validator=check_bool, default=None)): # type: (HttpRequest, UserProfile, Text, Text, int, Optional[Text], Optional[Text], Optional[Text], Optional[bool]) -> HttpResponse short_name = check_short_name(short_name_raw) service_name = short_name short_name += "-bot" full_name = check_full_name(full_name_raw) email = '%s@%s' % (short_name, user_profile.realm.get_bot_domain()) form = CreateUserForm({'full_name': full_name, 'email': email}) if not form.is_valid(): # We validate client-side as well return json_error(_('Bad name or username')) try: get_user(email, user_profile.realm) return json_error(_("Username already in use")) except UserProfile.DoesNotExist: pass check_valid_bot_type(bot_type) if len(request.FILES) == 0: avatar_source = UserProfile.AVATAR_FROM_GRAVATAR elif len(request.FILES) != 1: return json_error(_("You may only upload one file at a time")) else: avatar_source = UserProfile.AVATAR_FROM_USER default_sending_stream = None if default_sending_stream_name is not None: (default_sending_stream, ignored_rec, ignored_sub) = access_stream_by_name( user_profile, default_sending_stream_name) default_events_register_stream = None if default_events_register_stream_name is not None: (default_events_register_stream, ignored_rec, ignored_sub) = access_stream_by_name( user_profile, default_events_register_stream_name) bot_profile = do_create_user(email=email, password='', realm=user_profile.realm, full_name=full_name, short_name=short_name, active=True, bot_type=bot_type, bot_owner=user_profile, avatar_source=avatar_source, default_sending_stream=default_sending_stream, default_events_register_stream=default_events_register_stream, default_all_public_streams=default_all_public_streams) if len(request.FILES) == 1: user_file = list(request.FILES.values())[0] upload_avatar_image(user_file, user_profile, bot_profile) if bot_type == UserProfile.OUTGOING_WEBHOOK_BOT: add_outgoing_webhook_service(name=service_name, user_profile=bot_profile, base_url=payload_url, interface=1, token=random_api_key()) json_result = dict( api_key=bot_profile.api_key, avatar_url=avatar_url(bot_profile), default_sending_stream=get_stream_name(bot_profile.default_sending_stream), default_events_register_stream=get_stream_name(bot_profile.default_events_register_stream), default_all_public_streams=bot_profile.default_all_public_streams, ) return json_success(json_result)
def handle(self, **options: Any) -> None: if options["percent_huddles"] + options["percent_personals"] > 100: self.stderr.write("Error! More than 100% of messages allocated.\n") return # Get consistent data for backend tests. if options["test_suite"]: random.seed(0) if options["delete"]: # Start by clearing all the data in our database clear_database() # Create our two default realms # Could in theory be done via zerver.lib.actions.do_create_realm, but # welcome-bot (needed for do_create_realm) hasn't been created yet zulip_realm = Realm.objects.create( string_id="zulip", name="Zulip Dev", restricted_to_domain=True, description="The Zulip development environment default organization." " It's great for testing!", invite_required=False, org_type=Realm.CORPORATE) RealmDomain.objects.create(realm=zulip_realm, domain="zulip.com") if options["test_suite"]: mit_realm = Realm.objects.create( string_id="zephyr", name="MIT", restricted_to_domain=True, invite_required=False, org_type=Realm.CORPORATE) RealmDomain.objects.create(realm=mit_realm, domain="mit.edu") lear_realm = Realm.objects.create( string_id="lear", name="Lear & Co.", restricted_to_domain=False, invite_required=False, org_type=Realm.CORPORATE) # Create test Users (UserProfiles are automatically created, # as are subscriptions to the ability to receive personals). names = [ ("Zoe", "*****@*****.**"), ("Othello, the Moor of Venice", "*****@*****.**"), ("Iago", "*****@*****.**"), ("Prospero from The Tempest", "*****@*****.**"), ("Cordelia Lear", "*****@*****.**"), ("King Hamlet", "*****@*****.**"), ("aaron", "*****@*****.**"), ("Polonius", "*****@*****.**"), ] for i in range(options["extra_users"]): names.append(('Extra User %d' % (i,), '*****@*****.**' % (i,))) create_users(zulip_realm, names) iago = get_user("*****@*****.**", zulip_realm) do_change_is_admin(iago, True) iago.is_staff = True iago.save(update_fields=['is_staff']) guest_user = get_user("*****@*****.**", zulip_realm) guest_user.is_guest = True guest_user.save(update_fields=['is_guest']) # These bots are directly referenced from code and thus # are needed for the test suite. all_realm_bots = [(bot['name'], bot['email_template'] % (settings.INTERNAL_BOT_DOMAIN,)) for bot in settings.INTERNAL_BOTS] zulip_realm_bots = [ ("Zulip New User Bot", "*****@*****.**"), ("Zulip Error Bot", "*****@*****.**"), ("Zulip Default Bot", "*****@*****.**"), ("Welcome Bot", "*****@*****.**"), ] for i in range(options["extra_bots"]): zulip_realm_bots.append(('Extra Bot %d' % (i,), '*****@*****.**' % (i,))) zulip_realm_bots.extend(all_realm_bots) create_users(zulip_realm, zulip_realm_bots, bot_type=UserProfile.DEFAULT_BOT) # Initialize the email gateway bot as an API Super User email_gateway_bot = get_system_bot(settings.EMAIL_GATEWAY_BOT) email_gateway_bot.is_api_super_user = True email_gateway_bot.save() zoe = get_user("*****@*****.**", zulip_realm) zulip_webhook_bots = [ ("Zulip Webhook Bot", "*****@*****.**"), ] # If a stream is not supplied in the webhook URL, the webhook # will (in some cases) send the notification as a PM to the # owner of the webhook bot, so bot_owner can't be None create_users(zulip_realm, zulip_webhook_bots, bot_type=UserProfile.INCOMING_WEBHOOK_BOT, bot_owner=zoe) aaron = get_user("*****@*****.**", zulip_realm) zulip_outgoing_bots = [ ("Outgoing Webhook", "*****@*****.**") ] create_users(zulip_realm, zulip_outgoing_bots, bot_type=UserProfile.OUTGOING_WEBHOOK_BOT, bot_owner=aaron) outgoing_webhook = get_user("*****@*****.**", zulip_realm) add_service("outgoing-webhook", user_profile=outgoing_webhook, interface=Service.GENERIC, base_url="http://127.0.0.1:5002", token=random_api_key()) # Add the realm internl bots to each realm. create_if_missing_realm_internal_bots() # Create public streams. stream_list = ["Verona", "Denmark", "Scotland", "Venice", "Rome"] stream_dict = { "Verona": {"description": "A city in Italy"}, "Denmark": {"description": "A Scandinavian country"}, "Scotland": {"description": "Located in the United Kingdom"}, "Venice": {"description": "A northeastern Italian city"}, "Rome": {"description": "Yet another Italian city", "is_web_public": True} } # type: Dict[str, Dict[str, Any]] bulk_create_streams(zulip_realm, stream_dict) recipient_streams = [Stream.objects.get(name=name, realm=zulip_realm).id for name in stream_list] # type: List[int] # Create subscriptions to streams. The following # algorithm will give each of the users a different but # deterministic subset of the streams (given a fixed list # of users). For the test suite, we have a fixed list of # subscriptions to make sure test data is consistent # across platforms. subscriptions_list = [] # type: List[Tuple[UserProfile, Recipient]] profiles = UserProfile.objects.select_related().filter( is_bot=False).order_by("email") # type: Sequence[UserProfile] if options["test_suite"]: subscriptions_map = { '*****@*****.**': ['Verona'], '*****@*****.**': ['Verona'], '*****@*****.**': ['Verona', 'Denmark'], '*****@*****.**': ['Verona', 'Denmark', 'Scotland'], '*****@*****.**': ['Verona', 'Denmark', 'Scotland'], '*****@*****.**': ['Verona', 'Denmark', 'Scotland', 'Venice'], '*****@*****.**': ['Verona', 'Denmark', 'Scotland', 'Venice', 'Rome'], '*****@*****.**': ['Verona'], } for profile in profiles: if profile.email not in subscriptions_map: raise Exception('Subscriptions not listed for user %s' % (profile.email,)) for stream_name in subscriptions_map[profile.email]: stream = Stream.objects.get(name=stream_name) r = Recipient.objects.get(type=Recipient.STREAM, type_id=stream.id) subscriptions_list.append((profile, r)) else: for i, profile in enumerate(profiles): # Subscribe to some streams. for type_id in recipient_streams[:int(len(recipient_streams) * float(i)/len(profiles)) + 1]: r = Recipient.objects.get(type=Recipient.STREAM, type_id=type_id) subscriptions_list.append((profile, r)) subscriptions_to_add = [] # type: List[Subscription] event_time = timezone_now() all_subscription_logs = [] # type: (List[RealmAuditLog]) i = 0 for profile, recipient in subscriptions_list: i += 1 color = STREAM_ASSIGNMENT_COLORS[i % len(STREAM_ASSIGNMENT_COLORS)] s = Subscription( recipient=recipient, user_profile=profile, color=color) subscriptions_to_add.append(s) log = RealmAuditLog(realm=profile.realm, modified_user=profile, modified_stream_id=recipient.type_id, event_last_message_id=0, event_type='subscription_created', event_time=event_time) all_subscription_logs.append(log) Subscription.objects.bulk_create(subscriptions_to_add) RealmAuditLog.objects.bulk_create(all_subscription_logs) # Create custom profile field data phone_number = try_add_realm_custom_profile_field(zulip_realm, "Phone number", CustomProfileField.SHORT_TEXT, hint='') biography = try_add_realm_custom_profile_field(zulip_realm, "Biography", CustomProfileField.LONG_TEXT, hint='What are you known for?') favorite_food = try_add_realm_custom_profile_field(zulip_realm, "Favorite food", CustomProfileField.SHORT_TEXT, hint="Or drink, if you'd prefer") field_data = { 'vim': {'text': 'Vim', 'order': '1'}, 'emacs': {'text': 'Emacs', 'order': '2'}, } favorite_editor = try_add_realm_custom_profile_field(zulip_realm, "Favorite editor", CustomProfileField.CHOICE, field_data=field_data) birthday = try_add_realm_custom_profile_field(zulip_realm, "Birthday", CustomProfileField.DATE) favorite_website = try_add_realm_custom_profile_field(zulip_realm, "GitHub profile", CustomProfileField.URL, hint="Or your personal blog's URL") mentor = try_add_realm_custom_profile_field(zulip_realm, "Mentor", CustomProfileField.USER) # Fill in values for Iago and Hamlet hamlet = get_user("*****@*****.**", zulip_realm) do_update_user_custom_profile_data(iago, [ {"id": phone_number.id, "value": "+1-234-567-8901"}, {"id": biography.id, "value": "Betrayer of Othello."}, {"id": favorite_food.id, "value": "Apples"}, {"id": favorite_editor.id, "value": "emacs"}, {"id": birthday.id, "value": "2000-1-1"}, {"id": favorite_website.id, "value": "https://github.com/zulip/zulip"}, {"id": mentor.id, "value": hamlet.id}, ]) do_update_user_custom_profile_data(hamlet, [ {"id": phone_number.id, "value": "+0-11-23-456-7890"}, {"id": biography.id, "value": "Prince of Denmark, and other things!"}, {"id": favorite_food.id, "value": "Dark chocolate"}, {"id": favorite_editor.id, "value": "vim"}, {"id": birthday.id, "value": "1900-1-1"}, {"id": favorite_website.id, "value": "https://blog.zulig.org"}, {"id": mentor.id, "value": iago.id}, ]) else: zulip_realm = get_realm("zulip") recipient_streams = [klass.type_id for klass in Recipient.objects.filter(type=Recipient.STREAM)] # Extract a list of all users user_profiles = list(UserProfile.objects.filter(is_bot=False)) # type: List[UserProfile] # Create a test realm emoji. IMAGE_FILE_PATH = os.path.join(settings.STATIC_ROOT, 'images', 'test-images', 'checkbox.png') with open(IMAGE_FILE_PATH, 'rb') as fp: check_add_realm_emoji(zulip_realm, 'green_tick', iago, fp) if not options["test_suite"]: # Populate users with some bar data for user in user_profiles: status = UserPresence.ACTIVE # type: int date = timezone_now() client = get_client("website") if user.full_name[0] <= 'H': client = get_client("ZulipAndroid") UserPresence.objects.get_or_create(user_profile=user, client=client, timestamp=date, status=status) user_profiles_ids = [user_profile.id for user_profile in user_profiles] # Create several initial huddles for i in range(options["num_huddles"]): get_huddle(random.sample(user_profiles_ids, random.randint(3, 4))) # Create several initial pairs for personals personals_pairs = [random.sample(user_profiles_ids, 2) for i in range(options["num_personals"])] # Generate a new set of test data. create_test_data() # prepopulate the URL preview/embed data for the links present # in the config.generate_data.json data set. This makes it # possible for populate_db to run happily without Internet # access. with open("zerver/tests/fixtures/docs_url_preview_data.json", "r") as f: urls_with_preview_data = ujson.load(f) for url in urls_with_preview_data: cache_set(url, urls_with_preview_data[url], PREVIEW_CACHE_NAME) threads = options["threads"] jobs = [] # type: List[Tuple[int, List[List[int]], Dict[str, Any], Callable[[str], int], int]] for i in range(threads): count = options["num_messages"] // threads if i < options["num_messages"] % threads: count += 1 jobs.append((count, personals_pairs, options, self.stdout.write, random.randint(0, 10**10))) for job in jobs: send_messages(job) if options["delete"]: # Create the "website" and "API" clients; if we don't, the # default values in zerver/decorators.py will not work # with the Django test suite. get_client("website") get_client("API") if options["test_suite"]: # Create test users; the MIT ones are needed to test # the Zephyr mirroring codepaths. testsuite_mit_users = [ ("Fred Sipb (MIT)", "*****@*****.**"), ("Athena Consulting Exchange User (MIT)", "*****@*****.**"), ("Esp Classroom (MIT)", "*****@*****.**"), ] create_users(mit_realm, testsuite_mit_users) testsuite_lear_users = [ ("King Lear", "*****@*****.**"), ("Cordelia Lear", "*****@*****.**"), ] create_users(lear_realm, testsuite_lear_users) if not options["test_suite"]: # To keep the messages.json fixtures file for the test # suite fast, don't add these users and subscriptions # when running populate_db for the test suite zulip_stream_dict = { "devel": {"description": "For developing"}, "all": {"description": "For everything"}, "announce": {"description": "For announcements", 'is_announcement_only': True}, "design": {"description": "For design"}, "support": {"description": "For support"}, "social": {"description": "For socializing"}, "test": {"description": "For testing"}, "errors": {"description": "For errors"}, "sales": {"description": "For sales discussion"} } # type: Dict[str, Dict[str, Any]] # Calculate the maximum number of digits in any extra stream's # number, since a stream with name "Extra Stream 3" could show # up after "Extra Stream 29". (Used later to pad numbers with # 0s). maximum_digits = len(str(options['extra_streams'] - 1)) for i in range(options['extra_streams']): # Pad the number with 0s based on `maximum_digits`. number_str = str(i).zfill(maximum_digits) extra_stream_name = 'Extra Stream ' + number_str zulip_stream_dict[extra_stream_name] = { "description": "Auto-generated extra stream.", } bulk_create_streams(zulip_realm, zulip_stream_dict) # Now that we've created the notifications stream, configure it properly. zulip_realm.notifications_stream = get_stream("announce", zulip_realm) zulip_realm.save(update_fields=['notifications_stream']) # Add a few default streams for default_stream_name in ["design", "devel", "social", "support"]: DefaultStream.objects.create(realm=zulip_realm, stream=get_stream(default_stream_name, zulip_realm)) # Now subscribe everyone to these streams subscriptions_to_add = [] event_time = timezone_now() all_subscription_logs = [] profiles = UserProfile.objects.select_related().filter(realm=zulip_realm) for i, stream_name in enumerate(zulip_stream_dict): stream = Stream.objects.get(name=stream_name, realm=zulip_realm) recipient = Recipient.objects.get(type=Recipient.STREAM, type_id=stream.id) for profile in profiles: # Subscribe to some streams. s = Subscription( recipient=recipient, user_profile=profile, color=STREAM_ASSIGNMENT_COLORS[i % len(STREAM_ASSIGNMENT_COLORS)]) subscriptions_to_add.append(s) log = RealmAuditLog(realm=profile.realm, modified_user=profile, modified_stream=stream, event_last_message_id=0, event_type='subscription_created', event_time=event_time) all_subscription_logs.append(log) Subscription.objects.bulk_create(subscriptions_to_add) RealmAuditLog.objects.bulk_create(all_subscription_logs) # These bots are not needed by the test suite internal_zulip_users_nosubs = [ ("Zulip Commit Bot", "*****@*****.**"), ("Zulip Trac Bot", "*****@*****.**"), ("Zulip Nagios Bot", "*****@*****.**"), ] create_users(zulip_realm, internal_zulip_users_nosubs, bot_type=UserProfile.DEFAULT_BOT) zulip_cross_realm_bots = [ ("Zulip Feedback Bot", "*****@*****.**"), ] create_users(zulip_realm, zulip_cross_realm_bots, bot_type=UserProfile.DEFAULT_BOT) # Mark all messages as read UserMessage.objects.all().update(flags=UserMessage.flags.read) if not options["test_suite"]: # Update pointer of each user to point to the last message in their # UserMessage rows with sender_id=user_profile_id. users = list(UserMessage.objects.filter( message__sender_id=F('user_profile_id')).values( 'user_profile_id').annotate(pointer=Max('message_id'))) for user in users: UserProfile.objects.filter(id=user['user_profile_id']).update( pointer=user['pointer']) create_user_groups() self.stdout.write("Successfully populated test database.\n")
def do_import_realm(import_dir: Path, subdomain: str) -> Realm: logging.info("Importing realm dump %s" % (import_dir,)) if not os.path.exists(import_dir): raise Exception("Missing import directory!") realm_data_filename = os.path.join(import_dir, "realm.json") if not os.path.exists(realm_data_filename): raise Exception("Missing realm.json file!") logging.info("Importing realm data from %s" % (realm_data_filename,)) with open(realm_data_filename) as f: data = ujson.load(f) # We don't import the Stream model yet, since it depends on Realm, # which isn't imported yet. But we need the Stream model IDs for # notifications_stream. update_model_ids(Stream, data, 'stream') re_map_foreign_keys(data, 'zerver_realm', 'notifications_stream', related_table="stream") fix_datetime_fields(data, 'zerver_realm') # Fix realm subdomain information data['zerver_realm'][0]['string_id'] = subdomain data['zerver_realm'][0]['name'] = subdomain fix_realm_authentication_bitfield(data, 'zerver_realm', 'authentication_methods') update_model_ids(Realm, data, 'realm') realm = Realm(**data['zerver_realm'][0]) if realm.notifications_stream_id is not None: notifications_stream_id = int(realm.notifications_stream_id) # type: Optional[int] else: notifications_stream_id = None realm.notifications_stream_id = None realm.save() bulk_import_client(data, Client, 'zerver_client') # Email tokens will automatically be randomly generated when the # Stream objects are created by Django. fix_datetime_fields(data, 'zerver_stream') re_map_foreign_keys(data, 'zerver_stream', 'realm', related_table="realm") bulk_import_model(data, Stream) realm.notifications_stream_id = notifications_stream_id realm.save() # Remap the user IDs for notification_bot and friends to their # appropriate IDs on this server for item in data['zerver_userprofile_crossrealm']: logging.info("Adding to ID map: %s %s" % (item['id'], get_system_bot(item['email']).id)) new_user_id = get_system_bot(item['email']).id update_id_map(table='user_profile', old_id=item['id'], new_id=new_user_id) # Merge in zerver_userprofile_mirrordummy data['zerver_userprofile'] = data['zerver_userprofile'] + data['zerver_userprofile_mirrordummy'] del data['zerver_userprofile_mirrordummy'] data['zerver_userprofile'].sort(key=lambda r: r['id']) # To remap foreign key for UserProfile.last_active_message_id update_message_foreign_keys(import_dir) fix_datetime_fields(data, 'zerver_userprofile') update_model_ids(UserProfile, data, 'user_profile') re_map_foreign_keys(data, 'zerver_userprofile', 'realm', related_table="realm") re_map_foreign_keys(data, 'zerver_userprofile', 'bot_owner', related_table="user_profile") re_map_foreign_keys(data, 'zerver_userprofile', 'default_sending_stream', related_table="stream") re_map_foreign_keys(data, 'zerver_userprofile', 'default_events_register_stream', related_table="stream") re_map_foreign_keys(data, 'zerver_userprofile', 'last_active_message_id', related_table="message", id_field=True) for user_profile_dict in data['zerver_userprofile']: user_profile_dict['password'] = None user_profile_dict['api_key'] = random_api_key() # Since Zulip doesn't use these permissions, drop them del user_profile_dict['user_permissions'] del user_profile_dict['groups'] user_profiles = [UserProfile(**item) for item in data['zerver_userprofile']] for user_profile in user_profiles: user_profile.set_unusable_password() UserProfile.objects.bulk_create(user_profiles) re_map_foreign_keys(data, 'zerver_defaultstream', 'stream', related_table="stream") re_map_foreign_keys(data, 'zerver_realmemoji', 'author', related_table="user_profile") for (table, model, related_table) in realm_tables: re_map_foreign_keys(data, table, 'realm', related_table="realm") update_model_ids(model, data, related_table) bulk_import_model(data, model) if 'zerver_huddle' in data: update_model_ids(Huddle, data, 'huddle') # We don't import Huddle yet, since we don't have the data to # compute huddle hashes until we've imported some of the # tables below. # TODO: double-check this. re_map_foreign_keys(data, 'zerver_recipient', 'type_id', related_table="stream", recipient_field=True, id_field=True) re_map_foreign_keys(data, 'zerver_recipient', 'type_id', related_table="user_profile", recipient_field=True, id_field=True) re_map_foreign_keys(data, 'zerver_recipient', 'type_id', related_table="huddle", recipient_field=True, id_field=True) update_model_ids(Recipient, data, 'recipient') bulk_import_model(data, Recipient) re_map_foreign_keys(data, 'zerver_subscription', 'user_profile', related_table="user_profile") get_huddles_from_subscription(data, 'zerver_subscription') re_map_foreign_keys(data, 'zerver_subscription', 'recipient', related_table="recipient") update_model_ids(Subscription, data, 'subscription') bulk_import_model(data, Subscription) if 'zerver_realmauditlog' in data: fix_datetime_fields(data, 'zerver_realmauditlog') re_map_foreign_keys(data, 'zerver_realmauditlog', 'realm', related_table="realm") re_map_foreign_keys(data, 'zerver_realmauditlog', 'modified_user', related_table='user_profile') re_map_foreign_keys(data, 'zerver_realmauditlog', 'acting_user', related_table='user_profile') re_map_foreign_keys(data, 'zerver_realmauditlog', 'modified_stream', related_table="stream") update_model_ids(RealmAuditLog, data, related_table="realmauditlog") bulk_import_model(data, RealmAuditLog) else: create_subscription_events(data, 'zerver_subscription') if 'zerver_huddle' in data: process_huddle_hash(data, 'zerver_huddle') bulk_import_model(data, Huddle) if 'zerver_userhotspot' in data: fix_datetime_fields(data, 'zerver_userhotspot') re_map_foreign_keys(data, 'zerver_userhotspot', 'user', related_table='user_profile') update_model_ids(UserHotspot, data, 'userhotspot') bulk_import_model(data, UserHotspot) if 'zerver_mutedtopic' in data: re_map_foreign_keys(data, 'zerver_mutedtopic', 'user_profile', related_table='user_profile') re_map_foreign_keys(data, 'zerver_mutedtopic', 'stream', related_table='stream') re_map_foreign_keys(data, 'zerver_mutedtopic', 'recipient', related_table='recipient') update_model_ids(MutedTopic, data, 'mutedtopic') bulk_import_model(data, MutedTopic) if 'zerver_service' in data: re_map_foreign_keys(data, 'zerver_service', 'user_profile', related_table='user_profile') fix_service_tokens(data, 'zerver_service') update_model_ids(Service, data, 'service') bulk_import_model(data, Service) if 'zerver_usergroup' in data: re_map_foreign_keys(data, 'zerver_usergroup', 'realm', related_table='realm') re_map_foreign_keys_many_to_many(data, 'zerver_usergroup', 'members', related_table='user_profile') update_model_ids(UserGroup, data, 'usergroup') bulk_import_model(data, UserGroup) re_map_foreign_keys(data, 'zerver_usergroupmembership', 'user_group', related_table='usergroup') re_map_foreign_keys(data, 'zerver_usergroupmembership', 'user_profile', related_table='user_profile') update_model_ids(UserGroupMembership, data, 'usergroupmembership') bulk_import_model(data, UserGroupMembership) if 'zerver_botstoragedata' in data: re_map_foreign_keys(data, 'zerver_botstoragedata', 'bot_profile', related_table='user_profile') update_model_ids(BotStorageData, data, 'botstoragedata') bulk_import_model(data, BotStorageData) if 'zerver_botconfigdata' in data: re_map_foreign_keys(data, 'zerver_botconfigdata', 'bot_profile', related_table='user_profile') update_model_ids(BotConfigData, data, 'botconfigdata') bulk_import_model(data, BotConfigData) fix_datetime_fields(data, 'zerver_userpresence') re_map_foreign_keys(data, 'zerver_userpresence', 'user_profile', related_table="user_profile") re_map_foreign_keys(data, 'zerver_userpresence', 'client', related_table='client') update_model_ids(UserPresence, data, 'user_presence') bulk_import_model(data, UserPresence) fix_datetime_fields(data, 'zerver_useractivity') re_map_foreign_keys(data, 'zerver_useractivity', 'user_profile', related_table="user_profile") re_map_foreign_keys(data, 'zerver_useractivity', 'client', related_table='client') update_model_ids(UserActivity, data, 'useractivity') bulk_import_model(data, UserActivity) fix_datetime_fields(data, 'zerver_useractivityinterval') re_map_foreign_keys(data, 'zerver_useractivityinterval', 'user_profile', related_table="user_profile") update_model_ids(UserActivityInterval, data, 'useractivityinterval') bulk_import_model(data, UserActivityInterval) re_map_foreign_keys(data, 'zerver_customprofilefield', 'realm', related_table="realm") update_model_ids(CustomProfileField, data, related_table="customprofilefield") bulk_import_model(data, CustomProfileField) re_map_foreign_keys(data, 'zerver_customprofilefieldvalue', 'user_profile', related_table="user_profile") re_map_foreign_keys(data, 'zerver_customprofilefieldvalue', 'field', related_table="customprofilefield") fix_customprofilefield(data) update_model_ids(CustomProfileFieldValue, data, related_table="customprofilefieldvalue") bulk_import_model(data, CustomProfileFieldValue) # Import uploaded files and avatars import_uploads(os.path.join(import_dir, "avatars"), processing_avatars=True) import_uploads(os.path.join(import_dir, "uploads")) # We need to have this check as the emoji files are only present in the data # importer from slack # For Zulip export, this doesn't exist if os.path.exists(os.path.join(import_dir, "emoji")): import_uploads(os.path.join(import_dir, "emoji"), processing_emojis=True) # Import zerver_message and zerver_usermessage import_message_data(import_dir) re_map_foreign_keys(data, 'zerver_reaction', 'message', related_table="message") re_map_foreign_keys(data, 'zerver_reaction', 'user_profile', related_table="user_profile") re_map_foreign_keys(data, 'zerver_reaction', 'emoji_code', related_table="realmemoji", id_field=True, reaction_field=True) update_model_ids(Reaction, data, 'reaction') bulk_import_model(data, Reaction) # Do attachments AFTER message data is loaded. # TODO: de-dup how we read these json files. fn = os.path.join(import_dir, "attachment.json") if not os.path.exists(fn): raise Exception("Missing attachment.json file!") logging.info("Importing attachment data from %s" % (fn,)) with open(fn) as f: data = ujson.load(f) import_attachments(data) return realm