def fix_service_tokens(data: TableData, table: TableName) -> None:
"""
The tokens in the services are created by 'random_api_key'.
As the tokens are unique, they should be re-created for the imports.
"""
for item in data[table]:
item['token'] = random_api_key()
def handle(self, *args, **options):
# type: (*Any, **Any) -> None
if None in (options["api"], options["web"], options["domain"]):
print("\033[1;31mYou must provide a domain, an API URL, and a web URL.\033[0m\n", file=sys.stderr)
self.print_help("python manage.py", "create_realm")
exit(1)
if not options["no_realm"]:
CreateRealm().handle(*args, **options)
print() # Newline
realm = get_realm(options["domain"])
if realm is None:
print("\033[1;31mRealm does not exist!\033[0m\n", file=sys.stderr)
exit(2)
dep = Deployment()
dep.api_key = random_api_key()
dep.save()
old_dep = realm.deployment
if old_dep is not None:
old_dep.realms.remove(realm)
old_dep.save()
dep.realms = [realm]
dep.base_api_url = options["api"]
dep.base_site_url = options["web"]
dep.save()
print("Deployment %s created." % (dep.id,))
print("DEPLOYMENT_ROLE_NAME = %s" % (dep.name,))
print("DEPLOYMENT_ROLE_KEY = %s" % (dep.api_key,))
def fix_service_tokens(data: TableData, table: TableName) -> None:
"""
The tokens in the services are created by 'random_api_key'.
As the tokens are unique, they should be re-created for the imports.
"""
for item in data[table]:
item['token'] = random_api_key()
def do_import_realm(import_dir: Path, subdomain: str) -> Realm:
logging.info("Importing realm dump %s" % (import_dir, ))
if not os.path.exists(import_dir):
raise Exception("Missing import directory!")
realm_data_filename = os.path.join(import_dir, "realm.json")
if not os.path.exists(realm_data_filename):
raise Exception("Missing realm.json file!")
logging.info("Importing realm data from %s" % (realm_data_filename, ))
with open(realm_data_filename) as f:
data = ujson.load(f)
update_model_ids(Stream, data, 'zerver_stream', 'stream')
re_map_foreign_keys(data,
'zerver_realm',
'notifications_stream',
related_table="stream")
fix_datetime_fields(data, 'zerver_realm')
# Fix realm subdomain information
data['zerver_realm'][0]['string_id'] = subdomain
data['zerver_realm'][0]['name'] = subdomain
fix_realm_authentication_bitfield(data, 'zerver_realm',
'authentication_methods')
update_model_ids(Realm, data, 'zerver_realm', 'realm')
realm = Realm(**data['zerver_realm'][0])
if realm.notifications_stream_id is not None:
notifications_stream_id = int(
realm.notifications_stream_id) # type: Optional[int]
else:
notifications_stream_id = None
realm.notifications_stream_id = None
realm.save()
bulk_import_client(data, Client, 'zerver_client')
# Email tokens will automatically be randomly generated when the
# Stream objects are created by Django.
fix_datetime_fields(data, 'zerver_stream')
re_map_foreign_keys(data, 'zerver_stream', 'realm', related_table="realm")
bulk_import_model(data, Stream, 'zerver_stream')
realm.notifications_stream_id = notifications_stream_id
realm.save()
re_map_foreign_keys(data,
'zerver_defaultstream',
'stream',
related_table="stream")
re_map_foreign_keys(data,
'zerver_realmemoji',
'author',
related_table="user_profile")
for (table, model, related_table) in realm_tables:
re_map_foreign_keys(data, table, 'realm', related_table="realm")
update_model_ids(model, data, table, related_table)
bulk_import_model(data, model, table)
# Remap the user IDs for notification_bot and friends to their
# appropriate IDs on this server
for item in data['zerver_userprofile_crossrealm']:
logging.info("Adding to ID map: %s %s" %
(item['id'], get_system_bot(item['email']).id))
new_user_id = get_system_bot(item['email']).id
update_id_map(table='user_profile',
old_id=item['id'],
new_id=new_user_id)
# Merge in zerver_userprofile_mirrordummy
data['zerver_userprofile'] = data['zerver_userprofile'] + data[
'zerver_userprofile_mirrordummy']
del data['zerver_userprofile_mirrordummy']
data['zerver_userprofile'].sort(key=lambda r: r['id'])
# To remap foreign key for UserProfile.last_active_message_id
update_message_foreign_keys(import_dir)
fix_datetime_fields(data, 'zerver_userprofile')
update_model_ids(UserProfile, data, 'zerver_userprofile', 'user_profile')
re_map_foreign_keys(data,
'zerver_userprofile',
'realm',
related_table="realm")
re_map_foreign_keys(data,
'zerver_userprofile',
'bot_owner',
related_table="user_profile")
re_map_foreign_keys(data,
'zerver_userprofile',
'default_sending_stream',
related_table="stream")
re_map_foreign_keys(data,
'zerver_userprofile',
'default_events_register_stream',
related_table="stream")
re_map_foreign_keys(data,
'zerver_userprofile',
'last_active_message_id',
related_table="message",
id_field=True)
for user_profile_dict in data['zerver_userprofile']:
user_profile_dict['password'] = None
user_profile_dict['api_key'] = random_api_key()
# Since Zulip doesn't use these permissions, drop them
del user_profile_dict['user_permissions']
del user_profile_dict['groups']
user_profiles = [
UserProfile(**item) for item in data['zerver_userprofile']
]
for user_profile in user_profiles:
user_profile.set_unusable_password()
UserProfile.objects.bulk_create(user_profiles)
if 'zerver_huddle' in data:
bulk_import_model(data, Huddle, 'zerver_huddle')
re_map_foreign_keys(data,
'zerver_recipient',
'type_id',
related_table="stream",
recipient_field=True,
id_field=True)
re_map_foreign_keys(data,
'zerver_recipient',
'type_id',
related_table="user_profile",
recipient_field=True,
id_field=True)
update_model_ids(Recipient, data, 'zerver_recipient', 'recipient')
bulk_import_model(data, Recipient, 'zerver_recipient')
re_map_foreign_keys(data,
'zerver_subscription',
'user_profile',
related_table="user_profile")
re_map_foreign_keys(data,
'zerver_subscription',
'recipient',
related_table="recipient")
update_model_ids(Subscription, data, 'zerver_subscription', 'subscription')
bulk_import_model(data, Subscription, 'zerver_subscription')
fix_datetime_fields(data, 'zerver_userpresence')
re_map_foreign_keys(data,
'zerver_userpresence',
'user_profile',
related_table="user_profile")
re_map_foreign_keys(data,
'zerver_userpresence',
'client',
related_table='client')
update_model_ids(UserPresence, data, 'zerver_userpresence',
'user_presence')
bulk_import_model(data, UserPresence, 'zerver_userpresence')
fix_datetime_fields(data, 'zerver_useractivity')
re_map_foreign_keys(data,
'zerver_useractivity',
'user_profile',
related_table="user_profile")
re_map_foreign_keys(data,
'zerver_useractivity',
'client',
related_table='client')
update_model_ids(UserActivity, data, 'zerver_useractivity', 'useractivity')
bulk_import_model(data, UserActivity, 'zerver_useractivity')
fix_datetime_fields(data, 'zerver_useractivityinterval')
re_map_foreign_keys(data,
'zerver_useractivityinterval',
'user_profile',
related_table="user_profile")
update_model_ids(UserActivityInterval, data, 'zerver_useractivityinterval',
'useractivityinterval')
bulk_import_model(data, UserActivityInterval,
'zerver_useractivityinterval')
if 'zerver_customprofilefield' in data:
# As the export of Custom Profile fields is not supported, Zulip exported
# data would not contain this field.
# However this is supported in slack importer script
re_map_foreign_keys(data,
'zerver_customprofilefield',
'realm',
related_table="realm")
update_model_ids(CustomProfileField,
data,
'zerver_customprofilefield',
related_table="customprofilefield")
bulk_import_model(data, CustomProfileField,
'zerver_customprofilefield')
re_map_foreign_keys(data,
'zerver_customprofilefield_value',
'user_profile',
related_table="user_profile")
re_map_foreign_keys(data,
'zerver_customprofilefield_value',
'field',
related_table="customprofilefield")
update_model_ids(CustomProfileFieldValue,
data,
'zerver_customprofilefield_value',
related_table="customprofilefield_value")
bulk_import_model(data, CustomProfileFieldValue,
'zerver_customprofilefield_value')
# Import uploaded files and avatars
import_uploads(os.path.join(import_dir, "avatars"),
processing_avatars=True)
import_uploads(os.path.join(import_dir, "uploads"))
# We need to have this check as the emoji files are only present in the data
# importer from slack
# For Zulip export, this doesn't exist
if os.path.exists(os.path.join(import_dir, "emoji")):
import_uploads(os.path.join(import_dir, "emoji"),
processing_emojis=True)
# Import zerver_message and zerver_usermessage
import_message_data(import_dir)
# Do attachments AFTER message data is loaded.
# TODO: de-dup how we read these json files.
fn = os.path.join(import_dir, "attachment.json")
if not os.path.exists(fn):
raise Exception("Missing attachment.json file!")
logging.info("Importing attachment data from %s" % (fn, ))
with open(fn) as f:
data = ujson.load(f)
import_attachments(data)
return realm
def do_import_realm(import_dir: Path, subdomain: str) -> Realm:
logging.info("Importing realm dump %s" % (import_dir,))
if not os.path.exists(import_dir):
raise Exception("Missing import directory!")
realm_data_filename = os.path.join(import_dir, "realm.json")
if not os.path.exists(realm_data_filename):
raise Exception("Missing realm.json file!")
logging.info("Importing realm data from %s" % (realm_data_filename,))
with open(realm_data_filename) as f:
data = ujson.load(f)
update_model_ids(Stream, data, 'zerver_stream', 'stream')
re_map_foreign_keys(data, 'zerver_realm', 'notifications_stream', related_table="stream")
fix_datetime_fields(data, 'zerver_realm')
# Fix realm subdomain information
data['zerver_realm'][0]['string_id'] = subdomain
data['zerver_realm'][0]['name'] = subdomain
fix_realm_authentication_bitfield(data, 'zerver_realm', 'authentication_methods')
update_model_ids(Realm, data, 'zerver_realm', 'realm')
realm = Realm(**data['zerver_realm'][0])
if realm.notifications_stream_id is not None:
notifications_stream_id = int(realm.notifications_stream_id) # type: Optional[int]
else:
notifications_stream_id = None
realm.notifications_stream_id = None
realm.save()
bulk_import_client(data, Client, 'zerver_client')
# Email tokens will automatically be randomly generated when the
# Stream objects are created by Django.
fix_datetime_fields(data, 'zerver_stream')
re_map_foreign_keys(data, 'zerver_stream', 'realm', related_table="realm")
bulk_import_model(data, Stream, 'zerver_stream')
realm.notifications_stream_id = notifications_stream_id
realm.save()
re_map_foreign_keys(data, 'zerver_defaultstream', 'stream', related_table="stream")
re_map_foreign_keys(data, 'zerver_realmemoji', 'author', related_table="user_profile")
for (table, model, related_table) in realm_tables:
re_map_foreign_keys(data, table, 'realm', related_table="realm")
update_model_ids(model, data, table, related_table)
bulk_import_model(data, model, table)
# Remap the user IDs for notification_bot and friends to their
# appropriate IDs on this server
for item in data['zerver_userprofile_crossrealm']:
logging.info("Adding to ID map: %s %s" % (item['id'], get_system_bot(item['email']).id))
new_user_id = get_system_bot(item['email']).id
update_id_map(table='user_profile', old_id=item['id'], new_id=new_user_id)
# Merge in zerver_userprofile_mirrordummy
data['zerver_userprofile'] = data['zerver_userprofile'] + data['zerver_userprofile_mirrordummy']
del data['zerver_userprofile_mirrordummy']
data['zerver_userprofile'].sort(key=lambda r: r['id'])
# To remap foreign key for UserProfile.last_active_message_id
update_message_foreign_keys(import_dir)
fix_datetime_fields(data, 'zerver_userprofile')
update_model_ids(UserProfile, data, 'zerver_userprofile', 'user_profile')
re_map_foreign_keys(data, 'zerver_userprofile', 'realm', related_table="realm")
re_map_foreign_keys(data, 'zerver_userprofile', 'bot_owner', related_table="user_profile")
re_map_foreign_keys(data, 'zerver_userprofile', 'default_sending_stream',
related_table="stream")
re_map_foreign_keys(data, 'zerver_userprofile', 'default_events_register_stream',
related_table="stream")
re_map_foreign_keys(data, 'zerver_userprofile', 'last_active_message_id',
related_table="message", id_field=True)
for user_profile_dict in data['zerver_userprofile']:
user_profile_dict['password'] = None
user_profile_dict['api_key'] = random_api_key()
# Since Zulip doesn't use these permissions, drop them
del user_profile_dict['user_permissions']
del user_profile_dict['groups']
user_profiles = [UserProfile(**item) for item in data['zerver_userprofile']]
for user_profile in user_profiles:
user_profile.set_unusable_password()
UserProfile.objects.bulk_create(user_profiles)
if 'zerver_huddle' in data:
bulk_import_model(data, Huddle, 'zerver_huddle')
re_map_foreign_keys(data, 'zerver_recipient', 'type_id', related_table="stream",
recipient_field=True, id_field=True)
re_map_foreign_keys(data, 'zerver_recipient', 'type_id', related_table="user_profile",
recipient_field=True, id_field=True)
update_model_ids(Recipient, data, 'zerver_recipient', 'recipient')
bulk_import_model(data, Recipient, 'zerver_recipient')
re_map_foreign_keys(data, 'zerver_subscription', 'user_profile', related_table="user_profile")
re_map_foreign_keys(data, 'zerver_subscription', 'recipient', related_table="recipient")
update_model_ids(Subscription, data, 'zerver_subscription', 'subscription')
bulk_import_model(data, Subscription, 'zerver_subscription')
fix_datetime_fields(data, 'zerver_userpresence')
re_map_foreign_keys(data, 'zerver_userpresence', 'user_profile', related_table="user_profile")
re_map_foreign_keys(data, 'zerver_userpresence', 'client', related_table='client')
update_model_ids(UserPresence, data, 'zerver_userpresence', 'user_presence')
bulk_import_model(data, UserPresence, 'zerver_userpresence')
fix_datetime_fields(data, 'zerver_useractivity')
re_map_foreign_keys(data, 'zerver_useractivity', 'user_profile', related_table="user_profile")
re_map_foreign_keys(data, 'zerver_useractivity', 'client', related_table='client')
update_model_ids(UserActivity, data, 'zerver_useractivity', 'useractivity')
bulk_import_model(data, UserActivity, 'zerver_useractivity')
fix_datetime_fields(data, 'zerver_useractivityinterval')
re_map_foreign_keys(data, 'zerver_useractivityinterval', 'user_profile', related_table="user_profile")
update_model_ids(UserActivityInterval, data, 'zerver_useractivityinterval',
'useractivityinterval')
bulk_import_model(data, UserActivityInterval, 'zerver_useractivityinterval')
if 'zerver_customprofilefield' in data:
# As the export of Custom Profile fields is not supported, Zulip exported
# data would not contain this field.
# However this is supported in slack importer script
re_map_foreign_keys(data, 'zerver_customprofilefield', 'realm', related_table="realm")
update_model_ids(CustomProfileField, data, 'zerver_customprofilefield',
related_table="customprofilefield")
bulk_import_model(data, CustomProfileField, 'zerver_customprofilefield')
re_map_foreign_keys(data, 'zerver_customprofilefield_value', 'user_profile',
related_table="user_profile")
re_map_foreign_keys(data, 'zerver_customprofilefield_value', 'field',
related_table="customprofilefield")
update_model_ids(CustomProfileFieldValue, data, 'zerver_customprofilefield_value',
related_table="customprofilefield_value")
bulk_import_model(data, CustomProfileFieldValue, 'zerver_customprofilefield_value')
# Import uploaded files and avatars
import_uploads(os.path.join(import_dir, "avatars"), processing_avatars=True)
import_uploads(os.path.join(import_dir, "uploads"))
# We need to have this check as the emoji files are only present in the data
# importer from slack
# For Zulip export, this doesn't exist
if os.path.exists(os.path.join(import_dir, "emoji")):
import_uploads(os.path.join(import_dir, "emoji"), processing_emojis=True)
# Import zerver_message and zerver_usermessage
import_message_data(import_dir)
# Do attachments AFTER message data is loaded.
# TODO: de-dup how we read these json files.
fn = os.path.join(import_dir, "attachment.json")
if not os.path.exists(fn):
raise Exception("Missing attachment.json file!")
logging.info("Importing attachment data from %s" % (fn,))
with open(fn) as f:
data = ujson.load(f)
import_attachments(data)
return realm
def add_bot_backend(request,
user_profile,
full_name_raw=REQ("full_name"),
short_name_raw=REQ("short_name"),
bot_type=REQ(validator=check_int,
default=UserProfile.DEFAULT_BOT),
payload_url=REQ(validator=check_url, default=""),
service_name=REQ(default=None),
interface_type=REQ(validator=check_int,
default=Service.GENERIC),
default_sending_stream_name=REQ('default_sending_stream',
default=None),
default_events_register_stream_name=REQ(
'default_events_register_stream', default=None),
default_all_public_streams=REQ(validator=check_bool,
default=None)):
# type: (HttpRequest, UserProfile, Text, Text, int, Optional[Text], Optional[Text], int, Optional[Text], Optional[Text], Optional[bool]) -> HttpResponse
short_name = check_short_name(short_name_raw)
service_name = service_name or short_name
short_name += "-bot"
full_name = check_full_name(full_name_raw)
email = '%s@%s' % (short_name, user_profile.realm.get_bot_domain())
form = CreateUserForm({'full_name': full_name, 'email': email})
if bot_type == UserProfile.EMBEDDED_BOT:
if not settings.EMBEDDED_BOTS_ENABLED:
return json_error(_("Embedded bots are not enabled."))
if service_name not in [bot.name for bot in EMBEDDED_BOTS]:
return json_error(_("Invalid embedded bot name."))
if not form.is_valid():
# We validate client-side as well
return json_error(_('Bad name or username'))
try:
get_user(email, user_profile.realm)
return json_error(_("Username already in use"))
except UserProfile.DoesNotExist:
pass
check_valid_bot_type(bot_type)
check_valid_interface_type(interface_type)
if len(request.FILES) == 0:
avatar_source = UserProfile.AVATAR_FROM_GRAVATAR
elif len(request.FILES) != 1:
return json_error(_("You may only upload one file at a time"))
else:
avatar_source = UserProfile.AVATAR_FROM_USER
default_sending_stream = None
if default_sending_stream_name is not None:
(default_sending_stream, ignored_rec,
ignored_sub) = access_stream_by_name(user_profile,
default_sending_stream_name)
default_events_register_stream = None
if default_events_register_stream_name is not None:
(default_events_register_stream, ignored_rec,
ignored_sub) = access_stream_by_name(
user_profile, default_events_register_stream_name)
bot_profile = do_create_user(
email=email,
password='',
realm=user_profile.realm,
full_name=full_name,
short_name=short_name,
active=True,
bot_type=bot_type,
bot_owner=user_profile,
avatar_source=avatar_source,
default_sending_stream=default_sending_stream,
default_events_register_stream=default_events_register_stream,
default_all_public_streams=default_all_public_streams)
if len(request.FILES) == 1:
user_file = list(request.FILES.values())[0]
upload_avatar_image(user_file, user_profile, bot_profile)
if bot_type in (UserProfile.OUTGOING_WEBHOOK_BOT,
UserProfile.EMBEDDED_BOT):
add_service(name=service_name,
user_profile=bot_profile,
base_url=payload_url,
interface=interface_type,
token=random_api_key())
json_result = dict(
api_key=bot_profile.api_key,
avatar_url=avatar_url(bot_profile),
default_sending_stream=get_stream_name(
bot_profile.default_sending_stream),
default_events_register_stream=get_stream_name(
bot_profile.default_events_register_stream),
default_all_public_streams=bot_profile.default_all_public_streams,
)
return json_success(json_result)
def add_bot_backend(
request: HttpRequest, user_profile: UserProfile,
full_name_raw: Text=REQ("full_name"), short_name_raw: Text=REQ("short_name"),
bot_type: int=REQ(validator=check_int, default=UserProfile.DEFAULT_BOT),
payload_url: Optional[Text]=REQ(validator=check_url, default=""),
service_name: Optional[Text]=REQ(default=None),
config_data: Optional[Dict[Text, Text]]=REQ(default=None,
validator=check_dict(value_validator=check_string)),
interface_type: int=REQ(validator=check_int, default=Service.GENERIC),
default_sending_stream_name: Optional[Text]=REQ('default_sending_stream', default=None),
default_events_register_stream_name: Optional[Text]=REQ('default_events_register_stream',
default=None),
default_all_public_streams: Optional[bool]=REQ(validator=check_bool, default=None)
) -> HttpResponse:
short_name = check_short_name(short_name_raw)
service_name = service_name or short_name
short_name += "-bot"
full_name = check_full_name(full_name_raw)
email = '%s@%s' % (short_name, user_profile.realm.get_bot_domain())
form = CreateUserForm({'full_name': full_name, 'email': email})
if bot_type == UserProfile.EMBEDDED_BOT:
if not settings.EMBEDDED_BOTS_ENABLED:
return json_error(_("Embedded bots are not enabled."))
if service_name not in [bot.name for bot in EMBEDDED_BOTS]:
return json_error(_("Invalid embedded bot name."))
if not form.is_valid():
# We validate client-side as well
return json_error(_('Bad name or username'))
try:
get_user(email, user_profile.realm)
return json_error(_("Username already in use"))
except UserProfile.DoesNotExist:
pass
check_valid_bot_type(user_profile, bot_type)
check_valid_interface_type(interface_type)
if len(request.FILES) == 0:
avatar_source = UserProfile.AVATAR_FROM_GRAVATAR
elif len(request.FILES) != 1:
return json_error(_("You may only upload one file at a time"))
else:
avatar_source = UserProfile.AVATAR_FROM_USER
default_sending_stream = None
if default_sending_stream_name is not None:
(default_sending_stream, ignored_rec, ignored_sub) = access_stream_by_name(
user_profile, default_sending_stream_name)
default_events_register_stream = None
if default_events_register_stream_name is not None:
(default_events_register_stream, ignored_rec, ignored_sub) = access_stream_by_name(
user_profile, default_events_register_stream_name)
bot_profile = do_create_user(email=email, password='',
realm=user_profile.realm, full_name=full_name,
short_name=short_name,
bot_type=bot_type,
bot_owner=user_profile,
avatar_source=avatar_source,
default_sending_stream=default_sending_stream,
default_events_register_stream=default_events_register_stream,
default_all_public_streams=default_all_public_streams)
if len(request.FILES) == 1:
user_file = list(request.FILES.values())[0]
upload_avatar_image(user_file, user_profile, bot_profile)
if bot_type in (UserProfile.OUTGOING_WEBHOOK_BOT, UserProfile.EMBEDDED_BOT):
add_service(name=service_name,
user_profile=bot_profile,
base_url=payload_url,
interface=interface_type,
token=random_api_key())
if bot_type == UserProfile.EMBEDDED_BOT:
for key, value in config_data.items():
set_bot_config(bot_profile, key, value)
json_result = dict(
api_key=bot_profile.api_key,
avatar_url=avatar_url(bot_profile),
default_sending_stream=get_stream_name(bot_profile.default_sending_stream),
default_events_register_stream=get_stream_name(bot_profile.default_events_register_stream),
default_all_public_streams=bot_profile.default_all_public_streams,
)
return json_success(json_result)
def do_import_realm(import_dir):
# type: (Path) -> None
logging.info("Importing realm dump %s" % (import_dir,))
if not os.path.exists(import_dir):
raise Exception("Missing import directory!")
realm_data_filename = os.path.join(import_dir, "realm.json")
if not os.path.exists(realm_data_filename):
raise Exception("Missing realm.json file!")
logging.info("Importing realm data from %s" % (realm_data_filename,))
with open(realm_data_filename) as f:
data = ujson.load(f)
convert_to_id_fields(data, 'zerver_realm', 'notifications_stream')
fix_datetime_fields(data, 'zerver_realm')
realm = Realm(**data['zerver_realm'][0])
if realm.notifications_stream_id is not None:
notifications_stream_id = int(realm.notifications_stream_id)
else:
notifications_stream_id = None
realm.notifications_stream_id = None
realm.save()
bulk_import_client(data, Client, 'zerver_client')
# Email tokens will automatically be randomly generated when the
# Stream objects are created by Django.
fix_datetime_fields(data, 'zerver_stream')
convert_to_id_fields(data, 'zerver_stream', 'realm')
bulk_import_model(data, Stream, 'zerver_stream')
realm.notifications_stream_id = notifications_stream_id
realm.save()
convert_to_id_fields(data, "zerver_defaultstream", 'stream')
for (table, model) in realm_tables:
convert_to_id_fields(data, table, 'realm')
bulk_import_model(data, model, table)
# Remap the user IDs for notification_bot and friends to their
# appropriate IDs on this server
for item in data['zerver_userprofile_crossrealm']:
logging.info("Adding to ID map: %s %s" % (item['id'], get_user_profile_by_email(item['email']).id))
new_user_id = get_user_profile_by_email(item['email']).id
update_id_map(table='user_profile', old_id=item['id'], new_id=new_user_id)
fix_datetime_fields(data, 'zerver_userprofile')
convert_to_id_fields(data, 'zerver_userprofile', 'realm')
re_map_foreign_keys(data, 'zerver_userprofile', 'bot_owner', related_table="user_profile")
convert_to_id_fields(data, 'zerver_userprofile', 'default_sending_stream')
convert_to_id_fields(data, 'zerver_userprofile', 'default_events_register_stream')
for user_profile_dict in data['zerver_userprofile']:
user_profile_dict['password'] = None
user_profile_dict['api_key'] = random_api_key()
# Since Zulip doesn't use these permissions, drop them
del user_profile_dict['user_permissions']
del user_profile_dict['groups']
user_profiles = [UserProfile(**item) for item in data['zerver_userprofile']]
for user_profile in user_profiles:
user_profile.set_unusable_password()
UserProfile.objects.bulk_create(user_profiles)
if 'zerver_huddle' in data:
bulk_import_model(data, Huddle, 'zerver_huddle')
bulk_import_model(data, Recipient, 'zerver_recipient')
re_map_foreign_keys(data, 'zerver_subscription', 'user_profile', related_table="user_profile")
convert_to_id_fields(data, 'zerver_subscription', 'recipient')
bulk_import_model(data, Subscription, 'zerver_subscription')
fix_datetime_fields(data, 'zerver_userpresence')
re_map_foreign_keys(data, 'zerver_userpresence', 'user_profile', related_table="user_profile")
re_map_foreign_keys(data, 'zerver_userpresence', 'client', related_table='client')
bulk_import_model(data, UserPresence, 'zerver_userpresence')
fix_datetime_fields(data, 'zerver_useractivity')
re_map_foreign_keys(data, 'zerver_useractivity', 'user_profile', related_table="user_profile")
re_map_foreign_keys(data, 'zerver_useractivity', 'client', related_table='client')
bulk_import_model(data, UserActivity, 'zerver_useractivity')
fix_datetime_fields(data, 'zerver_useractivityinterval')
re_map_foreign_keys(data, 'zerver_useractivityinterval', 'user_profile', related_table="user_profile")
bulk_import_model(data, UserActivityInterval, 'zerver_useractivityinterval')
# Import uploaded files and avatars
import_uploads(os.path.join(import_dir, "avatars"), avatar_bucket=True)
import_uploads(os.path.join(import_dir, "uploads"))
# Import zerver_message and zerver_usermessage
import_message_data(import_dir)
# Do attachments AFTER message data is loaded.
import_attachments(data)
def do_import_realm(import_dir):
logging.info("Importing realm dump %s" % (import_dir,))
if not os.path.exists(import_dir):
raise Exception("Missing import directory!")
realm_data_filename = os.path.join(import_dir, "realm.json")
if not os.path.exists(realm_data_filename):
raise Exception("Missing realm.json file!")
logging.info("Importing realm data from %s" % (realm_data_filename,))
with open(realm_data_filename) as f:
data = ujson.load(f)
fix_foreign_keys(data, 'zerver_realm', 'notifications_stream')
fix_datetime_fields(data, 'zerver_realm', 'date_created')
realm = Realm(**data['zerver_realm'][0])
if realm.notifications_stream_id is not None:
notifications_stream_id = int(realm.notifications_stream_id)
else:
notifications_stream_id = None
realm.notifications_stream_id = None
realm.save()
bulk_import_client(data, Client, 'zerver_client')
# Email tokens will automatically be randomly generated when the
# Stream objects are created by Django.
fix_datetime_fields(data, 'zerver_stream', 'date_created')
fix_foreign_keys(data, 'zerver_stream', 'realm')
bulk_import_model(data, Stream, 'zerver_stream')
realm.notifications_stream_id = notifications_stream_id
realm.save()
fix_foreign_keys(data, "zerver_defaultstream", 'stream')
for (table, model) in realm_tables:
fix_foreign_keys(data, table, 'realm')
bulk_import_model(data, model, table)
# Remap the user IDs for notification_bot and friends to their
# appropriate IDs on this server
for item in data['zerver_userprofile_crossrealm']:
logging.info("Adding to ID map: %s %s" % (item['id'], get_user_profile_by_email(item['email']).id))
id_maps["user_profile"][item['id']] = get_user_profile_by_email(item['email']).id
fix_datetime_fields(data, 'zerver_userprofile', 'date_joined')
fix_datetime_fields(data, 'zerver_userprofile', 'last_login')
fix_datetime_fields(data, 'zerver_userprofile', 'last_reminder')
fix_foreign_keys(data, 'zerver_userprofile', 'realm')
fix_foreign_keys(data, 'zerver_userprofile', 'bot_owner', id_map_table="user_profile")
fix_foreign_keys(data, 'zerver_userprofile', 'default_sending_stream')
fix_foreign_keys(data, 'zerver_userprofile', 'default_events_register_stream')
for user_profile_dict in data['zerver_userprofile']:
user_profile_dict['password'] = None
user_profile_dict['api_key'] = random_api_key()
# Since Zulip doesn't use these permissions, drop them
del user_profile_dict['user_permissions']
del user_profile_dict['groups']
user_profiles = [UserProfile(**item) for item in data['zerver_userprofile']]
for user_profile in user_profiles:
user_profile.set_unusable_password()
UserProfile.objects.bulk_create(user_profiles)
if 'zerver_huddle' in data:
bulk_import_model(data, Huddle, 'zerver_huddle')
bulk_import_model(data, Recipient, 'zerver_recipient')
fix_foreign_keys(data, 'zerver_subscription', 'user_profile', id_map_table="user_profile")
fix_foreign_keys(data, 'zerver_subscription', 'recipient')
bulk_import_model(data, Subscription, 'zerver_subscription')
fix_datetime_fields(data, 'zerver_userpresence', 'timestamp')
fix_foreign_keys(data, 'zerver_userpresence', 'user_profile', id_map_table="user_profile")
fix_foreign_keys(data, 'zerver_userpresence', 'client', id_map_table='client')
bulk_import_model(data, UserPresence, 'zerver_userpresence')
fix_datetime_fields(data, 'zerver_useractivity', 'last_visit')
fix_foreign_keys(data, 'zerver_useractivity', 'user_profile', id_map_table="user_profile")
fix_foreign_keys(data, 'zerver_useractivity', 'client', id_map_table='client')
bulk_import_model(data, UserActivity, 'zerver_useractivity')
fix_datetime_fields(data, 'zerver_useractivityinterval', 'start')
fix_datetime_fields(data, 'zerver_useractivityinterval', 'end')
fix_foreign_keys(data, 'zerver_useractivityinterval', 'user_profile', id_map_table="user_profile")
bulk_import_model(data, UserActivityInterval, 'zerver_useractivityinterval')
# Import uploaded files and avatars
import_uploads(os.path.join(import_dir, "avatars"), avatar_bucket=True)
import_uploads(os.path.join(import_dir, "uploads"))
dump_file_id = 1
while True:
message_filename = os.path.join(import_dir, "messages-%06d.json" % (dump_file_id,))
if not os.path.exists(message_filename):
break
with open(message_filename) as f:
data = ujson.load(f)
logging.info("Importing message dump %s" % (message_filename,))
fix_foreign_keys(data, 'zerver_message', 'sender', id_map_table="user_profile")
fix_foreign_keys(data, 'zerver_message', 'recipient')
fix_foreign_keys(data, 'zerver_message', 'sending_client', id_map_table='client')
fix_datetime_fields(data, 'zerver_message', 'pub_date')
fix_datetime_fields(data, 'zerver_message', 'last_edit_time')
bulk_import_model(data, Message, 'zerver_message')
# Due to the structure of these message chunks, we're
# guaranteed to have already imported all the Message objects
# for this batch of UserMessage objects.
fix_foreign_keys(data, 'zerver_usermessage', 'message')
fix_foreign_keys(data, 'zerver_usermessage', 'user_profile', id_map_table="user_profile")
fix_bitfield_keys(data, 'zerver_usermessage', 'flags')
bulk_import_model(data, UserMessage, 'zerver_usermessage')
dump_file_id += 1
fix_datetime_fields(data, 'zerver_attachment', 'create_time')
fix_foreign_keys(data, 'zerver_attachment', 'owner', id_map_table="user_profile")
fix_foreign_keys(data, 'zerver_attachment', 'realm')
# TODO: Handle the `messages` keys.
# fix_foreign_keys(data, 'zerver_attachment', 'messages')
bulk_import_model(data, Attachment, 'zerver_attachment')
def handle(self, **options: Any) -> None:
if options["percent_huddles"] + options["percent_personals"] > 100:
self.stderr.write(
"Error! More than 100% of messages allocated.\n")
return
# Get consistent data for backend tests.
if options["test_suite"]:
random.seed(0)
if options["delete"]:
# Start by clearing all the data in our database
clear_database()
# Create our two default realms
# Could in theory be done via zerver.lib.actions.do_create_realm, but
# welcome-bot (needed for do_create_realm) hasn't been created yet
zulip_realm = Realm.objects.create(
string_id="zulip",
name="Zulip Dev",
restricted_to_domain=True,
description=
"The Zulip development environment default organization."
" It's great for testing!",
invite_required=False,
org_type=Realm.CORPORATE)
RealmDomain.objects.create(realm=zulip_realm, domain="zulip.com")
if options["test_suite"]:
mit_realm = Realm.objects.create(string_id="zephyr",
name="MIT",
restricted_to_domain=True,
invite_required=False,
org_type=Realm.CORPORATE)
RealmDomain.objects.create(realm=mit_realm, domain="mit.edu")
lear_realm = Realm.objects.create(string_id="lear",
name="Lear & Co.",
restricted_to_domain=False,
invite_required=False,
org_type=Realm.CORPORATE)
# Create test Users (UserProfiles are automatically created,
# as are subscriptions to the ability to receive personals).
names = [
("Zoe", "*****@*****.**"),
("Othello, the Moor of Venice", "*****@*****.**"),
("Iago", "*****@*****.**"),
("Prospero from The Tempest", "*****@*****.**"),
("Cordelia Lear", "*****@*****.**"),
("King Hamlet", "*****@*****.**"),
("aaron", "*****@*****.**"),
("Polonius", "*****@*****.**"),
]
for i in range(options["extra_users"]):
names.append(
('Extra User %d' % (i, ), '*****@*****.**' % (i, )))
create_users(zulip_realm, names)
iago = get_user("*****@*****.**", zulip_realm)
do_change_is_admin(iago, True)
iago.is_staff = True
iago.save(update_fields=['is_staff'])
guest_user = get_user("*****@*****.**", zulip_realm)
guest_user.is_guest = True
guest_user.save(update_fields=['is_guest'])
# These bots are directly referenced from code and thus
# are needed for the test suite.
all_realm_bots = [
(bot['name'],
bot['email_template'] % (settings.INTERNAL_BOT_DOMAIN, ))
for bot in settings.INTERNAL_BOTS
]
zulip_realm_bots = [
("Zulip New User Bot", "*****@*****.**"),
("Zulip Error Bot", "*****@*****.**"),
("Zulip Default Bot", "*****@*****.**"),
("Welcome Bot", "*****@*****.**"),
]
for i in range(options["extra_bots"]):
zulip_realm_bots.append(
('Extra Bot %d' % (i, ), '*****@*****.**' % (i, )))
zulip_realm_bots.extend(all_realm_bots)
create_users(zulip_realm,
zulip_realm_bots,
bot_type=UserProfile.DEFAULT_BOT)
# Initialize the email gateway bot as an API Super User
email_gateway_bot = get_system_bot(settings.EMAIL_GATEWAY_BOT)
email_gateway_bot.is_api_super_user = True
email_gateway_bot.save()
zoe = get_user("*****@*****.**", zulip_realm)
zulip_webhook_bots = [
("Zulip Webhook Bot", "*****@*****.**"),
]
# If a stream is not supplied in the webhook URL, the webhook
# will (in some cases) send the notification as a PM to the
# owner of the webhook bot, so bot_owner can't be None
create_users(zulip_realm,
zulip_webhook_bots,
bot_type=UserProfile.INCOMING_WEBHOOK_BOT,
bot_owner=zoe)
aaron = get_user("*****@*****.**", zulip_realm)
zulip_outgoing_bots = [("Outgoing Webhook",
"*****@*****.**")]
create_users(zulip_realm,
zulip_outgoing_bots,
bot_type=UserProfile.OUTGOING_WEBHOOK_BOT,
bot_owner=aaron)
outgoing_webhook = get_user("*****@*****.**",
zulip_realm)
add_service("outgoing-webhook",
user_profile=outgoing_webhook,
interface=Service.GENERIC,
base_url="http://127.0.0.1:5002",
token=random_api_key())
# Add the realm internl bots to each realm.
create_if_missing_realm_internal_bots()
# Create public streams.
stream_list = ["Verona", "Denmark", "Scotland", "Venice", "Rome"]
stream_dict = {
"Verona": {
"description": "A city in Italy"
},
"Denmark": {
"description": "A Scandinavian country"
},
"Scotland": {
"description": "Located in the United Kingdom"
},
"Venice": {
"description": "A northeastern Italian city"
},
"Rome": {
"description": "Yet another Italian city",
"is_web_public": True
}
} # type: Dict[str, Dict[str, Any]]
bulk_create_streams(zulip_realm, stream_dict)
recipient_streams = [
Stream.objects.get(name=name, realm=zulip_realm).id
for name in stream_list
] # type: List[int]
# Create subscriptions to streams. The following
# algorithm will give each of the users a different but
# deterministic subset of the streams (given a fixed list
# of users). For the test suite, we have a fixed list of
# subscriptions to make sure test data is consistent
# across platforms.
subscriptions_list = [
] # type: List[Tuple[UserProfile, Recipient]]
profiles = UserProfile.objects.select_related().filter(
is_bot=False).order_by("email") # type: Sequence[UserProfile]
if options["test_suite"]:
subscriptions_map = {
'*****@*****.**': ['Verona'],
'*****@*****.**': ['Verona'],
'*****@*****.**': ['Verona', 'Denmark'],
'*****@*****.**': ['Verona', 'Denmark', 'Scotland'],
'*****@*****.**': ['Verona', 'Denmark', 'Scotland'],
'*****@*****.**':
['Verona', 'Denmark', 'Scotland', 'Venice'],
'*****@*****.**':
['Verona', 'Denmark', 'Scotland', 'Venice', 'Rome'],
'*****@*****.**': ['Verona'],
}
for profile in profiles:
if profile.email not in subscriptions_map:
raise Exception(
'Subscriptions not listed for user %s' %
(profile.email, ))
for stream_name in subscriptions_map[profile.email]:
stream = Stream.objects.get(name=stream_name)
r = Recipient.objects.get(type=Recipient.STREAM,
type_id=stream.id)
subscriptions_list.append((profile, r))
else:
for i, profile in enumerate(profiles):
# Subscribe to some streams.
for type_id in recipient_streams[:int(
len(recipient_streams) * float(i) /
len(profiles)) + 1]:
r = Recipient.objects.get(type=Recipient.STREAM,
type_id=type_id)
subscriptions_list.append((profile, r))
subscriptions_to_add = [] # type: List[Subscription]
event_time = timezone_now()
all_subscription_logs = [] # type: (List[RealmAuditLog])
i = 0
for profile, recipient in subscriptions_list:
i += 1
color = STREAM_ASSIGNMENT_COLORS[i %
len(STREAM_ASSIGNMENT_COLORS)]
s = Subscription(recipient=recipient,
user_profile=profile,
color=color)
subscriptions_to_add.append(s)
log = RealmAuditLog(
realm=profile.realm,
modified_user=profile,
modified_stream_id=recipient.type_id,
event_last_message_id=0,
event_type=RealmAuditLog.SUBSCRIPTION_CREATED,
event_time=event_time)
all_subscription_logs.append(log)
Subscription.objects.bulk_create(subscriptions_to_add)
RealmAuditLog.objects.bulk_create(all_subscription_logs)
# Create custom profile field data
phone_number = try_add_realm_custom_profile_field(
zulip_realm,
"Phone number",
CustomProfileField.SHORT_TEXT,
hint='')
biography = try_add_realm_custom_profile_field(
zulip_realm,
"Biography",
CustomProfileField.LONG_TEXT,
hint='What are you known for?')
favorite_food = try_add_realm_custom_profile_field(
zulip_realm,
"Favorite food",
CustomProfileField.SHORT_TEXT,
hint="Or drink, if you'd prefer")
field_data = {
'vim': {
'text': 'Vim',
'order': '1'
},
'emacs': {
'text': 'Emacs',
'order': '2'
},
}
favorite_editor = try_add_realm_custom_profile_field(
zulip_realm,
"Favorite editor",
CustomProfileField.CHOICE,
field_data=field_data)
birthday = try_add_realm_custom_profile_field(
zulip_realm, "Birthday", CustomProfileField.DATE)
favorite_website = try_add_realm_custom_profile_field(
zulip_realm,
"GitHub profile",
CustomProfileField.URL,
hint="Or your personal blog's URL")
mentor = try_add_realm_custom_profile_field(
zulip_realm, "Mentor", CustomProfileField.USER)
# Fill in values for Iago and Hamlet
hamlet = get_user("*****@*****.**", zulip_realm)
do_update_user_custom_profile_data(iago, [
{
"id": phone_number.id,
"value": "+1-234-567-8901"
},
{
"id": biography.id,
"value": "Betrayer of Othello."
},
{
"id": favorite_food.id,
"value": "Apples"
},
{
"id": favorite_editor.id,
"value": "emacs"
},
{
"id": birthday.id,
"value": "2000-1-1"
},
{
"id": favorite_website.id,
"value": "https://github.com/zulip/zulip"
},
{
"id": mentor.id,
"value": [hamlet.id]
},
])
do_update_user_custom_profile_data(hamlet, [
{
"id": phone_number.id,
"value": "+0-11-23-456-7890"
},
{
"id": biography.id,
"value": "Prince of Denmark, and other things!"
},
{
"id": favorite_food.id,
"value": "Dark chocolate"
},
{
"id": favorite_editor.id,
"value": "vim"
},
{
"id": birthday.id,
"value": "1900-1-1"
},
{
"id": favorite_website.id,
"value": "https://blog.zulig.org"
},
{
"id": mentor.id,
"value": [iago.id]
},
])
else:
zulip_realm = get_realm("zulip")
recipient_streams = [
klass.type_id
for klass in Recipient.objects.filter(type=Recipient.STREAM)
]
# Extract a list of all users
user_profiles = list(UserProfile.objects.filter(
is_bot=False)) # type: List[UserProfile]
# Create a test realm emoji.
IMAGE_FILE_PATH = os.path.join(settings.STATIC_ROOT, 'images',
'test-images', 'checkbox.png')
with open(IMAGE_FILE_PATH, 'rb') as fp:
check_add_realm_emoji(zulip_realm, 'green_tick', iago, fp)
if not options["test_suite"]:
# Populate users with some bar data
for user in user_profiles:
status = UserPresence.ACTIVE # type: int
date = timezone_now()
client = get_client("website")
if user.full_name[0] <= 'H':
client = get_client("ZulipAndroid")
UserPresence.objects.get_or_create(user_profile=user,
client=client,
timestamp=date,
status=status)
user_profiles_ids = [user_profile.id for user_profile in user_profiles]
# Create several initial huddles
for i in range(options["num_huddles"]):
get_huddle(random.sample(user_profiles_ids, random.randint(3, 4)))
# Create several initial pairs for personals
personals_pairs = [
random.sample(user_profiles_ids, 2)
for i in range(options["num_personals"])
]
# Generate a new set of test data.
create_test_data()
# prepopulate the URL preview/embed data for the links present
# in the config.generate_data.json data set. This makes it
# possible for populate_db to run happily without Internet
# access.
with open("zerver/tests/fixtures/docs_url_preview_data.json",
"r") as f:
urls_with_preview_data = ujson.load(f)
for url in urls_with_preview_data:
cache_set(url, urls_with_preview_data[url], PREVIEW_CACHE_NAME)
threads = options["threads"]
jobs = [
] # type: List[Tuple[int, List[List[int]], Dict[str, Any], Callable[[str], int], int]]
for i in range(threads):
count = options["num_messages"] // threads
if i < options["num_messages"] % threads:
count += 1
jobs.append((count, personals_pairs, options, self.stdout.write,
random.randint(0, 10**10)))
for job in jobs:
send_messages(job)
if options["delete"]:
# Create the "website" and "API" clients; if we don't, the
# default values in zerver/decorators.py will not work
# with the Django test suite.
get_client("website")
get_client("API")
if options["test_suite"]:
# Create test users; the MIT ones are needed to test
# the Zephyr mirroring codepaths.
testsuite_mit_users = [
("Fred Sipb (MIT)", "*****@*****.**"),
("Athena Consulting Exchange User (MIT)",
"*****@*****.**"),
("Esp Classroom (MIT)", "*****@*****.**"),
]
create_users(mit_realm, testsuite_mit_users)
testsuite_lear_users = [
("King Lear", "*****@*****.**"),
("Cordelia Lear", "*****@*****.**"),
]
create_users(lear_realm, testsuite_lear_users)
if not options["test_suite"]:
# To keep the messages.json fixtures file for the test
# suite fast, don't add these users and subscriptions
# when running populate_db for the test suite
zulip_stream_dict = {
"devel": {
"description": "For developing"
},
"all": {
"description": "For everything"
},
"announce": {
"description": "For announcements",
'is_announcement_only': True
},
"design": {
"description": "For design"
},
"support": {
"description": "For support"
},
"social": {
"description": "For socializing"
},
"test": {
"description": "For testing"
},
"errors": {
"description": "For errors"
},
"sales": {
"description": "For sales discussion"
}
} # type: Dict[str, Dict[str, Any]]
# Calculate the maximum number of digits in any extra stream's
# number, since a stream with name "Extra Stream 3" could show
# up after "Extra Stream 29". (Used later to pad numbers with
# 0s).
maximum_digits = len(str(options['extra_streams'] - 1))
for i in range(options['extra_streams']):
# Pad the number with 0s based on `maximum_digits`.
number_str = str(i).zfill(maximum_digits)
extra_stream_name = 'Extra Stream ' + number_str
zulip_stream_dict[extra_stream_name] = {
"description": "Auto-generated extra stream.",
}
bulk_create_streams(zulip_realm, zulip_stream_dict)
# Now that we've created the notifications stream, configure it properly.
zulip_realm.notifications_stream = get_stream(
"announce", zulip_realm)
zulip_realm.save(update_fields=['notifications_stream'])
# Add a few default streams
for default_stream_name in [
"design", "devel", "social", "support"
]:
DefaultStream.objects.create(realm=zulip_realm,
stream=get_stream(
default_stream_name,
zulip_realm))
# Now subscribe everyone to these streams
subscriptions_to_add = []
event_time = timezone_now()
all_subscription_logs = []
profiles = UserProfile.objects.select_related().filter(
realm=zulip_realm)
for i, stream_name in enumerate(zulip_stream_dict):
stream = Stream.objects.get(name=stream_name,
realm=zulip_realm)
recipient = Recipient.objects.get(type=Recipient.STREAM,
type_id=stream.id)
for profile in profiles:
# Subscribe to some streams.
s = Subscription(
recipient=recipient,
user_profile=profile,
color=STREAM_ASSIGNMENT_COLORS[
i % len(STREAM_ASSIGNMENT_COLORS)])
subscriptions_to_add.append(s)
log = RealmAuditLog(
realm=profile.realm,
modified_user=profile,
modified_stream=stream,
event_last_message_id=0,
event_type=RealmAuditLog.SUBSCRIPTION_CREATED,
event_time=event_time)
all_subscription_logs.append(log)
Subscription.objects.bulk_create(subscriptions_to_add)
RealmAuditLog.objects.bulk_create(all_subscription_logs)
# These bots are not needed by the test suite
internal_zulip_users_nosubs = [
("Zulip Commit Bot", "*****@*****.**"),
("Zulip Trac Bot", "*****@*****.**"),
("Zulip Nagios Bot", "*****@*****.**"),
]
create_users(zulip_realm,
internal_zulip_users_nosubs,
bot_type=UserProfile.DEFAULT_BOT)
zulip_cross_realm_bots = [
("Zulip Feedback Bot", "*****@*****.**"),
]
create_users(zulip_realm,
zulip_cross_realm_bots,
bot_type=UserProfile.DEFAULT_BOT)
# Mark all messages as read
UserMessage.objects.all().update(flags=UserMessage.flags.read)
if not options["test_suite"]:
# Update pointer of each user to point to the last message in their
# UserMessage rows with sender_id=user_profile_id.
users = list(
UserMessage.objects.filter(message__sender_id=F(
'user_profile_id')).values('user_profile_id').annotate(
pointer=Max('message_id')))
for user in users:
UserProfile.objects.filter(
id=user['user_profile_id']).update(
pointer=user['pointer'])
create_user_groups()
if not options["test_suite"]:
# We populate the analytics database here for
# development purpose only
call_command('populate_analytics_db')
self.stdout.write("Successfully populated test database.\n")
def add_bot_backend(request, user_profile, full_name_raw=REQ("full_name"), short_name_raw=REQ("short_name"),
bot_type=REQ(validator=check_int, default=UserProfile.DEFAULT_BOT),
payload_url=REQ(validator=check_url, default=None),
default_sending_stream_name=REQ('default_sending_stream', default=None),
default_events_register_stream_name=REQ('default_events_register_stream', default=None),
default_all_public_streams=REQ(validator=check_bool, default=None)):
# type: (HttpRequest, UserProfile, Text, Text, int, Optional[Text], Optional[Text], Optional[Text], Optional[bool]) -> HttpResponse
short_name = check_short_name(short_name_raw)
service_name = short_name
short_name += "-bot"
full_name = check_full_name(full_name_raw)
email = '%s@%s' % (short_name, user_profile.realm.get_bot_domain())
form = CreateUserForm({'full_name': full_name, 'email': email})
if not form.is_valid():
# We validate client-side as well
return json_error(_('Bad name or username'))
try:
get_user(email, user_profile.realm)
return json_error(_("Username already in use"))
except UserProfile.DoesNotExist:
pass
check_valid_bot_type(bot_type)
if len(request.FILES) == 0:
avatar_source = UserProfile.AVATAR_FROM_GRAVATAR
elif len(request.FILES) != 1:
return json_error(_("You may only upload one file at a time"))
else:
avatar_source = UserProfile.AVATAR_FROM_USER
default_sending_stream = None
if default_sending_stream_name is not None:
(default_sending_stream, ignored_rec, ignored_sub) = access_stream_by_name(
user_profile, default_sending_stream_name)
default_events_register_stream = None
if default_events_register_stream_name is not None:
(default_events_register_stream, ignored_rec, ignored_sub) = access_stream_by_name(
user_profile, default_events_register_stream_name)
bot_profile = do_create_user(email=email, password='',
realm=user_profile.realm, full_name=full_name,
short_name=short_name, active=True,
bot_type=bot_type,
bot_owner=user_profile,
avatar_source=avatar_source,
default_sending_stream=default_sending_stream,
default_events_register_stream=default_events_register_stream,
default_all_public_streams=default_all_public_streams)
if len(request.FILES) == 1:
user_file = list(request.FILES.values())[0]
upload_avatar_image(user_file, user_profile, bot_profile)
if bot_type == UserProfile.OUTGOING_WEBHOOK_BOT:
add_outgoing_webhook_service(name=service_name,
user_profile=bot_profile,
base_url=payload_url,
interface=1,
token=random_api_key())
json_result = dict(
api_key=bot_profile.api_key,
avatar_url=avatar_url(bot_profile),
default_sending_stream=get_stream_name(bot_profile.default_sending_stream),
default_events_register_stream=get_stream_name(bot_profile.default_events_register_stream),
default_all_public_streams=bot_profile.default_all_public_streams,
)
return json_success(json_result)
def handle(self, **options: Any) -> None:
if options["percent_huddles"] + options["percent_personals"] > 100:
self.stderr.write("Error! More than 100% of messages allocated.\n")
return
# Get consistent data for backend tests.
if options["test_suite"]:
random.seed(0)
if options["delete"]:
# Start by clearing all the data in our database
clear_database()
# Create our two default realms
# Could in theory be done via zerver.lib.actions.do_create_realm, but
# welcome-bot (needed for do_create_realm) hasn't been created yet
zulip_realm = Realm.objects.create(
string_id="zulip", name="Zulip Dev", restricted_to_domain=True,
description="The Zulip development environment default organization."
" It's great for testing!",
invite_required=False, org_type=Realm.CORPORATE)
RealmDomain.objects.create(realm=zulip_realm, domain="zulip.com")
if options["test_suite"]:
mit_realm = Realm.objects.create(
string_id="zephyr", name="MIT", restricted_to_domain=True,
invite_required=False, org_type=Realm.CORPORATE)
RealmDomain.objects.create(realm=mit_realm, domain="mit.edu")
lear_realm = Realm.objects.create(
string_id="lear", name="Lear & Co.", restricted_to_domain=False,
invite_required=False, org_type=Realm.CORPORATE)
# Create test Users (UserProfiles are automatically created,
# as are subscriptions to the ability to receive personals).
names = [
("Zoe", "*****@*****.**"),
("Othello, the Moor of Venice", "*****@*****.**"),
("Iago", "*****@*****.**"),
("Prospero from The Tempest", "*****@*****.**"),
("Cordelia Lear", "*****@*****.**"),
("King Hamlet", "*****@*****.**"),
("aaron", "*****@*****.**"),
("Polonius", "*****@*****.**"),
]
for i in range(options["extra_users"]):
names.append(('Extra User %d' % (i,), '*****@*****.**' % (i,)))
create_users(zulip_realm, names)
iago = get_user("*****@*****.**", zulip_realm)
do_change_is_admin(iago, True)
iago.is_staff = True
iago.save(update_fields=['is_staff'])
guest_user = get_user("*****@*****.**", zulip_realm)
guest_user.is_guest = True
guest_user.save(update_fields=['is_guest'])
# These bots are directly referenced from code and thus
# are needed for the test suite.
all_realm_bots = [(bot['name'], bot['email_template'] % (settings.INTERNAL_BOT_DOMAIN,))
for bot in settings.INTERNAL_BOTS]
zulip_realm_bots = [
("Zulip New User Bot", "*****@*****.**"),
("Zulip Error Bot", "*****@*****.**"),
("Zulip Default Bot", "*****@*****.**"),
("Welcome Bot", "*****@*****.**"),
]
for i in range(options["extra_bots"]):
zulip_realm_bots.append(('Extra Bot %d' % (i,), '*****@*****.**' % (i,)))
zulip_realm_bots.extend(all_realm_bots)
create_users(zulip_realm, zulip_realm_bots, bot_type=UserProfile.DEFAULT_BOT)
# Initialize the email gateway bot as an API Super User
email_gateway_bot = get_system_bot(settings.EMAIL_GATEWAY_BOT)
email_gateway_bot.is_api_super_user = True
email_gateway_bot.save()
zoe = get_user("*****@*****.**", zulip_realm)
zulip_webhook_bots = [
("Zulip Webhook Bot", "*****@*****.**"),
]
# If a stream is not supplied in the webhook URL, the webhook
# will (in some cases) send the notification as a PM to the
# owner of the webhook bot, so bot_owner can't be None
create_users(zulip_realm, zulip_webhook_bots,
bot_type=UserProfile.INCOMING_WEBHOOK_BOT, bot_owner=zoe)
aaron = get_user("*****@*****.**", zulip_realm)
zulip_outgoing_bots = [
("Outgoing Webhook", "*****@*****.**")
]
create_users(zulip_realm, zulip_outgoing_bots,
bot_type=UserProfile.OUTGOING_WEBHOOK_BOT, bot_owner=aaron)
outgoing_webhook = get_user("*****@*****.**", zulip_realm)
add_service("outgoing-webhook", user_profile=outgoing_webhook, interface=Service.GENERIC,
base_url="http://127.0.0.1:5002", token=random_api_key())
# Add the realm internl bots to each realm.
create_if_missing_realm_internal_bots()
# Create public streams.
stream_list = ["Verona", "Denmark", "Scotland", "Venice", "Rome"]
stream_dict = {
"Verona": {"description": "A city in Italy"},
"Denmark": {"description": "A Scandinavian country"},
"Scotland": {"description": "Located in the United Kingdom"},
"Venice": {"description": "A northeastern Italian city"},
"Rome": {"description": "Yet another Italian city", "is_web_public": True}
} # type: Dict[str, Dict[str, Any]]
bulk_create_streams(zulip_realm, stream_dict)
recipient_streams = [Stream.objects.get(name=name, realm=zulip_realm).id
for name in stream_list] # type: List[int]
# Create subscriptions to streams. The following
# algorithm will give each of the users a different but
# deterministic subset of the streams (given a fixed list
# of users). For the test suite, we have a fixed list of
# subscriptions to make sure test data is consistent
# across platforms.
subscriptions_list = [] # type: List[Tuple[UserProfile, Recipient]]
profiles = UserProfile.objects.select_related().filter(
is_bot=False).order_by("email") # type: Sequence[UserProfile]
if options["test_suite"]:
subscriptions_map = {
'*****@*****.**': ['Verona'],
'*****@*****.**': ['Verona'],
'*****@*****.**': ['Verona', 'Denmark'],
'*****@*****.**': ['Verona', 'Denmark', 'Scotland'],
'*****@*****.**': ['Verona', 'Denmark', 'Scotland'],
'*****@*****.**': ['Verona', 'Denmark', 'Scotland', 'Venice'],
'*****@*****.**': ['Verona', 'Denmark', 'Scotland', 'Venice', 'Rome'],
'*****@*****.**': ['Verona'],
}
for profile in profiles:
if profile.email not in subscriptions_map:
raise Exception('Subscriptions not listed for user %s' % (profile.email,))
for stream_name in subscriptions_map[profile.email]:
stream = Stream.objects.get(name=stream_name)
r = Recipient.objects.get(type=Recipient.STREAM, type_id=stream.id)
subscriptions_list.append((profile, r))
else:
for i, profile in enumerate(profiles):
# Subscribe to some streams.
for type_id in recipient_streams[:int(len(recipient_streams) *
float(i)/len(profiles)) + 1]:
r = Recipient.objects.get(type=Recipient.STREAM, type_id=type_id)
subscriptions_list.append((profile, r))
subscriptions_to_add = [] # type: List[Subscription]
event_time = timezone_now()
all_subscription_logs = [] # type: (List[RealmAuditLog])
i = 0
for profile, recipient in subscriptions_list:
i += 1
color = STREAM_ASSIGNMENT_COLORS[i % len(STREAM_ASSIGNMENT_COLORS)]
s = Subscription(
recipient=recipient,
user_profile=profile,
color=color)
subscriptions_to_add.append(s)
log = RealmAuditLog(realm=profile.realm,
modified_user=profile,
modified_stream_id=recipient.type_id,
event_last_message_id=0,
event_type='subscription_created',
event_time=event_time)
all_subscription_logs.append(log)
Subscription.objects.bulk_create(subscriptions_to_add)
RealmAuditLog.objects.bulk_create(all_subscription_logs)
# Create custom profile field data
phone_number = try_add_realm_custom_profile_field(zulip_realm, "Phone number",
CustomProfileField.SHORT_TEXT,
hint='')
biography = try_add_realm_custom_profile_field(zulip_realm, "Biography",
CustomProfileField.LONG_TEXT,
hint='What are you known for?')
favorite_food = try_add_realm_custom_profile_field(zulip_realm, "Favorite food",
CustomProfileField.SHORT_TEXT,
hint="Or drink, if you'd prefer")
field_data = {
'vim': {'text': 'Vim', 'order': '1'},
'emacs': {'text': 'Emacs', 'order': '2'},
}
favorite_editor = try_add_realm_custom_profile_field(zulip_realm,
"Favorite editor",
CustomProfileField.CHOICE,
field_data=field_data)
birthday = try_add_realm_custom_profile_field(zulip_realm, "Birthday",
CustomProfileField.DATE)
favorite_website = try_add_realm_custom_profile_field(zulip_realm, "GitHub profile",
CustomProfileField.URL,
hint="Or your personal blog's URL")
mentor = try_add_realm_custom_profile_field(zulip_realm, "Mentor",
CustomProfileField.USER)
# Fill in values for Iago and Hamlet
hamlet = get_user("*****@*****.**", zulip_realm)
do_update_user_custom_profile_data(iago, [
{"id": phone_number.id, "value": "+1-234-567-8901"},
{"id": biography.id, "value": "Betrayer of Othello."},
{"id": favorite_food.id, "value": "Apples"},
{"id": favorite_editor.id, "value": "emacs"},
{"id": birthday.id, "value": "2000-1-1"},
{"id": favorite_website.id, "value": "https://github.com/zulip/zulip"},
{"id": mentor.id, "value": hamlet.id},
])
do_update_user_custom_profile_data(hamlet, [
{"id": phone_number.id, "value": "+0-11-23-456-7890"},
{"id": biography.id, "value": "Prince of Denmark, and other things!"},
{"id": favorite_food.id, "value": "Dark chocolate"},
{"id": favorite_editor.id, "value": "vim"},
{"id": birthday.id, "value": "1900-1-1"},
{"id": favorite_website.id, "value": "https://blog.zulig.org"},
{"id": mentor.id, "value": iago.id},
])
else:
zulip_realm = get_realm("zulip")
recipient_streams = [klass.type_id for klass in
Recipient.objects.filter(type=Recipient.STREAM)]
# Extract a list of all users
user_profiles = list(UserProfile.objects.filter(is_bot=False)) # type: List[UserProfile]
# Create a test realm emoji.
IMAGE_FILE_PATH = os.path.join(settings.STATIC_ROOT, 'images', 'test-images', 'checkbox.png')
with open(IMAGE_FILE_PATH, 'rb') as fp:
check_add_realm_emoji(zulip_realm, 'green_tick', iago, fp)
if not options["test_suite"]:
# Populate users with some bar data
for user in user_profiles:
status = UserPresence.ACTIVE # type: int
date = timezone_now()
client = get_client("website")
if user.full_name[0] <= 'H':
client = get_client("ZulipAndroid")
UserPresence.objects.get_or_create(user_profile=user,
client=client,
timestamp=date,
status=status)
user_profiles_ids = [user_profile.id for user_profile in user_profiles]
# Create several initial huddles
for i in range(options["num_huddles"]):
get_huddle(random.sample(user_profiles_ids, random.randint(3, 4)))
# Create several initial pairs for personals
personals_pairs = [random.sample(user_profiles_ids, 2)
for i in range(options["num_personals"])]
# Generate a new set of test data.
create_test_data()
# prepopulate the URL preview/embed data for the links present
# in the config.generate_data.json data set. This makes it
# possible for populate_db to run happily without Internet
# access.
with open("zerver/tests/fixtures/docs_url_preview_data.json", "r") as f:
urls_with_preview_data = ujson.load(f)
for url in urls_with_preview_data:
cache_set(url, urls_with_preview_data[url], PREVIEW_CACHE_NAME)
threads = options["threads"]
jobs = [] # type: List[Tuple[int, List[List[int]], Dict[str, Any], Callable[[str], int], int]]
for i in range(threads):
count = options["num_messages"] // threads
if i < options["num_messages"] % threads:
count += 1
jobs.append((count, personals_pairs, options, self.stdout.write, random.randint(0, 10**10)))
for job in jobs:
send_messages(job)
if options["delete"]:
# Create the "website" and "API" clients; if we don't, the
# default values in zerver/decorators.py will not work
# with the Django test suite.
get_client("website")
get_client("API")
if options["test_suite"]:
# Create test users; the MIT ones are needed to test
# the Zephyr mirroring codepaths.
testsuite_mit_users = [
("Fred Sipb (MIT)", "*****@*****.**"),
("Athena Consulting Exchange User (MIT)", "*****@*****.**"),
("Esp Classroom (MIT)", "*****@*****.**"),
]
create_users(mit_realm, testsuite_mit_users)
testsuite_lear_users = [
("King Lear", "*****@*****.**"),
("Cordelia Lear", "*****@*****.**"),
]
create_users(lear_realm, testsuite_lear_users)
if not options["test_suite"]:
# To keep the messages.json fixtures file for the test
# suite fast, don't add these users and subscriptions
# when running populate_db for the test suite
zulip_stream_dict = {
"devel": {"description": "For developing"},
"all": {"description": "For everything"},
"announce": {"description": "For announcements", 'is_announcement_only': True},
"design": {"description": "For design"},
"support": {"description": "For support"},
"social": {"description": "For socializing"},
"test": {"description": "For testing"},
"errors": {"description": "For errors"},
"sales": {"description": "For sales discussion"}
} # type: Dict[str, Dict[str, Any]]
# Calculate the maximum number of digits in any extra stream's
# number, since a stream with name "Extra Stream 3" could show
# up after "Extra Stream 29". (Used later to pad numbers with
# 0s).
maximum_digits = len(str(options['extra_streams'] - 1))
for i in range(options['extra_streams']):
# Pad the number with 0s based on `maximum_digits`.
number_str = str(i).zfill(maximum_digits)
extra_stream_name = 'Extra Stream ' + number_str
zulip_stream_dict[extra_stream_name] = {
"description": "Auto-generated extra stream.",
}
bulk_create_streams(zulip_realm, zulip_stream_dict)
# Now that we've created the notifications stream, configure it properly.
zulip_realm.notifications_stream = get_stream("announce", zulip_realm)
zulip_realm.save(update_fields=['notifications_stream'])
# Add a few default streams
for default_stream_name in ["design", "devel", "social", "support"]:
DefaultStream.objects.create(realm=zulip_realm,
stream=get_stream(default_stream_name, zulip_realm))
# Now subscribe everyone to these streams
subscriptions_to_add = []
event_time = timezone_now()
all_subscription_logs = []
profiles = UserProfile.objects.select_related().filter(realm=zulip_realm)
for i, stream_name in enumerate(zulip_stream_dict):
stream = Stream.objects.get(name=stream_name, realm=zulip_realm)
recipient = Recipient.objects.get(type=Recipient.STREAM, type_id=stream.id)
for profile in profiles:
# Subscribe to some streams.
s = Subscription(
recipient=recipient,
user_profile=profile,
color=STREAM_ASSIGNMENT_COLORS[i % len(STREAM_ASSIGNMENT_COLORS)])
subscriptions_to_add.append(s)
log = RealmAuditLog(realm=profile.realm,
modified_user=profile,
modified_stream=stream,
event_last_message_id=0,
event_type='subscription_created',
event_time=event_time)
all_subscription_logs.append(log)
Subscription.objects.bulk_create(subscriptions_to_add)
RealmAuditLog.objects.bulk_create(all_subscription_logs)
# These bots are not needed by the test suite
internal_zulip_users_nosubs = [
("Zulip Commit Bot", "*****@*****.**"),
("Zulip Trac Bot", "*****@*****.**"),
("Zulip Nagios Bot", "*****@*****.**"),
]
create_users(zulip_realm, internal_zulip_users_nosubs, bot_type=UserProfile.DEFAULT_BOT)
zulip_cross_realm_bots = [
("Zulip Feedback Bot", "*****@*****.**"),
]
create_users(zulip_realm, zulip_cross_realm_bots, bot_type=UserProfile.DEFAULT_BOT)
# Mark all messages as read
UserMessage.objects.all().update(flags=UserMessage.flags.read)
if not options["test_suite"]:
# Update pointer of each user to point to the last message in their
# UserMessage rows with sender_id=user_profile_id.
users = list(UserMessage.objects.filter(
message__sender_id=F('user_profile_id')).values(
'user_profile_id').annotate(pointer=Max('message_id')))
for user in users:
UserProfile.objects.filter(id=user['user_profile_id']).update(
pointer=user['pointer'])
create_user_groups()
self.stdout.write("Successfully populated test database.\n")
def do_import_realm(import_dir: Path, subdomain: str) -> Realm:
logging.info("Importing realm dump %s" % (import_dir,))
if not os.path.exists(import_dir):
raise Exception("Missing import directory!")
realm_data_filename = os.path.join(import_dir, "realm.json")
if not os.path.exists(realm_data_filename):
raise Exception("Missing realm.json file!")
logging.info("Importing realm data from %s" % (realm_data_filename,))
with open(realm_data_filename) as f:
data = ujson.load(f)
# We don't import the Stream model yet, since it depends on Realm,
# which isn't imported yet. But we need the Stream model IDs for
# notifications_stream.
update_model_ids(Stream, data, 'stream')
re_map_foreign_keys(data, 'zerver_realm', 'notifications_stream', related_table="stream")
fix_datetime_fields(data, 'zerver_realm')
# Fix realm subdomain information
data['zerver_realm'][0]['string_id'] = subdomain
data['zerver_realm'][0]['name'] = subdomain
fix_realm_authentication_bitfield(data, 'zerver_realm', 'authentication_methods')
update_model_ids(Realm, data, 'realm')
realm = Realm(**data['zerver_realm'][0])
if realm.notifications_stream_id is not None:
notifications_stream_id = int(realm.notifications_stream_id) # type: Optional[int]
else:
notifications_stream_id = None
realm.notifications_stream_id = None
realm.save()
bulk_import_client(data, Client, 'zerver_client')
# Email tokens will automatically be randomly generated when the
# Stream objects are created by Django.
fix_datetime_fields(data, 'zerver_stream')
re_map_foreign_keys(data, 'zerver_stream', 'realm', related_table="realm")
bulk_import_model(data, Stream)
realm.notifications_stream_id = notifications_stream_id
realm.save()
# Remap the user IDs for notification_bot and friends to their
# appropriate IDs on this server
for item in data['zerver_userprofile_crossrealm']:
logging.info("Adding to ID map: %s %s" % (item['id'], get_system_bot(item['email']).id))
new_user_id = get_system_bot(item['email']).id
update_id_map(table='user_profile', old_id=item['id'], new_id=new_user_id)
# Merge in zerver_userprofile_mirrordummy
data['zerver_userprofile'] = data['zerver_userprofile'] + data['zerver_userprofile_mirrordummy']
del data['zerver_userprofile_mirrordummy']
data['zerver_userprofile'].sort(key=lambda r: r['id'])
# To remap foreign key for UserProfile.last_active_message_id
update_message_foreign_keys(import_dir)
fix_datetime_fields(data, 'zerver_userprofile')
update_model_ids(UserProfile, data, 'user_profile')
re_map_foreign_keys(data, 'zerver_userprofile', 'realm', related_table="realm")
re_map_foreign_keys(data, 'zerver_userprofile', 'bot_owner', related_table="user_profile")
re_map_foreign_keys(data, 'zerver_userprofile', 'default_sending_stream',
related_table="stream")
re_map_foreign_keys(data, 'zerver_userprofile', 'default_events_register_stream',
related_table="stream")
re_map_foreign_keys(data, 'zerver_userprofile', 'last_active_message_id',
related_table="message", id_field=True)
for user_profile_dict in data['zerver_userprofile']:
user_profile_dict['password'] = None
user_profile_dict['api_key'] = random_api_key()
# Since Zulip doesn't use these permissions, drop them
del user_profile_dict['user_permissions']
del user_profile_dict['groups']
user_profiles = [UserProfile(**item) for item in data['zerver_userprofile']]
for user_profile in user_profiles:
user_profile.set_unusable_password()
UserProfile.objects.bulk_create(user_profiles)
re_map_foreign_keys(data, 'zerver_defaultstream', 'stream', related_table="stream")
re_map_foreign_keys(data, 'zerver_realmemoji', 'author', related_table="user_profile")
for (table, model, related_table) in realm_tables:
re_map_foreign_keys(data, table, 'realm', related_table="realm")
update_model_ids(model, data, related_table)
bulk_import_model(data, model)
if 'zerver_huddle' in data:
update_model_ids(Huddle, data, 'huddle')
# We don't import Huddle yet, since we don't have the data to
# compute huddle hashes until we've imported some of the
# tables below.
# TODO: double-check this.
re_map_foreign_keys(data, 'zerver_recipient', 'type_id', related_table="stream",
recipient_field=True, id_field=True)
re_map_foreign_keys(data, 'zerver_recipient', 'type_id', related_table="user_profile",
recipient_field=True, id_field=True)
re_map_foreign_keys(data, 'zerver_recipient', 'type_id', related_table="huddle",
recipient_field=True, id_field=True)
update_model_ids(Recipient, data, 'recipient')
bulk_import_model(data, Recipient)
re_map_foreign_keys(data, 'zerver_subscription', 'user_profile', related_table="user_profile")
get_huddles_from_subscription(data, 'zerver_subscription')
re_map_foreign_keys(data, 'zerver_subscription', 'recipient', related_table="recipient")
update_model_ids(Subscription, data, 'subscription')
bulk_import_model(data, Subscription)
if 'zerver_realmauditlog' in data:
fix_datetime_fields(data, 'zerver_realmauditlog')
re_map_foreign_keys(data, 'zerver_realmauditlog', 'realm', related_table="realm")
re_map_foreign_keys(data, 'zerver_realmauditlog', 'modified_user',
related_table='user_profile')
re_map_foreign_keys(data, 'zerver_realmauditlog', 'acting_user',
related_table='user_profile')
re_map_foreign_keys(data, 'zerver_realmauditlog', 'modified_stream',
related_table="stream")
update_model_ids(RealmAuditLog, data, related_table="realmauditlog")
bulk_import_model(data, RealmAuditLog)
else:
create_subscription_events(data, 'zerver_subscription')
if 'zerver_huddle' in data:
process_huddle_hash(data, 'zerver_huddle')
bulk_import_model(data, Huddle)
if 'zerver_userhotspot' in data:
fix_datetime_fields(data, 'zerver_userhotspot')
re_map_foreign_keys(data, 'zerver_userhotspot', 'user', related_table='user_profile')
update_model_ids(UserHotspot, data, 'userhotspot')
bulk_import_model(data, UserHotspot)
if 'zerver_mutedtopic' in data:
re_map_foreign_keys(data, 'zerver_mutedtopic', 'user_profile', related_table='user_profile')
re_map_foreign_keys(data, 'zerver_mutedtopic', 'stream', related_table='stream')
re_map_foreign_keys(data, 'zerver_mutedtopic', 'recipient', related_table='recipient')
update_model_ids(MutedTopic, data, 'mutedtopic')
bulk_import_model(data, MutedTopic)
if 'zerver_service' in data:
re_map_foreign_keys(data, 'zerver_service', 'user_profile', related_table='user_profile')
fix_service_tokens(data, 'zerver_service')
update_model_ids(Service, data, 'service')
bulk_import_model(data, Service)
if 'zerver_usergroup' in data:
re_map_foreign_keys(data, 'zerver_usergroup', 'realm', related_table='realm')
re_map_foreign_keys_many_to_many(data, 'zerver_usergroup',
'members', related_table='user_profile')
update_model_ids(UserGroup, data, 'usergroup')
bulk_import_model(data, UserGroup)
re_map_foreign_keys(data, 'zerver_usergroupmembership',
'user_group', related_table='usergroup')
re_map_foreign_keys(data, 'zerver_usergroupmembership',
'user_profile', related_table='user_profile')
update_model_ids(UserGroupMembership, data, 'usergroupmembership')
bulk_import_model(data, UserGroupMembership)
if 'zerver_botstoragedata' in data:
re_map_foreign_keys(data, 'zerver_botstoragedata', 'bot_profile', related_table='user_profile')
update_model_ids(BotStorageData, data, 'botstoragedata')
bulk_import_model(data, BotStorageData)
if 'zerver_botconfigdata' in data:
re_map_foreign_keys(data, 'zerver_botconfigdata', 'bot_profile', related_table='user_profile')
update_model_ids(BotConfigData, data, 'botconfigdata')
bulk_import_model(data, BotConfigData)
fix_datetime_fields(data, 'zerver_userpresence')
re_map_foreign_keys(data, 'zerver_userpresence', 'user_profile', related_table="user_profile")
re_map_foreign_keys(data, 'zerver_userpresence', 'client', related_table='client')
update_model_ids(UserPresence, data, 'user_presence')
bulk_import_model(data, UserPresence)
fix_datetime_fields(data, 'zerver_useractivity')
re_map_foreign_keys(data, 'zerver_useractivity', 'user_profile', related_table="user_profile")
re_map_foreign_keys(data, 'zerver_useractivity', 'client', related_table='client')
update_model_ids(UserActivity, data, 'useractivity')
bulk_import_model(data, UserActivity)
fix_datetime_fields(data, 'zerver_useractivityinterval')
re_map_foreign_keys(data, 'zerver_useractivityinterval', 'user_profile', related_table="user_profile")
update_model_ids(UserActivityInterval, data, 'useractivityinterval')
bulk_import_model(data, UserActivityInterval)
re_map_foreign_keys(data, 'zerver_customprofilefield', 'realm', related_table="realm")
update_model_ids(CustomProfileField, data, related_table="customprofilefield")
bulk_import_model(data, CustomProfileField)
re_map_foreign_keys(data, 'zerver_customprofilefieldvalue', 'user_profile',
related_table="user_profile")
re_map_foreign_keys(data, 'zerver_customprofilefieldvalue', 'field',
related_table="customprofilefield")
fix_customprofilefield(data)
update_model_ids(CustomProfileFieldValue, data, related_table="customprofilefieldvalue")
bulk_import_model(data, CustomProfileFieldValue)
# Import uploaded files and avatars
import_uploads(os.path.join(import_dir, "avatars"), processing_avatars=True)
import_uploads(os.path.join(import_dir, "uploads"))
# We need to have this check as the emoji files are only present in the data
# importer from slack
# For Zulip export, this doesn't exist
if os.path.exists(os.path.join(import_dir, "emoji")):
import_uploads(os.path.join(import_dir, "emoji"), processing_emojis=True)
# Import zerver_message and zerver_usermessage
import_message_data(import_dir)
re_map_foreign_keys(data, 'zerver_reaction', 'message', related_table="message")
re_map_foreign_keys(data, 'zerver_reaction', 'user_profile', related_table="user_profile")
re_map_foreign_keys(data, 'zerver_reaction', 'emoji_code', related_table="realmemoji", id_field=True,
reaction_field=True)
update_model_ids(Reaction, data, 'reaction')
bulk_import_model(data, Reaction)
# Do attachments AFTER message data is loaded.
# TODO: de-dup how we read these json files.
fn = os.path.join(import_dir, "attachment.json")
if not os.path.exists(fn):
raise Exception("Missing attachment.json file!")
logging.info("Importing attachment data from %s" % (fn,))
with open(fn) as f:
data = ujson.load(f)
import_attachments(data)
return realm
