def do_convert_data(slack_zip_file: str, output_dir: str, token: str, threads: int=6) -> None: # Subdomain is set by the user while running the import command realm_subdomain = "" realm_id = 0 domain_name = settings.EXTERNAL_HOST slack_data_dir = slack_zip_file.replace('.zip', '') if not os.path.exists(slack_data_dir): os.makedirs(slack_data_dir) os.makedirs(output_dir, exist_ok=True) # output directory should be empty initially if os.listdir(output_dir): raise Exception('Output directory should be empty!') subprocess.check_call(['unzip', '-q', slack_zip_file, '-d', slack_data_dir]) # with zipfile.ZipFile(slack_zip_file, 'r') as zip_ref: # zip_ref.extractall(slack_data_dir) # We get the user data from the legacy token method of slack api, which is depreciated # but we use it as the user email data is provided only in this method user_list = get_slack_api_data(token, "https://slack.com/api/users.list", "members") # Get custom emoji from slack api custom_emoji_list = get_slack_api_data(token, "https://slack.com/api/emoji.list", "emoji") realm, added_users, added_recipient, added_channels, avatar_list, \ emoji_url_map = slack_workspace_to_realm(domain_name, realm_id, user_list, realm_subdomain, slack_data_dir, custom_emoji_list) reactions, uploads_list, zerver_attachment = convert_slack_workspace_messages( slack_data_dir, user_list, realm_id, added_users, added_recipient, added_channels, realm, realm['zerver_realmemoji'], domain_name, output_dir) # Move zerver_reactions to realm.json file realm['zerver_reaction'] = reactions emoji_folder = os.path.join(output_dir, 'emoji') os.makedirs(emoji_folder, exist_ok=True) emoji_records = process_emojis(realm['zerver_realmemoji'], emoji_folder, emoji_url_map, threads) avatar_folder = os.path.join(output_dir, 'avatars') avatar_realm_folder = os.path.join(avatar_folder, str(realm_id)) os.makedirs(avatar_realm_folder, exist_ok=True) avatar_records = process_avatars(avatar_list, avatar_folder, realm_id, threads, size_url_suffix='-512') uploads_folder = os.path.join(output_dir, 'uploads') os.makedirs(os.path.join(uploads_folder, str(realm_id)), exist_ok=True) uploads_records = process_uploads(uploads_list, uploads_folder, threads) attachment = {"zerver_attachment": zerver_attachment} # IO realm.json create_converted_data_files(realm, output_dir, '/realm.json') # IO emoji records create_converted_data_files(emoji_records, output_dir, '/emoji/records.json') # IO avatar records create_converted_data_files(avatar_records, output_dir, '/avatars/records.json') # IO uploads records create_converted_data_files(uploads_records, output_dir, '/uploads/records.json') # IO attachments records create_converted_data_files(attachment, output_dir, '/attachment.json') # remove slack dir rm_tree(slack_data_dir) subprocess.check_call(["tar", "-czf", output_dir + '.tar.gz', output_dir, '-P']) logging.info('######### DATA CONVERSION FINISHED #########\n') logging.info("Zulip data dump created at %s" % (output_dir))
def do_convert_data(slack_zip_file: str, output_dir: str, token: str, threads: int = 6) -> None: # Subdomain is set by the user while running the import command realm_subdomain = "" realm_id = 0 domain_name = settings.EXTERNAL_HOST check_token_access(token) slack_data_dir = slack_zip_file.replace(".zip", "") if not os.path.exists(slack_data_dir): os.makedirs(slack_data_dir) os.makedirs(output_dir, exist_ok=True) if os.listdir(output_dir): raise Exception("Output directory should be empty!") subprocess.check_call(["unzip", "-q", slack_zip_file, "-d", slack_data_dir]) # We get the user data from the legacy token method of Slack API, which is depreciated # but we use it as the user email data is provided only in this method user_list = get_slack_api_data("https://slack.com/api/users.list", "members", token=token) fetch_shared_channel_users(user_list, slack_data_dir, token) custom_emoji_list = get_slack_api_data("https://slack.com/api/emoji.list", "emoji", token=token) ( realm, slack_user_id_to_zulip_user_id, slack_recipient_name_to_zulip_recipient_id, added_channels, added_mpims, dm_members, avatar_list, emoji_url_map, ) = slack_workspace_to_realm( domain_name, realm_id, user_list, realm_subdomain, slack_data_dir, custom_emoji_list ) reactions, uploads_list, zerver_attachment = convert_slack_workspace_messages( slack_data_dir, user_list, realm_id, slack_user_id_to_zulip_user_id, slack_recipient_name_to_zulip_recipient_id, added_channels, added_mpims, dm_members, realm, realm["zerver_userprofile"], realm["zerver_realmemoji"], domain_name, output_dir, ) # Move zerver_reactions to realm.json file realm["zerver_reaction"] = reactions emoji_folder = os.path.join(output_dir, "emoji") os.makedirs(emoji_folder, exist_ok=True) emoji_records = process_emojis(realm["zerver_realmemoji"], emoji_folder, emoji_url_map, threads) avatar_folder = os.path.join(output_dir, "avatars") avatar_realm_folder = os.path.join(avatar_folder, str(realm_id)) os.makedirs(avatar_realm_folder, exist_ok=True) avatar_records = process_avatars( avatar_list, avatar_folder, realm_id, threads, size_url_suffix="-512" ) uploads_folder = os.path.join(output_dir, "uploads") os.makedirs(os.path.join(uploads_folder, str(realm_id)), exist_ok=True) uploads_records = process_uploads(uploads_list, uploads_folder, threads) attachment = {"zerver_attachment": zerver_attachment} team_info_dict = get_slack_api_data("https://slack.com/api/team.info", "team", token=token) realm_icons_folder = os.path.join(output_dir, "realm_icons") realm_icon_records = fetch_team_icons( realm["zerver_realm"][0], team_info_dict, realm_icons_folder ) create_converted_data_files(realm, output_dir, "/realm.json") create_converted_data_files(emoji_records, output_dir, "/emoji/records.json") create_converted_data_files(avatar_records, output_dir, "/avatars/records.json") create_converted_data_files(uploads_records, output_dir, "/uploads/records.json") create_converted_data_files(attachment, output_dir, "/attachment.json") create_converted_data_files(realm_icon_records, output_dir, "/realm_icons/records.json") rm_tree(slack_data_dir) subprocess.check_call(["tar", "-czf", output_dir + ".tar.gz", output_dir, "-P"]) logging.info("######### DATA CONVERSION FINISHED #########\n") logging.info("Zulip data dump created at %s", output_dir)