def handle(self, *args: Any, **options: Any) -> None: subdomain = options['subdomain'] if options["destroy_rebuild_database"]: print("Rebuilding the database!") db_name = settings.DATABASES['default']['NAME'] self.do_destroy_and_rebuild_database(db_name) elif options["import_into_nonempty"]: print( "NOTE: The argument 'import_into_nonempty' is now the default behavior." ) check_subdomain_available(subdomain, from_management_command=True) for path in options['export_paths']: if not os.path.exists(path): print("Directory not found: '%s'" % (path, )) exit(1) if not os.path.isdir(path): print( "Export file should be folder; if it's a tarball, please unpack it first." ) exit(1) for path in options['export_paths']: print("Processing dump: %s ..." % (path, )) realm = do_import_realm(path, subdomain) print("Checking the system bots.") do_import_system_bots(realm)
def handle(self, *args: Any, **options: Any) -> None: subdomain = options['subdomain'] if options["destroy_rebuild_database"]: print("Rebuilding the database!") db_name = settings.DATABASES['default']['NAME'] self.do_destroy_and_rebuild_database(db_name) elif options["import_into_nonempty"]: print("NOTE: The argument 'import_into_nonempty' is now the default behavior.") check_subdomain_available(subdomain, from_management_command=True) paths = [] for path in options['export_paths']: path = os.path.realpath(os.path.expanduser(path)) if not os.path.exists(path): print("Directory not found: '%s'" % (path,)) exit(1) if not os.path.isdir(path): print("Export file should be folder; if it's a tarball, please unpack it first.") exit(1) paths.append(path) for path in paths: print("Processing dump: %s ..." % (path,)) realm = do_import_realm(path, subdomain) print("Checking the system bots.") do_import_system_bots(realm)
def handle(self, *args: Any, **options: Any) -> None: models_to_import = [Realm, Stream, UserProfile, Recipient, Subscription, Client, Message, UserMessage, Huddle, DefaultStream, RealmDomain, RealmFilter] subdomain = options['subdomain'] if subdomain is None: print("Enter subdomain!") exit(1) if options["destroy_rebuild_database"]: print("Rebuilding the database!") db_name = settings.DATABASES['default']['NAME'] self.do_destroy_and_rebuild_database(db_name) elif not options["import_into_nonempty"]: for model in models_to_import: self.new_instance_check(model) check_subdomain_available(subdomain, from_management_command=True) for path in options['export_files']: if not os.path.exists(path): print("Directory not found: '%s'" % (path,)) exit(1) print("Processing dump: %s ..." % (path,)) realm = do_import_realm(path, subdomain) print("Checking the system bots.") do_import_system_bots(realm)
def handle(self, *args: Any, **options: Any) -> None: num_processes = int(options['processes']) if num_processes < 1: raise CommandError('You must have at least one process.') subdomain = options['subdomain'] if options["destroy_rebuild_database"]: print("Rebuilding the database!") db_name = settings.DATABASES['default']['NAME'] self.do_destroy_and_rebuild_database(db_name) elif options["import_into_nonempty"]: print( "NOTE: The argument 'import_into_nonempty' is now the default behavior." ) check_subdomain_available(subdomain, from_management_command=True) paths = [] for path in options['export_paths']: path = os.path.realpath(os.path.expanduser(path)) if not os.path.exists(path): raise CommandError(f"Directory not found: '{path}'") if not os.path.isdir(path): raise CommandError("Export file should be folder; if it's a " "tarball, please unpack it first.") paths.append(path) for path in paths: print(f"Processing dump: {path} ...") realm = do_import_realm(path, subdomain, num_processes) print("Checking the system bots.") do_import_system_bots(realm)
def do_convert_data(slack_zip_file: str, realm_subdomain: str, output_dir: str, token: str) -> None: check_subdomain_available(realm_subdomain) domain_name = settings.EXTERNAL_HOST slack_data_dir = slack_zip_file.replace('.zip', '') if not os.path.exists(slack_data_dir): os.makedirs(slack_data_dir) os.makedirs(output_dir, exist_ok=True) # output directory should be empty initially if os.listdir(output_dir): raise Exception('Output directory should be empty!') subprocess.check_call(['unzip', '-q', slack_zip_file, '-d', slack_data_dir]) # with zipfile.ZipFile(slack_zip_file, 'r') as zip_ref: # zip_ref.extractall(slack_data_dir) script_path = os.path.dirname(os.path.abspath(__file__)) + '/' fixtures_path = script_path + '../fixtures/' realm_id = allocate_ids(Realm, 1)[0] user_list = get_user_data(token) realm, added_users, added_recipient, added_channels, avatar_list = slack_workspace_to_realm( domain_name, realm_id, user_list, realm_subdomain, fixtures_path, slack_data_dir) message_json, uploads_list, zerver_attachment = convert_slack_workspace_messages( slack_data_dir, user_list, realm_id, added_users, added_recipient, added_channels, realm, domain_name) avatar_folder = os.path.join(output_dir, 'avatars') avatar_realm_folder = os.path.join(avatar_folder, str(realm_id)) os.makedirs(avatar_realm_folder, exist_ok=True) avatar_records = process_avatars(avatar_list, avatar_folder, realm_id) uploads_folder = os.path.join(output_dir, 'uploads') os.makedirs(os.path.join(uploads_folder, str(realm_id)), exist_ok=True) uploads_records = process_uploads(uploads_list, uploads_folder) attachment = {"zerver_attachment": zerver_attachment} # IO realm.json create_converted_data_files(realm, output_dir, '/realm.json') # IO message.json create_converted_data_files(message_json, output_dir, '/messages-000001.json') # IO avatar records create_converted_data_files(avatar_records, output_dir, '/avatars/records.json') # IO uploads TODO create_converted_data_files(uploads_records, output_dir, '/uploads/records.json') # IO attachments create_converted_data_files(attachment, output_dir, '/attachment.json') # remove slack dir rm_tree(slack_data_dir) subprocess.check_call(["tar", "-czf", output_dir + '.tar.gz', output_dir, '-P']) logging.info('######### DATA CONVERSION FINISHED #########\n') logging.info("Zulip data dump created at %s" % (output_dir))
def do_convert_data(slack_zip_file: str, realm_subdomain: str, output_dir: str, token: str) -> None: check_subdomain_available(realm_subdomain) slack_data_dir = slack_zip_file.replace('.zip', '') if not os.path.exists(slack_data_dir): os.makedirs(slack_data_dir) os.makedirs(output_dir, exist_ok=True) # output directory should be empty initially if os.listdir(output_dir): raise Exception('Output directory should be empty!') subprocess.check_call( ['unzip', '-q', slack_zip_file, '-d', slack_data_dir]) # with zipfile.ZipFile(slack_zip_file, 'r') as zip_ref: # zip_ref.extractall(slack_data_dir) script_path = os.path.dirname(os.path.abspath(__file__)) + '/' fixtures_path = script_path + '../fixtures/' REALM_ID = allocate_ids(Realm, 1)[0] user_list = get_user_data(token) realm, added_users, added_recipient, added_channels = slack_workspace_to_realm( REALM_ID, user_list, realm_subdomain, fixtures_path, slack_data_dir) message_json = convert_slack_workspace_messages(slack_data_dir, user_list, REALM_ID, added_users, added_recipient, added_channels, realm) zerver_attachment = [] # type: List[ZerverFieldsT] attachment = {"zerver_attachment": zerver_attachment} # IO realm.json create_converted_data_files(realm, output_dir, '/realm.json', False) # IO message.json create_converted_data_files(message_json, output_dir, '/messages-000001.json', False) # IO avatar records create_converted_data_files([], output_dir, '/avatars/records.json', True) # IO uploads TODO create_converted_data_files([], output_dir, '/uploads/records.json', True) # IO attachments create_converted_data_files(attachment, output_dir, '/attachment.json', False) # remove slack dir rm_tree(slack_data_dir) subprocess.check_call( ["tar", "-czf", output_dir + '.tar.gz', output_dir, '-P']) logging.info('######### DATA CONVERSION FINISHED #########\n') logging.info("Zulip data dump created at %s" % (output_dir))
def do_convert_data(slack_zip_file: str, realm_subdomain: str, output_dir: str) -> None: check_subdomain_available(realm_subdomain) slack_data_dir = slack_zip_file.replace('.zip', '') if not os.path.exists(slack_data_dir): os.makedirs(slack_data_dir) subprocess.check_call( ['unzip', '-q', slack_zip_file, '-d', slack_data_dir]) # with zipfile.ZipFile(slack_zip_file, 'r') as zip_ref: # zip_ref.extractall(slack_data_dir) script_path = os.path.dirname(os.path.abspath(__file__)) + '/' fixtures_path = script_path + '../fixtures/' REALM_ID = get_model_id(Realm, 'zerver_realm', 1) realm, added_users, added_recipient, added_channels = slack_workspace_to_realm( REALM_ID, realm_subdomain, fixtures_path, slack_data_dir) message_json = convert_slack_workspace_messages(slack_data_dir, REALM_ID, added_users, added_recipient, added_channels, realm) zerver_attachment = [] # type: List[ZerverFieldsT] attachment = {"zerver_attachment": zerver_attachment} # IO realm.json create_converted_data_files(realm, output_dir, '/realm.json', False) # IO message.json create_converted_data_files(message_json, output_dir, '/messages-000001.json', False) # IO avatar records create_converted_data_files([], output_dir, '/avatars/records.json', True) # IO uploads TODO create_converted_data_files([], output_dir, '/uploads/records.json', True) # IO attachments create_converted_data_files(attachment, output_dir, '/attachment.json', False) # remove slack dir rm_tree(slack_data_dir) subprocess.check_call( ["tar", "-czf", output_dir + '.tar.gz', output_dir, '-P']) print('######### DATA CONVERSION FINISHED #########\n') print("Zulip data dump created at %s" % (output_dir)) sys.exit(0)
def handle(self, *args: Any, **options: Any) -> None: num_processes = int(options["processes"]) if num_processes < 1: raise CommandError("You must have at least one process.") subdomain = options["subdomain"] if options["destroy_rebuild_database"]: print("Rebuilding the database!") db_name = settings.DATABASES["default"]["NAME"] self.do_destroy_and_rebuild_database(db_name) elif options["import_into_nonempty"]: print( "NOTE: The argument 'import_into_nonempty' is now the default behavior." ) allow_reserved_subdomain = False if options["allow_reserved_subdomain"]: allow_reserved_subdomain = True try: check_subdomain_available(subdomain, allow_reserved_subdomain) except ValidationError: raise CommandError( "Subdomain reserved: pass --allow-reserved-subdomain to use.") paths = [] for path in options["export_paths"]: path = os.path.realpath(os.path.expanduser(path)) if not os.path.exists(path): raise CommandError(f"Directory not found: '{path}'") if not os.path.isdir(path): raise CommandError( "Export file should be folder; if it's a tarball, please unpack it first." ) paths.append(path) for path in paths: print(f"Processing dump: {path} ...") realm = do_import_realm(path, subdomain, num_processes) print("Checking the system bots.") do_import_system_bots(realm)
def support( request: HttpRequest, realm_id: Optional[int] = REQ(default=None, converter=to_non_negative_int), plan_type: Optional[int] = REQ(default=None, converter=to_non_negative_int), discount: Optional[Decimal] = REQ(default=None, converter=to_decimal), new_subdomain: Optional[str] = REQ(default=None), status: Optional[str] = REQ( default=None, str_validator=check_string_in(VALID_STATUS_VALUES)), billing_method: Optional[str] = REQ( default=None, str_validator=check_string_in(VALID_BILLING_METHODS)), sponsorship_pending: Optional[bool] = REQ(default=None, json_validator=check_bool), approve_sponsorship: Optional[bool] = REQ(default=None, json_validator=check_bool), downgrade_method: Optional[str] = REQ( default=None, str_validator=check_string_in(VALID_DOWNGRADE_METHODS)), scrub_realm: Optional[bool] = REQ(default=None, json_validator=check_bool), query: Optional[str] = REQ("q", default=None), org_type: Optional[int] = REQ(default=None, converter=to_non_negative_int), ) -> HttpResponse: context: Dict[str, Any] = {} if "success_message" in request.session: context["success_message"] = request.session["success_message"] del request.session["success_message"] if settings.BILLING_ENABLED and request.method == "POST": # We check that request.POST only has two keys in it: The # realm_id and a field to change. keys = set(request.POST.keys()) if "csrfmiddlewaretoken" in keys: keys.remove("csrfmiddlewaretoken") if len(keys) != 2: raise JsonableError(_("Invalid parameters")) realm = Realm.objects.get(id=realm_id) acting_user = request.user assert isinstance(acting_user, UserProfile) if plan_type is not None: current_plan_type = realm.plan_type do_change_plan_type(realm, plan_type, acting_user=acting_user) msg = f"Plan type of {realm.string_id} changed from {get_plan_name(current_plan_type)} to {get_plan_name(plan_type)} " context["success_message"] = msg elif org_type is not None: current_realm_type = realm.org_type do_change_realm_org_type(realm, org_type, acting_user=acting_user) msg = f"Org type of {realm.string_id} changed from {get_org_type_display_name(current_realm_type)} to {get_org_type_display_name(org_type)} " context["success_message"] = msg elif discount is not None: current_discount = get_discount_for_realm(realm) or 0 attach_discount_to_realm(realm, discount, acting_user=acting_user) context[ "success_message"] = f"Discount of {realm.string_id} changed to {discount}% from {current_discount}%." elif new_subdomain is not None: old_subdomain = realm.string_id try: check_subdomain_available(new_subdomain) except ValidationError as error: context["error_message"] = error.message else: do_change_realm_subdomain(realm, new_subdomain, acting_user=acting_user) request.session[ "success_message"] = f"Subdomain changed from {old_subdomain} to {new_subdomain}" return HttpResponseRedirect( reverse("support") + "?" + urlencode({"q": new_subdomain})) elif status is not None: if status == "active": do_send_realm_reactivation_email(realm, acting_user=acting_user) context[ "success_message"] = f"Realm reactivation email sent to admins of {realm.string_id}." elif status == "deactivated": do_deactivate_realm(realm, acting_user=acting_user) context["success_message"] = f"{realm.string_id} deactivated." elif billing_method is not None: if billing_method == "send_invoice": update_billing_method_of_current_plan( realm, charge_automatically=False, acting_user=acting_user) context[ "success_message"] = f"Billing method of {realm.string_id} updated to pay by invoice." elif billing_method == "charge_automatically": update_billing_method_of_current_plan( realm, charge_automatically=True, acting_user=acting_user) context[ "success_message"] = f"Billing method of {realm.string_id} updated to charge automatically." elif sponsorship_pending is not None: if sponsorship_pending: update_sponsorship_status(realm, True, acting_user=acting_user) context[ "success_message"] = f"{realm.string_id} marked as pending sponsorship." else: update_sponsorship_status(realm, False, acting_user=acting_user) context[ "success_message"] = f"{realm.string_id} is no longer pending sponsorship." elif approve_sponsorship: do_approve_sponsorship(realm, acting_user=acting_user) context[ "success_message"] = f"Sponsorship approved for {realm.string_id}" elif downgrade_method is not None: if downgrade_method == "downgrade_at_billing_cycle_end": downgrade_at_the_end_of_billing_cycle(realm) context[ "success_message"] = f"{realm.string_id} marked for downgrade at the end of billing cycle" elif downgrade_method == "downgrade_now_without_additional_licenses": downgrade_now_without_creating_additional_invoices(realm) context[ "success_message"] = f"{realm.string_id} downgraded without creating additional invoices" elif downgrade_method == "downgrade_now_void_open_invoices": downgrade_now_without_creating_additional_invoices(realm) voided_invoices_count = void_all_open_invoices(realm) context[ "success_message"] = f"{realm.string_id} downgraded and voided {voided_invoices_count} open invoices" elif scrub_realm: do_scrub_realm(realm, acting_user=acting_user) context["success_message"] = f"{realm.string_id} scrubbed." if query: key_words = get_invitee_emails_set(query) users = set(UserProfile.objects.filter(delivery_email__in=key_words)) realms = set(Realm.objects.filter(string_id__in=key_words)) for key_word in key_words: try: URLValidator()(key_word) parse_result = urllib.parse.urlparse(key_word) hostname = parse_result.hostname assert hostname is not None if parse_result.port: hostname = f"{hostname}:{parse_result.port}" subdomain = get_subdomain_from_hostname(hostname) try: realms.add(get_realm(subdomain)) except Realm.DoesNotExist: pass except ValidationError: users.update( UserProfile.objects.filter(full_name__iexact=key_word)) for realm in realms: realm.customer = get_customer_by_realm(realm) current_plan = get_current_plan_by_realm(realm) if current_plan is not None: new_plan, last_ledger_entry = make_end_of_cycle_updates_if_needed( current_plan, timezone_now()) if last_ledger_entry is not None: if new_plan is not None: realm.current_plan = new_plan else: realm.current_plan = current_plan realm.current_plan.licenses = last_ledger_entry.licenses realm.current_plan.licenses_used = get_latest_seat_count( realm) # full_names can have , in them users.update(UserProfile.objects.filter(full_name__iexact=query)) context["users"] = users context["realms"] = realms confirmations: List[Dict[str, Any]] = [] preregistration_users = PreregistrationUser.objects.filter( email__in=key_words) confirmations += get_confirmations( [ Confirmation.USER_REGISTRATION, Confirmation.INVITATION, Confirmation.REALM_CREATION ], preregistration_users, hostname=request.get_host(), ) multiuse_invites = MultiuseInvite.objects.filter(realm__in=realms) confirmations += get_confirmations([Confirmation.MULTIUSE_INVITE], multiuse_invites) confirmations += get_confirmations([Confirmation.REALM_REACTIVATION], [realm.id for realm in realms]) context["confirmations"] = confirmations def get_realm_owner_emails_as_string(realm: Realm) -> str: return ", ".join(realm.get_human_owner_users().order_by( "delivery_email").values_list("delivery_email", flat=True)) def get_realm_admin_emails_as_string(realm: Realm) -> str: return ", ".join( realm.get_human_admin_users(include_realm_owners=False).order_by( "delivery_email").values_list("delivery_email", flat=True)) context[ "get_realm_owner_emails_as_string"] = get_realm_owner_emails_as_string context[ "get_realm_admin_emails_as_string"] = get_realm_admin_emails_as_string context["get_discount_for_realm"] = get_discount_for_realm context["get_org_type_display_name"] = get_org_type_display_name context["realm_icon_url"] = realm_icon_url context["Confirmation"] = Confirmation context["sorted_realm_types"] = sorted(Realm.ORG_TYPES.values(), key=lambda d: d["display_order"]) return render(request, "analytics/support.html", context=context)
def do_convert_data(slack_zip_file: str, realm_subdomain: str, output_dir: str) -> None: check_subdomain_available(realm_subdomain) slack_data_dir = slack_zip_file.replace('.zip', '') if not os.path.exists(slack_data_dir): os.makedirs(slack_data_dir) subprocess.check_call( ['unzip', '-q', slack_zip_file, '-d', slack_data_dir]) # with zipfile.ZipFile(slack_zip_file, 'r') as zip_ref: # zip_ref.extractall(slack_data_dir) # TODO fetch realm config from zulip config DOMAIN_NAME = "zulipchat.com" REALM_ID = get_model_id(Realm, 'zerver_realm', 1) NOW = float(timezone_now().timestamp()) script_path = os.path.dirname(os.path.abspath(__file__)) + '/' fixtures_path = script_path + '../fixtures/' zerver_realm_skeleton = json.load( open(fixtures_path + 'zerver_realm_skeleton.json')) zerver_realm_skeleton[0]['id'] = REALM_ID zerver_realm_skeleton[0][ 'string_id'] = realm_subdomain # subdomain / short_name of realm zerver_realm_skeleton[0]['name'] = realm_subdomain zerver_realm_skeleton[0]['date_created'] = NOW realm = dict( zerver_client=[{ "name": "populate_db", "id": 1 }, { "name": "website", "id": 2 }, { "name": "API", "id": 3 }], zerver_userpresence= [], # shows last logged in data, which is not available in slack zerver_userprofile_mirrordummy=[], zerver_realmdomain=[{ "realm": REALM_ID, "allow_subdomains": False, "domain": DOMAIN_NAME, "id": REALM_ID }], zerver_useractivity=[], zerver_realm=zerver_realm_skeleton, zerver_huddle=[], zerver_userprofile_crossrealm=[], zerver_useractivityinterval=[], zerver_realmfilter=[], zerver_realmemoji=[]) zerver_userprofile, added_users = users_to_zerver_userprofile( slack_data_dir, REALM_ID, int(NOW), DOMAIN_NAME) realm['zerver_userprofile'] = zerver_userprofile channels_to_zerver_stream_fields = channels_to_zerver_stream( slack_data_dir, REALM_ID, added_users, zerver_userprofile) # See https://zulipchat.com/help/set-default-streams-for-new-users # for documentation on zerver_defaultstream realm['zerver_defaultstream'] = channels_to_zerver_stream_fields[0] realm['zerver_stream'] = channels_to_zerver_stream_fields[1] realm['zerver_subscription'] = channels_to_zerver_stream_fields[3] realm['zerver_recipient'] = channels_to_zerver_stream_fields[4] added_channels = channels_to_zerver_stream_fields[2] added_recipient = channels_to_zerver_stream_fields[5] # IO realm.json realm_file = output_dir + '/realm.json' json.dump(realm, open(realm_file, 'w')) # now for message.json message_json = {} zerver_message = [] # type: List[ZerverFieldsT] zerver_usermessage = [] # type: List[ZerverFieldsT] zerver_attachment = [] # type: List[ZerverFieldsT] print('######### IMPORTING MESSAGES STARTED #########\n') # To pre-compute the total number of messages and usermessages total_messages = 0 total_usermessages = 0 for channel in added_channels.keys(): tm, tum = get_total_messages_and_usermessages( slack_data_dir, channel, zerver_userprofile, realm['zerver_subscription'], added_recipient) total_messages += tm total_usermessages += tum message_id_count = get_model_id(Message, 'zerver_message', total_messages) usermessage_id_count = get_model_id(UserMessage, 'zerver_usermessage', total_usermessages) constants = [slack_data_dir, REALM_ID] for channel in added_channels.keys(): message_id = len( zerver_message) + message_id_count # For the id of the messages usermessage_id = len(zerver_usermessage) + usermessage_id_count id_list = [message_id, usermessage_id] zm, zum = channel_message_to_zerver_message( constants, channel, added_users, added_recipient, zerver_userprofile, realm['zerver_subscription'], id_list) zerver_message += zm zerver_usermessage += zum print('######### IMPORTING MESSAGES FINISHED #########\n') message_json['zerver_message'] = zerver_message message_json['zerver_usermessage'] = zerver_usermessage # IO message.json message_file = output_dir + '/messages-000001.json' json.dump(message_json, open(message_file, 'w')) # IO avatar records avatar_records_file = output_dir + '/avatars/records.json' os.makedirs(output_dir + '/avatars', exist_ok=True) json.dump([], open(avatar_records_file, 'w')) # IO uploads TODO uploads_records_file = output_dir + '/uploads/records.json' os.makedirs(output_dir + '/uploads', exist_ok=True) json.dump([], open(uploads_records_file, 'w')) # IO attachments attachment_file = output_dir + '/attachment.json' attachment = {"zerver_attachment": zerver_attachment} json.dump(attachment, open(attachment_file, 'w')) # remove slack dir rm_tree(slack_data_dir) subprocess.check_call( ["tar", "-czf", output_dir + '.tar.gz', output_dir, '-P']) print('######### DATA CONVERSION FINISHED #########\n') print("Zulip data dump created at %s" % (output_dir)) sys.exit(0)
def do_convert_data(slack_zip_file: str, realm_subdomain: str, output_dir: str) -> None: check_subdomain_available(realm_subdomain) slack_data_dir = slack_zip_file.replace('.zip', '') zip_file_dir = os.path.dirname(slack_data_dir) subprocess.check_call(['unzip', '-q', slack_zip_file, '-d', zip_file_dir]) # with zipfile.ZipFile(slack_zip_file, 'r') as zip_ref: # zip_ref.extractall(slack_data_dir) # TODO fetch realm config from zulip config DOMAIN_NAME = "zulipchat.com" REALM_ID = get_model_id(Realm) NOW = float(timezone_now().timestamp()) script_path = os.path.dirname(os.path.abspath(__file__)) + '/' fixtures_path = script_path + '../fixtures/' zerver_realm_skeleton = json.load(open(fixtures_path + 'zerver_realm_skeleton.json')) zerver_realm_skeleton[0]['id'] = REALM_ID zerver_realm_skeleton[0]['string_id'] = realm_subdomain # subdomain / short_name of realm zerver_realm_skeleton[0]['name'] = realm_subdomain zerver_realm_skeleton[0]['date_created'] = NOW realm = dict(zerver_client=[{"name": "populate_db", "id": 1}, {"name": "website", "id": 2}, {"name": "API", "id": 3}], zerver_userpresence=[], # shows last logged in data, which is not available in slack zerver_userprofile_mirrordummy=[], zerver_realmdomain=[{"realm": REALM_ID, "allow_subdomains": False, "domain": DOMAIN_NAME, "id": REALM_ID}], zerver_useractivity=[], zerver_realm=zerver_realm_skeleton, zerver_huddle=[], zerver_userprofile_crossrealm=[], zerver_useractivityinterval=[], zerver_realmfilter=[], zerver_realmemoji=[]) zerver_userprofile, added_users = users_to_zerver_userprofile(slack_data_dir, REALM_ID, int(NOW), DOMAIN_NAME) realm['zerver_userprofile'] = zerver_userprofile channels_to_zerver_stream_fields = channels_to_zerver_stream(slack_data_dir, REALM_ID, added_users, zerver_userprofile) # See https://zulipchat.com/help/set-default-streams-for-new-users # for documentation on zerver_defaultstream realm['zerver_defaultstream'] = channels_to_zerver_stream_fields[0] realm['zerver_stream'] = channels_to_zerver_stream_fields[1] realm['zerver_subscription'] = channels_to_zerver_stream_fields[3] realm['zerver_recipient'] = channels_to_zerver_stream_fields[4] added_channels = channels_to_zerver_stream_fields[2] added_recipient = channels_to_zerver_stream_fields[5] # IO realm.json realm_file = output_dir + '/realm.json' json.dump(realm, open(realm_file, 'w')) # now for message.json message_json = {} zerver_message = [] # type: List[ZerverFieldsT] zerver_usermessage = [] # type: List[ZerverFieldsT] zerver_attachment = [] # type: List[ZerverFieldsT] constants = [slack_data_dir, REALM_ID] message_id_count = get_model_id(Message) usermessage_id_count = get_model_id(UserMessage) print('######### IMPORTING MESSAGES STARTED #########\n') for channel in added_channels.keys(): message_id = len(zerver_message) + message_id_count # For the id of the messages usermessage_id = len(zerver_usermessage) + usermessage_id_count id_list = [message_id, usermessage_id] zm, zum = channel_message_to_zerver_message(constants, channel, added_users, added_recipient, zerver_userprofile, realm['zerver_subscription'], id_list) zerver_message += zm zerver_usermessage += zum print('######### IMPORTING MESSAGES FINISHED #########\n') message_json['zerver_message'] = zerver_message message_json['zerver_usermessage'] = zerver_usermessage # IO message.json message_file = output_dir + '/messages-000001.json' json.dump(message_json, open(message_file, 'w')) # IO avatar records avatar_records_file = output_dir + '/avatars/records.json' os.makedirs(output_dir + '/avatars', exist_ok=True) json.dump([], open(avatar_records_file, 'w')) # IO uploads TODO uploads_records_file = output_dir + '/uploads/records.json' os.makedirs(output_dir + '/uploads', exist_ok=True) json.dump([], open(uploads_records_file, 'w')) # IO attachments attachment_file = output_dir + '/attachment.json' attachment = {"zerver_attachment": zerver_attachment} json.dump(attachment, open(attachment_file, 'w')) # remove slack dir rm_tree(slack_data_dir) subprocess.check_call(["tar", "-czf", output_dir + '.tar.gz', output_dir, '-P']) print('######### DATA CONVERSION FINISHED #########\n') print("Zulip data dump created at %s" % (output_dir)) print("Import Command: ./manage.py import --destroy-rebuild-database %s\n" % (output_dir)) sys.exit(0)
def do_convert_data(slack_zip_file: str, realm_subdomain: str, output_dir: str, token: str) -> None: check_subdomain_available(realm_subdomain) domain_name = settings.EXTERNAL_HOST slack_data_dir = slack_zip_file.replace('.zip', '') if not os.path.exists(slack_data_dir): os.makedirs(slack_data_dir) os.makedirs(output_dir, exist_ok=True) # output directory should be empty initially if os.listdir(output_dir): raise Exception('Output directory should be empty!') subprocess.check_call( ['unzip', '-q', slack_zip_file, '-d', slack_data_dir]) # with zipfile.ZipFile(slack_zip_file, 'r') as zip_ref: # zip_ref.extractall(slack_data_dir) script_path = os.path.dirname(os.path.abspath(__file__)) + '/' fixtures_path = script_path + '../fixtures/' realm_id = 0 # We get the user data from the legacy token method of slack api, which is depreciated # but we use it as the user email data is provided only in this method user_list = get_slack_api_data(token, "https://slack.com/api/users.list", "members") # Get custom emoji from slack api custom_emoji_list = get_slack_api_data(token, "https://slack.com/api/emoji.list", "emoji") realm, added_users, added_recipient, added_channels, avatar_list, \ emoji_url_map = slack_workspace_to_realm(domain_name, realm_id, user_list, realm_subdomain, fixtures_path, slack_data_dir, custom_emoji_list) message_json, uploads_list, zerver_attachment = convert_slack_workspace_messages( slack_data_dir, user_list, realm_id, added_users, added_recipient, added_channels, realm, realm['zerver_realmemoji'], domain_name) emoji_folder = os.path.join(output_dir, 'emoji') os.makedirs(emoji_folder, exist_ok=True) emoji_records = process_emojis(realm['zerver_realmemoji'], emoji_folder, emoji_url_map) avatar_folder = os.path.join(output_dir, 'avatars') avatar_realm_folder = os.path.join(avatar_folder, str(realm_id)) os.makedirs(avatar_realm_folder, exist_ok=True) avatar_records = process_avatars(avatar_list, avatar_folder, realm_id) uploads_folder = os.path.join(output_dir, 'uploads') os.makedirs(os.path.join(uploads_folder, str(realm_id)), exist_ok=True) uploads_records = process_uploads(uploads_list, uploads_folder) attachment = {"zerver_attachment": zerver_attachment} # IO realm.json create_converted_data_files(realm, output_dir, '/realm.json') # IO message.json create_converted_data_files(message_json, output_dir, '/messages-000001.json') # IO emoji records create_converted_data_files(emoji_records, output_dir, '/emoji/records.json') # IO avatar records create_converted_data_files(avatar_records, output_dir, '/avatars/records.json') # IO uploads TODO create_converted_data_files(uploads_records, output_dir, '/uploads/records.json') # IO attachments create_converted_data_files(attachment, output_dir, '/attachment.json') # remove slack dir rm_tree(slack_data_dir) subprocess.check_call( ["tar", "-czf", output_dir + '.tar.gz', output_dir, '-P']) logging.info('######### DATA CONVERSION FINISHED #########\n') logging.info("Zulip data dump created at %s" % (output_dir))