def test_convert_huddle_data(self) -> None: fixture_dir_name = self.fixture_file_name("", "rocketchat_fixtures") rocketchat_data = rocketchat_data_to_dict(fixture_dir_name) realm_id = 3 domain_name = "zulip.com" user_handler = UserHandler() subscriber_handler = SubscriberHandler() user_id_mapper = IdMapper() huddle_id_mapper = IdMapper() user_id_to_user_map = map_user_id_to_user(rocketchat_data["user"]) process_users( user_id_to_user_map=user_id_to_user_map, realm_id=realm_id, domain_name=domain_name, user_handler=user_handler, user_id_mapper=user_id_mapper, ) room_id_to_room_map: Dict[str, Dict[str, Any]] = {} team_id_to_team_map: Dict[str, Dict[str, Any]] = {} dsc_id_to_dsc_map: Dict[str, Dict[str, Any]] = {} direct_id_to_direct_map: Dict[str, Dict[str, Any]] = {} huddle_id_to_huddle_map: Dict[str, Dict[str, Any]] = {} livechat_id_to_livechat_map: Dict[str, Dict[str, Any]] = {} categorize_channels_and_map_with_id( channel_data=rocketchat_data["room"], room_id_to_room_map=room_id_to_room_map, team_id_to_team_map=team_id_to_team_map, dsc_id_to_dsc_map=dsc_id_to_dsc_map, direct_id_to_direct_map=direct_id_to_direct_map, huddle_id_to_huddle_map=huddle_id_to_huddle_map, livechat_id_to_livechat_map=livechat_id_to_livechat_map, ) zerver_huddle = convert_huddle_data( huddle_id_to_huddle_map=huddle_id_to_huddle_map, huddle_id_mapper=huddle_id_mapper, user_id_mapper=user_id_mapper, subscriber_handler=subscriber_handler, ) self.assert_length(zerver_huddle, 1) rc_huddle_id = rocketchat_data["room"][12]["_id"] self.assertTrue(huddle_id_mapper.has(rc_huddle_id)) huddle_id = huddle_id_mapper.get(rc_huddle_id) self.assertEqual(subscriber_handler.get_users(huddle_id=huddle_id), {3, 4, 5})
def test_convert_user_data(self) -> None: user_id_mapper = IdMapper() realm_id = 3 team_name = "gryffindor" user_handler = UserHandler() convert_user_data(user_handler, user_id_mapper, self.username_to_user, realm_id, team_name) self.assertTrue(user_id_mapper.has("harry")) self.assertTrue(user_id_mapper.has("ron")) self.assertEqual(user_handler.get_user(user_id_mapper.get("harry"))["full_name"], "Harry Potter") self.assertEqual(user_handler.get_user(user_id_mapper.get("ron"))["full_name"], "Ron Weasley") team_name = "slytherin" user_handler = UserHandler() convert_user_data(user_handler, user_id_mapper, self.username_to_user, realm_id, team_name) self.assertEqual(len(user_handler.get_all_users()), 3) self.assertTrue(user_id_mapper.has("malfoy")) self.assertTrue(user_id_mapper.has("pansy")) self.assertTrue(user_id_mapper.has("snape")) team_name = "gryffindor" # Snape is a mirror dummy user in Harry's team. label_mirror_dummy_users(team_name, self.mattermost_data, self.username_to_user) user_handler = UserHandler() convert_user_data(user_handler, user_id_mapper, self.username_to_user, realm_id, team_name) self.assertEqual(len(user_handler.get_all_users()), 3) self.assertTrue(user_id_mapper.has("snape")) team_name = "slytherin" user_handler = UserHandler() convert_user_data(user_handler, user_id_mapper, self.username_to_user, realm_id, team_name) self.assertEqual(len(user_handler.get_all_users()), 3)
def test_build_reactions(self) -> None: total_reactions = [] # type: List[Dict[str, Any]] reactions = [{ "user": "******", "create_at": 1553165521410, "emoji_name": "tick" }, { "user": "******", "create_at": 1553166530805, "emoji_name": "smile" }, { "user": "******", "create_at": 1553166540953, "emoji_name": "world_map" }, { "user": "******", "create_at": 1553166540957, "emoji_name": "world_map" }] zerver_realmemoji = write_emoticon_data( realm_id=3, custom_emoji_data=self.mattermost_data["emoji"], data_dir=self.fixture_file_name("", "mattermost_fixtures"), output_dir=self.make_import_output_dir("mattermost")) # Make sure tick is present in fixture data self.assertEqual(zerver_realmemoji[1]["name"], "tick") tick_emoji_code = zerver_realmemoji[1]["id"] name_to_codepoint = get_name_to_codepoint_dict() user_id_mapper = IdMapper() harry_id = user_id_mapper.get("harry") ron_id = user_id_mapper.get("ron") build_reactions(realm_id=3, total_reactions=total_reactions, reactions=reactions, message_id=5, name_to_codepoint=name_to_codepoint, user_id_mapper=user_id_mapper, zerver_realmemoji=zerver_realmemoji) smile_emoji_code = name_to_codepoint["smile"] world_map_emoji_code = name_to_codepoint["world_map"] self.assertEqual(len(total_reactions), 4) self.assertEqual(self.get_set(total_reactions, "reaction_type"), set([Reaction.REALM_EMOJI, Reaction.UNICODE_EMOJI])) self.assertEqual(self.get_set(total_reactions, "emoji_name"), set(["tick", "smile", "world_map"])) self.assertEqual( self.get_set(total_reactions, "emoji_code"), set([tick_emoji_code, smile_emoji_code, world_map_emoji_code])) self.assertEqual(self.get_set(total_reactions, "user_profile"), set([harry_id, ron_id])) self.assertEqual(len(self.get_set(total_reactions, "id")), 4) self.assertEqual(len(self.get_set(total_reactions, "message")), 1)
def test_build_reactions(self) -> None: total_reactions = [] # type: List[Dict[str, Any]] reactions = [ {"user": "******", "create_at": 1553165521410, "emoji_name": "tick"}, {"user": "******", "create_at": 1553166530805, "emoji_name": "smile"}, {"user": "******", "create_at": 1553166540953, "emoji_name": "world_map"}, {"user": "******", "create_at": 1553166540957, "emoji_name": "world_map"} ] zerver_realmemoji = write_emoticon_data( realm_id=3, custom_emoji_data=self.mattermost_data["emoji"], data_dir=self.fixture_file_name("", "mattermost_fixtures"), output_dir=self.make_import_output_dir("mattermost") ) # Make sure tick is present in fixture data self.assertEqual(zerver_realmemoji[1]["name"], "tick") tick_emoji_code = zerver_realmemoji[1]["id"] name_to_codepoint = get_name_to_codepoint_dict() user_id_mapper = IdMapper() harry_id = user_id_mapper.get("harry") ron_id = user_id_mapper.get("ron") build_reactions( realm_id=3, total_reactions=total_reactions, reactions=reactions, message_id=5, name_to_codepoint=name_to_codepoint, user_id_mapper=user_id_mapper, zerver_realmemoji=zerver_realmemoji ) smile_emoji_code = name_to_codepoint["smile"] world_map_emoji_code = name_to_codepoint["world_map"] expected_total_reactions = [ { 'user_profile': harry_id, 'message': 5, 'id': 1, 'reaction_type': Reaction.REALM_EMOJI, 'emoji_code': tick_emoji_code, 'emoji_name': 'tick' }, { 'user_profile': ron_id, 'message': 5, 'id': 2, 'reaction_type': Reaction.UNICODE_EMOJI, 'emoji_code': smile_emoji_code, 'emoji_name': 'smile' }, { 'user_profile': ron_id, 'message': 5, 'id': 3, 'reaction_type': Reaction.UNICODE_EMOJI, 'emoji_code': world_map_emoji_code, 'emoji_name': 'world_map' }, { 'user_profile': harry_id, 'message': 5, 'id': 4, 'reaction_type': Reaction.UNICODE_EMOJI, 'emoji_code': world_map_emoji_code, 'emoji_name': 'world_map' } ] self.assertEqual(total_reactions, expected_total_reactions)
def test_process_message_attachment(self) -> None: fixture_dir_name = self.fixture_file_name("", "rocketchat_fixtures") rocketchat_data = rocketchat_data_to_dict(fixture_dir_name) output_dir = self.make_import_output_dir("mattermost") user_id_to_user_map = map_user_id_to_user(rocketchat_data["user"]) realm_id = 3 domain_name = "zulip.com" user_handler = UserHandler() user_id_mapper = IdMapper() process_users( user_id_to_user_map=user_id_to_user_map, realm_id=realm_id, domain_name=domain_name, user_handler=user_handler, user_id_mapper=user_id_mapper, ) zerver_attachments: List[ZerverFieldsT] = [] uploads_list: List[ZerverFieldsT] = [] upload_id_to_upload_data_map = map_upload_id_to_upload_data( rocketchat_data["upload"]) message_with_attachment = rocketchat_data["message"][55] process_message_attachment( upload=message_with_attachment["file"], realm_id=3, message_id=1, user_id=3, user_handler=user_handler, zerver_attachment=zerver_attachments, uploads_list=uploads_list, upload_id_to_upload_data_map=upload_id_to_upload_data_map, output_dir=output_dir, ) self.assert_length(zerver_attachments, 1) self.assertEqual(zerver_attachments[0]["file_name"], "harry-ron.jpg") self.assertEqual(zerver_attachments[0]["owner"], 3) self.assertEqual( user_handler.get_user(zerver_attachments[0]["owner"])["email"], "*****@*****.**") # TODO: Assert this for False after fixing the file permissions in PMs self.assertTrue(zerver_attachments[0]["is_realm_public"]) self.assert_length(uploads_list, 1) self.assertEqual(uploads_list[0]["user_profile_email"], "*****@*****.**") attachment_out_path = os.path.join(output_dir, "uploads", zerver_attachments[0]["path_id"]) self.assertTrue(os.path.exists(attachment_out_path)) self.assertTrue(os.path.isfile(attachment_out_path))
def test_sender_ids(self) -> None: realm_id = 5 user_handler = UserHandler() user_id_mapper = IdMapper() user_id_mapper.has = lambda key: True # type: ignore[assignment] # it's just a stub # Simulate a "normal" user first. user_with_id = dict(id=1, # other fields don't matter here ) user_handler.add_user(user=user_with_id) normal_message: Dict[str, Any] = dict(sender=dict(id=1, ), ) sender_id = get_hipchat_sender_id( realm_id=realm_id, slim_mode=False, message_dict=normal_message, user_id_mapper=user_id_mapper, user_handler=user_handler, ) self.assertEqual(sender_id, 1) bot_message = dict(sender='fred_bot', ) # Every message from fred_bot should # return the same sender_id. fred_bot_sender_id = 2 for i in range(3): sender_id = get_hipchat_sender_id( realm_id=realm_id, slim_mode=False, message_dict=bot_message, user_id_mapper=user_id_mapper, user_handler=user_handler, ) self.assertEqual(sender_id, fred_bot_sender_id) id_zero_message = dict(sender=dict( id=0, name='hal_bot', ), ) hal_bot_sender_id = 3 for i in range(3): sender_id = get_hipchat_sender_id( realm_id=realm_id, slim_mode=False, message_dict=id_zero_message, user_id_mapper=user_id_mapper, user_handler=user_handler, ) self.assertEqual(sender_id, hal_bot_sender_id)
def test_convert_huddle_data(self) -> None: fixture_file_name = self.fixture_file_name( "export.json", "mattermost_fixtures/direct_channel") mattermost_data = mattermost_data_file_to_dict(fixture_file_name) username_to_user = create_username_to_user_mapping( mattermost_data["user"]) reset_mirror_dummy_users(username_to_user) user_handler = UserHandler() subscriber_handler = SubscriberHandler() huddle_id_mapper = IdMapper() user_id_mapper = IdMapper() team_name = "gryffindor" convert_user_data( user_handler=user_handler, user_id_mapper=user_id_mapper, user_data_map=username_to_user, realm_id=3, team_name=team_name, ) zerver_huddle = convert_huddle_data( huddle_data=mattermost_data["direct_channel"], user_data_map=username_to_user, subscriber_handler=subscriber_handler, huddle_id_mapper=huddle_id_mapper, user_id_mapper=user_id_mapper, realm_id=3, team_name=team_name, ) self.assert_length(zerver_huddle, 1) huddle_members = mattermost_data["direct_channel"][1]["members"] huddle_name = generate_huddle_name(huddle_members) self.assertTrue(huddle_id_mapper.has(huddle_name)) self.assertEqual( subscriber_handler.get_users( huddle_id=huddle_id_mapper.get(huddle_name)), {1, 2, 3})
def test_get_mentioned_user_ids(self) -> None: user_id_mapper = IdMapper() harry_id = user_id_mapper.get("harry") raw_message = {"content": "Hello @harry"} ids = get_mentioned_user_ids(raw_message, user_id_mapper) self.assertEqual(list(ids), [harry_id]) raw_message = {"content": "Hello"} ids = get_mentioned_user_ids(raw_message, user_id_mapper) self.assertEqual(list(ids), []) raw_message = {"content": "@harry How are you?"} ids = get_mentioned_user_ids(raw_message, user_id_mapper) self.assertEqual(list(ids), [harry_id]) raw_message = {"content": "@harry @ron Where are you folks?"} ron_id = user_id_mapper.get("ron") ids = get_mentioned_user_ids(raw_message, user_id_mapper) self.assertEqual(list(ids), [harry_id, ron_id]) raw_message = {"content": "@harry.com How are you?"} ids = get_mentioned_user_ids(raw_message, user_id_mapper) self.assertEqual(list(ids), []) raw_message = {"content": "[email protected] How are you?"} ids = get_mentioned_user_ids(raw_message, user_id_mapper) self.assertEqual(list(ids), []) harry_id = user_id_mapper.get("harry_") raw_message = {"content": "Hello @harry_"} ids = get_mentioned_user_ids(raw_message, user_id_mapper) self.assertEqual(list(ids), [harry_id]) harry_id = user_id_mapper.get("harry.") raw_message = {"content": "Hello @harry."} ids = get_mentioned_user_ids(raw_message, user_id_mapper) self.assertEqual(list(ids), [harry_id]) harry_id = user_id_mapper.get("ha_rry.") raw_message = {"content": "Hello @ha_rry."} ids = get_mentioned_user_ids(raw_message, user_id_mapper) self.assertEqual(list(ids), [harry_id]) ron_id = user_id_mapper.get("ron") raw_message = {"content": "Hello @ron."} ids = get_mentioned_user_ids(raw_message, user_id_mapper) self.assertEqual(list(ids), []) raw_message = {"content": "Hello @ron_"} ids = get_mentioned_user_ids(raw_message, user_id_mapper) self.assertEqual(list(ids), [])
def test_build_reactions(self) -> None: fixture_file_name = self.fixture_file_name("export.json", "mattermost_fixtures") mattermost_data = mattermost_data_file_to_dict(fixture_file_name) total_reactions: List[Dict[str, Any]] = [] reactions = [ {"user": "******", "create_at": 1553165521410, "emoji_name": "tick"}, {"user": "******", "create_at": 1553166530805, "emoji_name": "smile"}, {"user": "******", "create_at": 1553166540953, "emoji_name": "world_map"}, {"user": "******", "create_at": 1553166540957, "emoji_name": "world_map"}, ] with self.assertLogs(level="INFO"): zerver_realmemoji = write_emoticon_data( realm_id=3, custom_emoji_data=mattermost_data["emoji"], data_dir=self.fixture_file_name("", "mattermost_fixtures"), output_dir=self.make_import_output_dir("mattermost"), ) # Make sure tick is present in fixture data self.assertEqual(zerver_realmemoji[1]["name"], "tick") tick_emoji_code = zerver_realmemoji[1]["id"] user_id_mapper = IdMapper() harry_id = user_id_mapper.get("harry") ron_id = user_id_mapper.get("ron") build_reactions( realm_id=3, total_reactions=total_reactions, reactions=reactions, message_id=5, user_id_mapper=user_id_mapper, zerver_realmemoji=zerver_realmemoji, ) smile_emoji_code = name_to_codepoint["smile"] world_map_emoji_code = name_to_codepoint["world_map"] self.assertEqual(len(total_reactions), 4) self.assertEqual(self.get_set(total_reactions, "reaction_type"), {Reaction.REALM_EMOJI, Reaction.UNICODE_EMOJI}) self.assertEqual(self.get_set(total_reactions, "emoji_name"), {"tick", "smile", "world_map"}) self.assertEqual(self.get_set(total_reactions, "emoji_code"), {tick_emoji_code, smile_emoji_code, world_map_emoji_code}) self.assertEqual(self.get_set(total_reactions, "user_profile"), {harry_id, ron_id}) self.assertEqual(len(self.get_set(total_reactions, "id")), 4) self.assertEqual(len(self.get_set(total_reactions, "message")), 1)
def test_convert_user_data(self) -> None: user_id_mapper = IdMapper() realm_id = 3 fixture_file_name = self.fixture_file_name("export.json", "mattermost_fixtures") mattermost_data = mattermost_data_file_to_dict(fixture_file_name) username_to_user = create_username_to_user_mapping( mattermost_data["user"]) reset_mirror_dummy_users(username_to_user) team_name = "gryffindor" user_handler = UserHandler() convert_user_data(user_handler, user_id_mapper, username_to_user, realm_id, team_name) self.assert_length(user_handler.get_all_users(), 2) self.assertTrue(user_id_mapper.has("harry")) self.assertTrue(user_id_mapper.has("ron")) self.assertEqual( user_handler.get_user(user_id_mapper.get("harry"))["full_name"], "Harry Potter") self.assertEqual( user_handler.get_user(user_id_mapper.get("ron"))["full_name"], "Ron Weasley") team_name = "slytherin" user_handler = UserHandler() convert_user_data(user_handler, user_id_mapper, username_to_user, realm_id, team_name) self.assert_length(user_handler.get_all_users(), 3) self.assertTrue(user_id_mapper.has("malfoy")) self.assertTrue(user_id_mapper.has("pansy")) self.assertTrue(user_id_mapper.has("snape")) team_name = "gryffindor" # Snape is a mirror dummy user in Harry's team. label_mirror_dummy_users(2, team_name, mattermost_data, username_to_user) user_handler = UserHandler() convert_user_data(user_handler, user_id_mapper, username_to_user, realm_id, team_name) self.assert_length(user_handler.get_all_users(), 3) self.assertTrue(user_id_mapper.has("snape")) team_name = "slytherin" user_handler = UserHandler() convert_user_data(user_handler, user_id_mapper, username_to_user, realm_id, team_name) self.assert_length(user_handler.get_all_users(), 3)
def test_process_user(self) -> None: user_id_mapper = IdMapper() fixture_file_name = self.fixture_file_name("export.json", "mattermost_fixtures") mattermost_data = mattermost_data_file_to_dict(fixture_file_name) username_to_user = create_username_to_user_mapping( mattermost_data["user"]) reset_mirror_dummy_users(username_to_user) harry_dict = username_to_user["harry"] harry_dict["is_mirror_dummy"] = False realm_id = 3 team_name = "gryffindor" user = process_user(harry_dict, realm_id, team_name, user_id_mapper) self.assertEqual(user["avatar_source"], "G") self.assertEqual(user["delivery_email"], "*****@*****.**") self.assertEqual(user["email"], "*****@*****.**") self.assertEqual(user["full_name"], "Harry Potter") self.assertEqual(user["id"], 1) self.assertEqual(user["is_active"], True) self.assertEqual(user["role"], UserProfile.ROLE_REALM_OWNER) self.assertEqual(user["is_mirror_dummy"], False) self.assertEqual(user["realm"], 3) self.assertEqual(user["short_name"], "harry") self.assertEqual(user["timezone"], "UTC") # A user with a `null` team value shouldn't be an admin. harry_dict["teams"] = None user = process_user(harry_dict, realm_id, team_name, user_id_mapper) self.assertEqual(user["role"], UserProfile.ROLE_MEMBER) team_name = "slytherin" snape_dict = username_to_user["snape"] snape_dict["is_mirror_dummy"] = True user = process_user(snape_dict, realm_id, team_name, user_id_mapper) self.assertEqual(user["avatar_source"], "G") self.assertEqual(user["delivery_email"], "*****@*****.**") self.assertEqual(user["email"], "*****@*****.**") self.assertEqual(user["full_name"], "Severus Snape") self.assertEqual(user["id"], 2) self.assertEqual(user["is_active"], False) self.assertEqual(user["role"], UserProfile.ROLE_MEMBER) self.assertEqual(user["is_mirror_dummy"], True) self.assertEqual(user["realm"], 3) self.assertEqual(user["short_name"], "snape") self.assertEqual(user["timezone"], "UTC")
def test_process_user(self) -> None: user_id_mapper = IdMapper() harry_dict = self.username_to_user["harry"] harry_dict["is_mirror_dummy"] = False realm_id = 3 team_name = "gryffindor" user = process_user(harry_dict, realm_id, team_name, user_id_mapper) self.assertEqual(user["avatar_source"], 'G') self.assertEqual(user["delivery_email"], "*****@*****.**") self.assertEqual(user["email"], "*****@*****.**") self.assertEqual(user["full_name"], "Harry Potter") self.assertEqual(user["id"], 1) self.assertEqual(user["is_active"], True) self.assertEqual(user["is_realm_admin"], True) self.assertEqual(user["is_guest"], False) self.assertEqual(user["is_mirror_dummy"], False) self.assertEqual(user["realm"], 3) self.assertEqual(user["short_name"], "harry") self.assertEqual(user["timezone"], "UTC") # A user with a `null` team value shouldn't be an admin. harry_dict["teams"] = None user = process_user(harry_dict, realm_id, team_name, user_id_mapper) self.assertEqual(user["is_realm_admin"], False) team_name = "slytherin" snape_dict = self.username_to_user["snape"] snape_dict["is_mirror_dummy"] = True user = process_user(snape_dict, realm_id, team_name, user_id_mapper) self.assertEqual(user["avatar_source"], 'G') self.assertEqual(user["delivery_email"], "*****@*****.**") self.assertEqual(user["email"], "*****@*****.**") self.assertEqual(user["full_name"], "Severus Snape") self.assertEqual(user["id"], 2) self.assertEqual(user["is_active"], False) self.assertEqual(user["is_realm_admin"], False) self.assertEqual(user["is_guest"], False) self.assertEqual(user["is_mirror_dummy"], True) self.assertEqual(user["realm"], 3) self.assertEqual(user["short_name"], "snape") self.assertEqual(user["timezone"], "UTC")
def test_convert_channel_data(self) -> None: fixture_file_name = self.fixture_file_name("export.json", "mattermost_fixtures") mattermost_data = mattermost_data_file_to_dict(fixture_file_name) username_to_user = create_username_to_user_mapping(mattermost_data["user"]) reset_mirror_dummy_users(username_to_user) user_handler = UserHandler() subscriber_handler = SubscriberHandler() stream_id_mapper = IdMapper() user_id_mapper = IdMapper() team_name = "gryffindor" convert_user_data( user_handler=user_handler, user_id_mapper=user_id_mapper, user_data_map=username_to_user, realm_id=3, team_name=team_name, ) zerver_stream = convert_channel_data( channel_data=mattermost_data["channel"], user_data_map=username_to_user, subscriber_handler=subscriber_handler, stream_id_mapper=stream_id_mapper, user_id_mapper=user_id_mapper, realm_id=3, team_name=team_name, ) self.assertEqual(len(zerver_stream), 3) self.assertEqual(zerver_stream[0]["name"], "Gryffindor common room") self.assertEqual(zerver_stream[0]["invite_only"], False) self.assertEqual(zerver_stream[0]["description"], "A place for talking about Gryffindor common room") self.assertEqual(zerver_stream[0]["rendered_description"], "") self.assertEqual(zerver_stream[0]["realm"], 3) self.assertEqual(zerver_stream[1]["name"], "Gryffindor quidditch team") self.assertEqual(zerver_stream[1]["invite_only"], False) self.assertEqual(zerver_stream[1]["description"], "A place for talking about Gryffindor quidditch team") self.assertEqual(zerver_stream[1]["rendered_description"], "") self.assertEqual(zerver_stream[1]["realm"], 3) self.assertEqual(zerver_stream[2]["name"], "Dumbledores army") self.assertEqual(zerver_stream[2]["invite_only"], True) self.assertEqual(zerver_stream[2]["description"], "A place for talking about Dumbledores army") self.assertEqual(zerver_stream[2]["rendered_description"], "") self.assertEqual(zerver_stream[2]["realm"], 3) self.assertTrue(stream_id_mapper.has("gryffindor-common-room")) self.assertTrue(stream_id_mapper.has("gryffindor-quidditch-team")) self.assertTrue(stream_id_mapper.has("dumbledores-army")) # TODO: Add ginny ron_id = user_id_mapper.get("ron") harry_id = user_id_mapper.get("harry") self.assertEqual({ron_id, harry_id}, {1, 2}) self.assertEqual(subscriber_handler.get_users(stream_id=stream_id_mapper.get("gryffindor-common-room")), {ron_id, harry_id}) self.assertEqual(subscriber_handler.get_users(stream_id=stream_id_mapper.get("gryffindor-quidditch-team")), {ron_id, harry_id}) self.assertEqual(subscriber_handler.get_users(stream_id=stream_id_mapper.get("dumbledores-army")), {ron_id, harry_id}) # Converting channel data when a user's `teams` value is `null`. username_to_user["ron"].update({"teams": None}) zerver_stream = convert_channel_data( channel_data=mattermost_data["channel"], user_data_map=username_to_user, subscriber_handler=subscriber_handler, stream_id_mapper=stream_id_mapper, user_id_mapper=user_id_mapper, realm_id=3, team_name=team_name, ) harry_id = user_id_mapper.get("harry") self.assertIn(harry_id, {1, 2}) self.assertEqual(subscriber_handler.get_users(stream_id=stream_id_mapper.get("gryffindor-common-room")), {harry_id}) self.assertEqual(subscriber_handler.get_users(stream_id=stream_id_mapper.get("gryffindor-quidditch-team")), {harry_id}) self.assertEqual(subscriber_handler.get_users(stream_id=stream_id_mapper.get("dumbledores-army")), {harry_id}) team_name = "slytherin" zerver_stream = convert_channel_data( channel_data=mattermost_data["channel"], user_data_map=username_to_user, subscriber_handler=subscriber_handler, stream_id_mapper=stream_id_mapper, user_id_mapper=user_id_mapper, realm_id=4, team_name=team_name, ) malfoy_id = user_id_mapper.get("malfoy") pansy_id = user_id_mapper.get("pansy") snape_id = user_id_mapper.get("snape") self.assertEqual({malfoy_id, pansy_id, snape_id}, {3, 4, 5}) self.assertEqual(subscriber_handler.get_users(stream_id=stream_id_mapper.get("slytherin-common-room")), {malfoy_id, pansy_id, snape_id}) self.assertEqual(subscriber_handler.get_users(stream_id=stream_id_mapper.get("slytherin-quidditch-team")), {malfoy_id, pansy_id})
def do_convert_data(input_tar_file: str, output_dir: str, masking_content: bool, api_token: Optional[str] = None, slim_mode: bool = False) -> None: input_data_dir = untar_input_file(input_tar_file) attachment_handler = AttachmentHandler() user_handler = UserHandler() subscriber_handler = SubscriberHandler() user_id_mapper = IdMapper() stream_id_mapper = IdMapper() realm_id = 0 realm = make_realm(realm_id=realm_id) # users.json -> UserProfile raw_user_data = read_user_data(data_dir=input_data_dir) convert_user_data( user_handler=user_handler, slim_mode=slim_mode, user_id_mapper=user_id_mapper, raw_data=raw_user_data, realm_id=realm_id, ) normal_users = user_handler.get_normal_users() # Don't write zerver_userprofile here, because we # may add more users later. # streams.json -> Stream raw_stream_data = read_room_data(data_dir=input_data_dir) zerver_stream = convert_room_data( raw_data=raw_stream_data, subscriber_handler=subscriber_handler, stream_id_mapper=stream_id_mapper, user_id_mapper=user_id_mapper, realm_id=realm_id, api_token=api_token, ) realm['zerver_stream'] = zerver_stream zerver_recipient = build_recipients( zerver_userprofile=normal_users, zerver_stream=zerver_stream, ) realm['zerver_recipient'] = zerver_recipient if api_token is None: if slim_mode: public_stream_subscriptions: List[ZerverFieldsT] = [] else: public_stream_subscriptions = build_public_stream_subscriptions( zerver_userprofile=normal_users, zerver_recipient=zerver_recipient, zerver_stream=zerver_stream, ) private_stream_subscriptions = build_stream_subscriptions( get_users=subscriber_handler.get_users, zerver_recipient=zerver_recipient, zerver_stream=[ stream_dict for stream_dict in zerver_stream if stream_dict['invite_only'] ], ) stream_subscriptions = public_stream_subscriptions + private_stream_subscriptions else: stream_subscriptions = build_stream_subscriptions( get_users=subscriber_handler.get_users, zerver_recipient=zerver_recipient, zerver_stream=zerver_stream, ) personal_subscriptions = build_personal_subscriptions( zerver_recipient=zerver_recipient, ) zerver_subscription = personal_subscriptions + stream_subscriptions realm['zerver_subscription'] = zerver_subscription zerver_realmemoji = write_emoticon_data( realm_id=realm_id, data_dir=input_data_dir, output_dir=output_dir, ) realm['zerver_realmemoji'] = zerver_realmemoji subscriber_map = make_subscriber_map( zerver_subscription=zerver_subscription, ) logging.info('Start importing message data') for message_key in [ 'UserMessage', 'NotificationMessage', 'PrivateUserMessage' ]: write_message_data( realm_id=realm_id, slim_mode=slim_mode, message_key=message_key, zerver_recipient=zerver_recipient, subscriber_map=subscriber_map, data_dir=input_data_dir, output_dir=output_dir, masking_content=masking_content, stream_id_mapper=stream_id_mapper, user_id_mapper=user_id_mapper, user_handler=user_handler, attachment_handler=attachment_handler, ) # Order is important here...don't write users until # we process everything else, since we may introduce # mirror users when processing messages. realm['zerver_userprofile'] = user_handler.get_all_users() realm['sort_by_date'] = True create_converted_data_files(realm, output_dir, '/realm.json') logging.info('Start importing avatar data') write_avatar_data( raw_user_data=raw_user_data, output_dir=output_dir, user_id_mapper=user_id_mapper, realm_id=realm_id, ) attachment_handler.write_info( output_dir=output_dir, realm_id=realm_id, ) logging.info('Start making tarball') subprocess.check_call( ["tar", "-czf", output_dir + '.tar.gz', output_dir, '-P']) logging.info('Done making tarball')
def test_convert_channel_data(self) -> None: user_handler = UserHandler() subscriber_handler = SubscriberHandler() stream_id_mapper = IdMapper() user_id_mapper = IdMapper() team_name = "gryffindor" convert_user_data( user_handler=user_handler, user_id_mapper=user_id_mapper, user_data_map=self.username_to_user, realm_id=3, team_name=team_name, ) zerver_stream = convert_channel_data( channel_data=self.mattermost_data["channel"], user_data_map=self.username_to_user, subscriber_handler=subscriber_handler, stream_id_mapper=stream_id_mapper, user_id_mapper=user_id_mapper, realm_id=3, team_name=team_name, ) self.assertEqual(len(zerver_stream), 3) self.assertEqual(zerver_stream[0]["name"], "Gryffindor common room") self.assertEqual(zerver_stream[0]["invite_only"], False) self.assertEqual(zerver_stream[0]["description"], "A place for talking about Gryffindor common room") self.assertEqual(zerver_stream[0]["rendered_description"], "") self.assertEqual(zerver_stream[0]["realm"], 3) self.assertEqual(zerver_stream[1]["name"], "Gryffindor quidditch team") self.assertEqual(zerver_stream[1]["invite_only"], False) self.assertEqual( zerver_stream[1]["description"], "A place for talking about Gryffindor quidditch team") self.assertEqual(zerver_stream[1]["rendered_description"], "") self.assertEqual(zerver_stream[1]["realm"], 3) self.assertEqual(zerver_stream[2]["name"], "Dumbledores army") self.assertEqual(zerver_stream[2]["invite_only"], True) self.assertEqual(zerver_stream[2]["description"], "A place for talking about Dumbledores army") self.assertEqual(zerver_stream[2]["rendered_description"], "") self.assertEqual(zerver_stream[2]["realm"], 3) self.assertTrue(stream_id_mapper.has("gryffindor-common-room")) self.assertTrue(stream_id_mapper.has("gryffindor-quidditch-team")) self.assertTrue(stream_id_mapper.has("dumbledores-army")) # TODO: Add ginny self.assertEqual( subscriber_handler.get_users( stream_id_mapper.get("gryffindor-common-room")), {1, 2}) self.assertEqual( subscriber_handler.get_users( stream_id_mapper.get("gryffindor-quidditch-team")), {1, 2}) self.assertEqual( subscriber_handler.get_users( stream_id_mapper.get("dumbledores-army")), {1, 2}) team_name = "slytherin" zerver_stream = convert_channel_data( channel_data=self.mattermost_data["channel"], user_data_map=self.username_to_user, subscriber_handler=subscriber_handler, stream_id_mapper=stream_id_mapper, user_id_mapper=user_id_mapper, realm_id=4, team_name=team_name, ) self.assertEqual( subscriber_handler.get_users( stream_id_mapper.get("slytherin-common-room")), {3, 4, 5}) self.assertEqual( subscriber_handler.get_users( stream_id_mapper.get("slytherin-quidditch-team")), {3, 4})
def test_process_users(self) -> None: fixture_dir_name = self.fixture_file_name("", "rocketchat_fixtures") rocketchat_data = rocketchat_data_to_dict(fixture_dir_name) user_id_to_user_map = map_user_id_to_user(rocketchat_data["user"]) realm_id = 3 domain_name = "zulip.com" user_handler = UserHandler() user_id_mapper = IdMapper() process_users( user_id_to_user_map=user_id_to_user_map, realm_id=realm_id, domain_name=domain_name, user_handler=user_handler, user_id_mapper=user_id_mapper, ) self.assert_length(user_handler.get_all_users(), 6) self.assertTrue(user_id_mapper.has(rocketchat_data["user"][0]["_id"])) self.assertTrue(user_id_mapper.has(rocketchat_data["user"][4]["_id"])) user_id = user_id_mapper.get(rocketchat_data["user"][0]["_id"]) user = user_handler.get_user(user_id) self.assertEqual(user["full_name"], rocketchat_data["user"][0]["name"]) self.assertEqual(user["avatar_source"], "G") self.assertEqual(user["delivery_email"], "*****@*****.**") self.assertEqual(user["email"], "*****@*****.**") self.assertEqual(user["full_name"], "Rocket.Cat") self.assertEqual(user["id"], 1) self.assertEqual(user["is_active"], False) self.assertEqual(user["is_mirror_dummy"], False) self.assertEqual(user["is_bot"], True) self.assertEqual(user["bot_type"], 1) self.assertEqual(user["bot_owner"], 2) self.assertEqual(user["role"], UserProfile.ROLE_MEMBER) self.assertEqual(user["realm"], realm_id) self.assertEqual(user["short_name"], "rocket.cat") self.assertEqual(user["timezone"], "UTC") user_id = user_id_mapper.get(rocketchat_data["user"][2]["_id"]) user = user_handler.get_user(user_id) self.assertEqual(user["full_name"], rocketchat_data["user"][2]["name"]) self.assertEqual(user["avatar_source"], "G") self.assertEqual(user["delivery_email"], "*****@*****.**") self.assertEqual(user["email"], "*****@*****.**") self.assertEqual(user["full_name"], "Harry Potter") self.assertEqual(user["id"], 3) self.assertEqual(user["is_active"], True) self.assertEqual(user["is_mirror_dummy"], False) self.assertEqual(user["is_bot"], False) self.assertEqual(user["bot_type"], None) self.assertEqual(user["bot_owner"], None) self.assertEqual(user["role"], UserProfile.ROLE_REALM_OWNER) self.assertEqual(user["realm"], realm_id) self.assertEqual(user["short_name"], "harry.potter") self.assertEqual(user["timezone"], "UTC") # Test `is_mirror_dummy` set for users of type `unknown` rocketchat_data["user"].append({ "_id": "s0m34ndmID", "createdAt": datetime.datetime(2019, 11, 6, 0, 38, 42, 796000), "type": "unknown", "roles": ["unknown"], "name": "Unknown user", "username": "******", }) user_id_to_user_map = map_user_id_to_user(rocketchat_data["user"]) process_users( user_id_to_user_map=user_id_to_user_map, realm_id=realm_id, domain_name=domain_name, user_handler=user_handler, user_id_mapper=user_id_mapper, ) self.assert_length(user_handler.get_all_users(), 7) self.assertTrue(user_id_mapper.has(rocketchat_data["user"][6]["_id"])) user_id = user_id_mapper.get(rocketchat_data["user"][6]["_id"]) user = user_handler.get_user(user_id) self.assertEqual(user["id"], 7) self.assertEqual(user["is_active"], False) self.assertEqual(user["is_mirror_dummy"], True) self.assertEqual(user["is_bot"], False)
def do_convert_data(mattermost_data_dir: str, output_dir: str, masking_content: bool) -> None: username_to_user: Dict[str, Dict[str, Any]] = {} os.makedirs(output_dir, exist_ok=True) if os.listdir(output_dir): # nocoverage raise Exception("Output directory should be empty!") mattermost_data_file = os.path.join(mattermost_data_dir, "export.json") mattermost_data = mattermost_data_file_to_dict(mattermost_data_file) username_to_user = create_username_to_user_mapping(mattermost_data["user"]) for team in mattermost_data["team"]: realm_id = NEXT_ID("realm_id") team_name = team["name"] user_handler = UserHandler() subscriber_handler = SubscriberHandler() user_id_mapper = IdMapper() stream_id_mapper = IdMapper() huddle_id_mapper = IdMapper() print("Generating data for", team_name) realm = make_realm(realm_id, team) realm_output_dir = os.path.join(output_dir, team_name) reset_mirror_dummy_users(username_to_user) label_mirror_dummy_users( len(mattermost_data["team"]), team_name, mattermost_data, username_to_user ) convert_user_data( user_handler=user_handler, user_id_mapper=user_id_mapper, user_data_map=username_to_user, realm_id=realm_id, team_name=team_name, ) zerver_stream = convert_channel_data( channel_data=mattermost_data["channel"], user_data_map=username_to_user, subscriber_handler=subscriber_handler, stream_id_mapper=stream_id_mapper, user_id_mapper=user_id_mapper, realm_id=realm_id, team_name=team_name, ) realm["zerver_stream"] = zerver_stream zerver_huddle: List[ZerverFieldsT] = [] if len(mattermost_data["team"]) == 1: zerver_huddle = convert_huddle_data( huddle_data=mattermost_data["direct_channel"], user_data_map=username_to_user, subscriber_handler=subscriber_handler, huddle_id_mapper=huddle_id_mapper, user_id_mapper=user_id_mapper, realm_id=realm_id, team_name=team_name, ) realm["zerver_huddle"] = zerver_huddle all_users = user_handler.get_all_users() zerver_recipient = build_recipients( zerver_userprofile=all_users, zerver_stream=zerver_stream, zerver_huddle=zerver_huddle, ) realm["zerver_recipient"] = zerver_recipient stream_subscriptions = build_stream_subscriptions( get_users=subscriber_handler.get_users, zerver_recipient=zerver_recipient, zerver_stream=zerver_stream, ) huddle_subscriptions = build_huddle_subscriptions( get_users=subscriber_handler.get_users, zerver_recipient=zerver_recipient, zerver_huddle=zerver_huddle, ) personal_subscriptions = build_personal_subscriptions( zerver_recipient=zerver_recipient, ) # Mattermost currently supports only exporting messages from channels. # Personal messages and huddles are not exported. zerver_subscription = personal_subscriptions + stream_subscriptions + huddle_subscriptions realm["zerver_subscription"] = zerver_subscription zerver_realmemoji = write_emoticon_data( realm_id=realm_id, custom_emoji_data=mattermost_data["emoji"], data_dir=mattermost_data_dir, output_dir=realm_output_dir, ) realm["zerver_realmemoji"] = zerver_realmemoji subscriber_map = make_subscriber_map( zerver_subscription=zerver_subscription, ) total_reactions: List[Dict[str, Any]] = [] uploads_list: List[ZerverFieldsT] = [] zerver_attachment: List[ZerverFieldsT] = [] write_message_data( num_teams=len(mattermost_data["team"]), team_name=team_name, realm_id=realm_id, post_data=mattermost_data["post"], zerver_recipient=zerver_recipient, subscriber_map=subscriber_map, output_dir=realm_output_dir, masking_content=masking_content, stream_id_mapper=stream_id_mapper, huddle_id_mapper=huddle_id_mapper, user_id_mapper=user_id_mapper, user_handler=user_handler, zerver_realmemoji=zerver_realmemoji, total_reactions=total_reactions, uploads_list=uploads_list, zerver_attachment=zerver_attachment, mattermost_data_dir=mattermost_data_dir, ) realm["zerver_reaction"] = total_reactions realm["zerver_userprofile"] = user_handler.get_all_users() realm["sort_by_date"] = True create_converted_data_files(realm, realm_output_dir, "/realm.json") # Mattermost currently doesn't support exporting avatars create_converted_data_files([], realm_output_dir, "/avatars/records.json") # Export message attachments attachment: Dict[str, List[Any]] = {"zerver_attachment": zerver_attachment} create_converted_data_files(uploads_list, realm_output_dir, "/uploads/records.json") create_converted_data_files(attachment, realm_output_dir, "/attachment.json") logging.info("Start making tarball") subprocess.check_call(["tar", "-czf", realm_output_dir + ".tar.gz", realm_output_dir, "-P"]) logging.info("Done making tarball")
def test_map_receiver_id_to_recipient_id(self) -> None: fixture_dir_name = self.fixture_file_name("", "rocketchat_fixtures") rocketchat_data = rocketchat_data_to_dict(fixture_dir_name) realm_id = 3 domain_name = "zulip.com" user_handler = UserHandler() subscriber_handler = SubscriberHandler() user_id_mapper = IdMapper() stream_id_mapper = IdMapper() huddle_id_mapper = IdMapper() user_id_to_user_map = map_user_id_to_user(rocketchat_data["user"]) process_users( user_id_to_user_map=user_id_to_user_map, realm_id=realm_id, domain_name=domain_name, user_handler=user_handler, user_id_mapper=user_id_mapper, ) room_id_to_room_map: Dict[str, Dict[str, Any]] = {} team_id_to_team_map: Dict[str, Dict[str, Any]] = {} dsc_id_to_dsc_map: Dict[str, Dict[str, Any]] = {} direct_id_to_direct_map: Dict[str, Dict[str, Any]] = {} huddle_id_to_huddle_map: Dict[str, Dict[str, Any]] = {} categorize_channels_and_map_with_id( channel_data=rocketchat_data["room"], room_id_to_room_map=room_id_to_room_map, team_id_to_team_map=team_id_to_team_map, dsc_id_to_dsc_map=dsc_id_to_dsc_map, direct_id_to_direct_map=direct_id_to_direct_map, huddle_id_to_huddle_map=huddle_id_to_huddle_map, ) zerver_stream = convert_channel_data( room_id_to_room_map=room_id_to_room_map, team_id_to_team_map=team_id_to_team_map, stream_id_mapper=stream_id_mapper, realm_id=realm_id, ) zerver_huddle = convert_huddle_data( huddle_id_to_huddle_map=huddle_id_to_huddle_map, huddle_id_mapper=huddle_id_mapper, user_id_mapper=user_id_mapper, subscriber_handler=subscriber_handler, ) all_users = user_handler.get_all_users() zerver_recipient = build_recipients( zerver_userprofile=all_users, zerver_stream=zerver_stream, zerver_huddle=zerver_huddle, ) stream_id_to_recipient_id: Dict[int, int] = {} user_id_to_recipient_id: Dict[int, int] = {} huddle_id_to_recipient_id: Dict[int, int] = {} map_receiver_id_to_recipient_id( zerver_recipient=zerver_recipient, stream_id_to_recipient_id=stream_id_to_recipient_id, user_id_to_recipient_id=user_id_to_recipient_id, huddle_id_to_recipient_id=huddle_id_to_recipient_id, ) # 6 for streams and 6 for users. self.assert_length(zerver_recipient, 13) self.assert_length(stream_id_to_recipient_id, 6) self.assert_length(user_id_to_recipient_id, 6) self.assert_length(huddle_id_to_recipient_id, 1) # First user recipients are built, followed by stream recipients in `build_recipients`. self.assertEqual( user_id_to_recipient_id[zerver_recipient[0]["type_id"]], zerver_recipient[0]["id"]) self.assertEqual( user_id_to_recipient_id[zerver_recipient[1]["type_id"]], zerver_recipient[1]["id"]) self.assertEqual( stream_id_to_recipient_id[zerver_recipient[6]["type_id"]], zerver_recipient[6]["id"]) self.assertEqual( stream_id_to_recipient_id[zerver_recipient[7]["type_id"]], zerver_recipient[7]["id"]) self.assertEqual( huddle_id_to_recipient_id[zerver_recipient[12]["type_id"]], zerver_recipient[12]["id"])
def test_convert_stream_subscription_data(self) -> None: fixture_dir_name = self.fixture_file_name("", "rocketchat_fixtures") rocketchat_data = rocketchat_data_to_dict(fixture_dir_name) realm_id = 3 domain_name = "zulip.com" user_handler = UserHandler() subscriber_handler = SubscriberHandler() user_id_mapper = IdMapper() stream_id_mapper = IdMapper() user_id_to_user_map = map_user_id_to_user(rocketchat_data["user"]) process_users( user_id_to_user_map=user_id_to_user_map, realm_id=realm_id, domain_name=domain_name, user_handler=user_handler, user_id_mapper=user_id_mapper, ) room_id_to_room_map: Dict[str, Dict[str, Any]] = {} team_id_to_team_map: Dict[str, Dict[str, Any]] = {} dsc_id_to_dsc_map: Dict[str, Dict[str, Any]] = {} direct_id_to_direct_map: Dict[str, Dict[str, Any]] = {} huddle_id_to_huddle_map: Dict[str, Dict[str, Any]] = {} categorize_channels_and_map_with_id( channel_data=rocketchat_data["room"], room_id_to_room_map=room_id_to_room_map, team_id_to_team_map=team_id_to_team_map, dsc_id_to_dsc_map=dsc_id_to_dsc_map, direct_id_to_direct_map=direct_id_to_direct_map, huddle_id_to_huddle_map=huddle_id_to_huddle_map, ) zerver_stream = convert_channel_data( room_id_to_room_map=room_id_to_room_map, team_id_to_team_map=team_id_to_team_map, stream_id_mapper=stream_id_mapper, realm_id=realm_id, ) convert_stream_subscription_data( user_id_to_user_map=user_id_to_user_map, dsc_id_to_dsc_map=dsc_id_to_dsc_map, zerver_stream=zerver_stream, stream_id_mapper=stream_id_mapper, user_id_mapper=user_id_mapper, subscriber_handler=subscriber_handler, ) priyansh_id = user_id_mapper.get(rocketchat_data["user"][1]["_id"]) harry_id = user_id_mapper.get(rocketchat_data["user"][2]["_id"]) hermione_id = user_id_mapper.get(rocketchat_data["user"][3]["_id"]) ron_id = user_id_mapper.get(rocketchat_data["user"][4]["_id"]) voldemort_id = user_id_mapper.get(rocketchat_data["user"][5]["_id"]) self.assertEqual( subscriber_handler.get_users(stream_id=zerver_stream[0]["id"]), {priyansh_id, harry_id, ron_id, hermione_id, voldemort_id}, ) self.assertEqual( subscriber_handler.get_users(stream_id=zerver_stream[1]["id"]), {priyansh_id, harry_id}) self.assertEqual( subscriber_handler.get_users(stream_id=zerver_stream[2]["id"]), {harry_id, hermione_id}) self.assertEqual( subscriber_handler.get_users(stream_id=zerver_stream[3]["id"]), {harry_id, ron_id, hermione_id}, ) self.assertEqual( subscriber_handler.get_users(stream_id=zerver_stream[4]["id"]), {harry_id}) self.assertEqual( subscriber_handler.get_users(stream_id=zerver_stream[5]["id"]), {harry_id}) # Add a new channel with no user. no_user_channel: Dict[str, Any] = { "_id": "rand0mID", "ts": datetime.datetime(2021, 7, 15, 10, 58, 23, 647000), "t": "c", "name": "no-user-channel", } room_id_to_room_map[no_user_channel["_id"]] = no_user_channel zerver_stream = convert_channel_data( room_id_to_room_map=room_id_to_room_map, team_id_to_team_map=team_id_to_team_map, stream_id_mapper=stream_id_mapper, realm_id=realm_id, ) convert_stream_subscription_data( user_id_to_user_map=user_id_to_user_map, dsc_id_to_dsc_map=dsc_id_to_dsc_map, zerver_stream=zerver_stream, stream_id_mapper=stream_id_mapper, user_id_mapper=user_id_mapper, subscriber_handler=subscriber_handler, ) self.assert_length( subscriber_handler.get_users(stream_id=zerver_stream[6]["id"]), 0) self.assertTrue(zerver_stream[6]["deactivated"])
def test_convert_channel_data(self) -> None: fixture_dir_name = self.fixture_file_name("", "rocketchat_fixtures") rocketchat_data = rocketchat_data_to_dict(fixture_dir_name) realm_id = 3 stream_id_mapper = IdMapper() room_id_to_room_map: Dict[str, Dict[str, Any]] = {} team_id_to_team_map: Dict[str, Dict[str, Any]] = {} dsc_id_to_dsc_map: Dict[str, Dict[str, Any]] = {} direct_id_to_direct_map: Dict[str, Dict[str, Any]] = {} huddle_id_to_huddle_map: Dict[str, Dict[str, Any]] = {} categorize_channels_and_map_with_id( channel_data=rocketchat_data["room"], room_id_to_room_map=room_id_to_room_map, team_id_to_team_map=team_id_to_team_map, dsc_id_to_dsc_map=dsc_id_to_dsc_map, direct_id_to_direct_map=direct_id_to_direct_map, huddle_id_to_huddle_map=huddle_id_to_huddle_map, ) zerver_stream = convert_channel_data( room_id_to_room_map=room_id_to_room_map, team_id_to_team_map=team_id_to_team_map, stream_id_mapper=stream_id_mapper, realm_id=realm_id, ) # Only rooms are converted to streams. self.assert_length(room_id_to_room_map, 6) self.assert_length(zerver_stream, 6) # Normal public stream self.assertEqual(zerver_stream[0]["name"], "general") self.assertEqual(zerver_stream[0]["invite_only"], False) self.assertEqual(zerver_stream[0]["description"], "This is a general channel.") self.assertEqual(zerver_stream[0]["rendered_description"], "") self.assertEqual(zerver_stream[0]["stream_post_policy"], 1) self.assertEqual(zerver_stream[0]["realm"], realm_id) # Private stream self.assertEqual(zerver_stream[1]["name"], "random") self.assertEqual(zerver_stream[1]["invite_only"], True) self.assertEqual(zerver_stream[1]["description"], "") self.assertEqual(zerver_stream[1]["rendered_description"], "") self.assertEqual(zerver_stream[1]["stream_post_policy"], 1) self.assertEqual(zerver_stream[1]["realm"], realm_id) # Team main self.assertEqual(zerver_stream[3]["name"], "[TEAM] team-harry-potter") self.assertEqual(zerver_stream[3]["invite_only"], True) self.assertEqual(zerver_stream[3]["description"], "Welcome to the official Harry Potter team.") self.assertEqual(zerver_stream[3]["rendered_description"], "") self.assertEqual(zerver_stream[3]["stream_post_policy"], 1) self.assertEqual(zerver_stream[3]["realm"], realm_id) # Team channel self.assertEqual(zerver_stream[5]["name"], "thp-channel-2") self.assertEqual(zerver_stream[5]["invite_only"], False) self.assertEqual(zerver_stream[5]["description"], "[Team team-harry-potter channel]. ") self.assertEqual(zerver_stream[5]["rendered_description"], "") self.assertEqual(zerver_stream[5]["stream_post_policy"], 1) self.assertEqual(zerver_stream[5]["realm"], realm_id)
def do_convert_data(rocketchat_data_dir: str, output_dir: str) -> None: # Get all required exported data in a dictionary rocketchat_data = rocketchat_data_to_dict(rocketchat_data_dir) # Subdomain is set by the user while running the import command realm_subdomain = "" realm_id = 0 domain_name = settings.EXTERNAL_HOST realm = make_realm(realm_id, realm_subdomain, domain_name, rocketchat_data["instance"][0]) user_id_to_user_map: Dict[str, Dict[str, Any]] = map_user_id_to_user( rocketchat_data["user"]) username_to_user_id_map: Dict[str, str] = map_username_to_user_id( user_id_to_user_map) user_handler = UserHandler() subscriber_handler = SubscriberHandler() user_id_mapper = IdMapper() stream_id_mapper = IdMapper() huddle_id_mapper = IdMapper() process_users( user_id_to_user_map=user_id_to_user_map, realm_id=realm_id, domain_name=domain_name, user_handler=user_handler, user_id_mapper=user_id_mapper, ) room_id_to_room_map: Dict[str, Dict[str, Any]] = {} team_id_to_team_map: Dict[str, Dict[str, Any]] = {} dsc_id_to_dsc_map: Dict[str, Dict[str, Any]] = {} direct_id_to_direct_map: Dict[str, Dict[str, Any]] = {} huddle_id_to_huddle_map: Dict[str, Dict[str, Any]] = {} categorize_channels_and_map_with_id( channel_data=rocketchat_data["room"], room_id_to_room_map=room_id_to_room_map, team_id_to_team_map=team_id_to_team_map, dsc_id_to_dsc_map=dsc_id_to_dsc_map, direct_id_to_direct_map=direct_id_to_direct_map, huddle_id_to_huddle_map=huddle_id_to_huddle_map, ) zerver_stream = convert_channel_data( room_id_to_room_map=room_id_to_room_map, team_id_to_team_map=team_id_to_team_map, stream_id_mapper=stream_id_mapper, realm_id=realm_id, ) realm["zerver_stream"] = zerver_stream # Add stream subscription data to `subscriber_handler` convert_stream_subscription_data( user_id_to_user_map=user_id_to_user_map, dsc_id_to_dsc_map=dsc_id_to_dsc_map, zerver_stream=zerver_stream, stream_id_mapper=stream_id_mapper, user_id_mapper=user_id_mapper, subscriber_handler=subscriber_handler, ) zerver_huddle = convert_huddle_data( huddle_id_to_huddle_map=huddle_id_to_huddle_map, huddle_id_mapper=huddle_id_mapper, user_id_mapper=user_id_mapper, subscriber_handler=subscriber_handler, ) realm["zerver_huddle"] = zerver_huddle all_users = user_handler.get_all_users() zerver_recipient = build_recipients( zerver_userprofile=all_users, zerver_stream=zerver_stream, zerver_huddle=zerver_huddle, ) realm["zerver_recipient"] = zerver_recipient stream_subscriptions = build_stream_subscriptions( get_users=subscriber_handler.get_users, zerver_recipient=zerver_recipient, zerver_stream=zerver_stream, ) huddle_subscriptions = build_huddle_subscriptions( get_users=subscriber_handler.get_users, zerver_recipient=zerver_recipient, zerver_huddle=zerver_huddle, ) personal_subscriptions = build_personal_subscriptions( zerver_recipient=zerver_recipient, ) zerver_subscription = personal_subscriptions + stream_subscriptions + huddle_subscriptions realm["zerver_subscription"] = zerver_subscription zerver_realmemoji = build_custom_emoji( realm_id=realm_id, custom_emoji_data=rocketchat_data["custom_emoji"], output_dir=output_dir, ) realm["zerver_realmemoji"] = zerver_realmemoji subscriber_map = make_subscriber_map( zerver_subscription=zerver_subscription, ) stream_id_to_recipient_id: Dict[int, int] = {} huddle_id_to_recipient_id: Dict[int, int] = {} user_id_to_recipient_id: Dict[int, int] = {} map_receiver_id_to_recipient_id( zerver_recipient=zerver_recipient, stream_id_to_recipient_id=stream_id_to_recipient_id, huddle_id_to_recipient_id=huddle_id_to_recipient_id, user_id_to_recipient_id=user_id_to_recipient_id, ) channel_messages: List[Dict[str, Any]] = [] private_messages: List[Dict[str, Any]] = [] separate_channel_and_private_messages( messages=rocketchat_data["message"], direct_id_to_direct_map=direct_id_to_direct_map, huddle_id_to_huddle_map=huddle_id_to_huddle_map, channel_messages=channel_messages, private_messages=private_messages, ) total_reactions: List[ZerverFieldsT] = [] uploads_list: List[ZerverFieldsT] = [] zerver_attachment: List[ZerverFieldsT] = [] upload_id_to_upload_data_map = map_upload_id_to_upload_data( rocketchat_data["upload"]) # Process channel messages process_messages( realm_id=realm_id, messages=channel_messages, subscriber_map=subscriber_map, is_pm_data=False, username_to_user_id_map=username_to_user_id_map, user_id_mapper=user_id_mapper, user_handler=user_handler, user_id_to_recipient_id=user_id_to_recipient_id, stream_id_mapper=stream_id_mapper, stream_id_to_recipient_id=stream_id_to_recipient_id, huddle_id_mapper=huddle_id_mapper, huddle_id_to_recipient_id=huddle_id_to_recipient_id, room_id_to_room_map=room_id_to_room_map, dsc_id_to_dsc_map=dsc_id_to_dsc_map, direct_id_to_direct_map=direct_id_to_direct_map, huddle_id_to_huddle_map=huddle_id_to_huddle_map, zerver_realmemoji=zerver_realmemoji, total_reactions=total_reactions, uploads_list=uploads_list, zerver_attachment=zerver_attachment, upload_id_to_upload_data_map=upload_id_to_upload_data_map, output_dir=output_dir, ) # Process private messages process_messages( realm_id=realm_id, messages=private_messages, subscriber_map=subscriber_map, is_pm_data=True, username_to_user_id_map=username_to_user_id_map, user_id_mapper=user_id_mapper, user_handler=user_handler, user_id_to_recipient_id=user_id_to_recipient_id, stream_id_mapper=stream_id_mapper, stream_id_to_recipient_id=stream_id_to_recipient_id, huddle_id_mapper=huddle_id_mapper, huddle_id_to_recipient_id=huddle_id_to_recipient_id, room_id_to_room_map=room_id_to_room_map, dsc_id_to_dsc_map=dsc_id_to_dsc_map, direct_id_to_direct_map=direct_id_to_direct_map, huddle_id_to_huddle_map=huddle_id_to_huddle_map, zerver_realmemoji=zerver_realmemoji, total_reactions=total_reactions, uploads_list=uploads_list, zerver_attachment=zerver_attachment, upload_id_to_upload_data_map=upload_id_to_upload_data_map, output_dir=output_dir, ) realm["zerver_reaction"] = total_reactions realm["zerver_userprofile"] = user_handler.get_all_users() realm["sort_by_date"] = True create_converted_data_files(realm, output_dir, "/realm.json") # TODO: Add support for importing avatars create_converted_data_files([], output_dir, "/avatars/records.json") # Import attachments attachment: Dict[str, List[Any]] = {"zerver_attachment": zerver_attachment} create_converted_data_files(attachment, output_dir, "/attachment.json") create_converted_data_files(uploads_list, output_dir, "/uploads/records.json") logging.info("Start making tarball") subprocess.check_call( ["tar", "-czf", output_dir + ".tar.gz", output_dir, "-P"]) logging.info("Done making tarball")
def test_process_message_attachments(self) -> None: mattermost_data_dir = self.fixture_file_name( "", "mattermost_fixtures/direct_channel") output_dir = self.make_import_output_dir("mattermost") fixture_file_name = self.fixture_file_name( "export.json", "mattermost_fixtures/direct_channel") mattermost_data = mattermost_data_file_to_dict(fixture_file_name) username_to_user = create_username_to_user_mapping( mattermost_data["user"]) reset_mirror_dummy_users(username_to_user) user_handler = UserHandler() user_id_mapper = IdMapper() team_name = "gryffindor" convert_user_data( user_handler=user_handler, user_id_mapper=user_id_mapper, user_data_map=username_to_user, realm_id=3, team_name=team_name, ) zerver_attachments: List[ZerverFieldsT] = [] uploads_list: List[ZerverFieldsT] = [] process_message_attachments( attachments=mattermost_data["post"]["direct_post"][0] ["attachments"], realm_id=3, message_id=1, user_id=2, user_handler=user_handler, zerver_attachment=zerver_attachments, uploads_list=uploads_list, mattermost_data_dir=mattermost_data_dir, output_dir=output_dir, ) self.assert_length(zerver_attachments, 1) self.assertEqual(zerver_attachments[0]["file_name"], "harry-ron.jpg") self.assertEqual(zerver_attachments[0]["owner"], 2) self.assertEqual( user_handler.get_user(zerver_attachments[0]["owner"])["email"], "*****@*****.**") # TODO: Assert this for False after fixing the file permissions in PMs self.assertTrue(zerver_attachments[0]["is_realm_public"]) self.assert_length(uploads_list, 1) self.assertEqual(uploads_list[0]["user_profile_email"], "*****@*****.**") attachment_path = self.fixture_file_name( mattermost_data["post"]["direct_post"][0]["attachments"][0] ["path"], "mattermost_fixtures/direct_channel/data", ) attachment_out_path = os.path.join(output_dir, "uploads", zerver_attachments[0]["path_id"]) self.assertTrue(os.path.exists(attachment_out_path)) self.assertTrue(filecmp.cmp(attachment_path, attachment_out_path))