def test_convert_huddle_data(self) -> None:
fixture_dir_name = self.fixture_file_name("", "rocketchat_fixtures")
rocketchat_data = rocketchat_data_to_dict(fixture_dir_name)
realm_id = 3
domain_name = "zulip.com"
user_handler = UserHandler()
subscriber_handler = SubscriberHandler()
user_id_mapper = IdMapper()
huddle_id_mapper = IdMapper()
user_id_to_user_map = map_user_id_to_user(rocketchat_data["user"])
process_users(
user_id_to_user_map=user_id_to_user_map,
realm_id=realm_id,
domain_name=domain_name,
user_handler=user_handler,
user_id_mapper=user_id_mapper,
)
room_id_to_room_map: Dict[str, Dict[str, Any]] = {}
team_id_to_team_map: Dict[str, Dict[str, Any]] = {}
dsc_id_to_dsc_map: Dict[str, Dict[str, Any]] = {}
direct_id_to_direct_map: Dict[str, Dict[str, Any]] = {}
huddle_id_to_huddle_map: Dict[str, Dict[str, Any]] = {}
livechat_id_to_livechat_map: Dict[str, Dict[str, Any]] = {}
categorize_channels_and_map_with_id(
channel_data=rocketchat_data["room"],
room_id_to_room_map=room_id_to_room_map,
team_id_to_team_map=team_id_to_team_map,
dsc_id_to_dsc_map=dsc_id_to_dsc_map,
direct_id_to_direct_map=direct_id_to_direct_map,
huddle_id_to_huddle_map=huddle_id_to_huddle_map,
livechat_id_to_livechat_map=livechat_id_to_livechat_map,
)
zerver_huddle = convert_huddle_data(
huddle_id_to_huddle_map=huddle_id_to_huddle_map,
huddle_id_mapper=huddle_id_mapper,
user_id_mapper=user_id_mapper,
subscriber_handler=subscriber_handler,
)
self.assert_length(zerver_huddle, 1)
rc_huddle_id = rocketchat_data["room"][12]["_id"]
self.assertTrue(huddle_id_mapper.has(rc_huddle_id))
huddle_id = huddle_id_mapper.get(rc_huddle_id)
self.assertEqual(subscriber_handler.get_users(huddle_id=huddle_id),
{3, 4, 5})
def test_convert_user_data(self) -> None:
user_id_mapper = IdMapper()
realm_id = 3
team_name = "gryffindor"
user_handler = UserHandler()
convert_user_data(user_handler, user_id_mapper, self.username_to_user, realm_id, team_name)
self.assertTrue(user_id_mapper.has("harry"))
self.assertTrue(user_id_mapper.has("ron"))
self.assertEqual(user_handler.get_user(user_id_mapper.get("harry"))["full_name"], "Harry Potter")
self.assertEqual(user_handler.get_user(user_id_mapper.get("ron"))["full_name"], "Ron Weasley")
team_name = "slytherin"
user_handler = UserHandler()
convert_user_data(user_handler, user_id_mapper, self.username_to_user, realm_id, team_name)
self.assertEqual(len(user_handler.get_all_users()), 3)
self.assertTrue(user_id_mapper.has("malfoy"))
self.assertTrue(user_id_mapper.has("pansy"))
self.assertTrue(user_id_mapper.has("snape"))
team_name = "gryffindor"
# Snape is a mirror dummy user in Harry's team.
label_mirror_dummy_users(team_name, self.mattermost_data, self.username_to_user)
user_handler = UserHandler()
convert_user_data(user_handler, user_id_mapper, self.username_to_user, realm_id, team_name)
self.assertEqual(len(user_handler.get_all_users()), 3)
self.assertTrue(user_id_mapper.has("snape"))
team_name = "slytherin"
user_handler = UserHandler()
convert_user_data(user_handler, user_id_mapper, self.username_to_user, realm_id, team_name)
self.assertEqual(len(user_handler.get_all_users()), 3)
def test_build_reactions(self) -> None:
total_reactions = [] # type: List[Dict[str, Any]]
reactions = [{
"user": "******",
"create_at": 1553165521410,
"emoji_name": "tick"
}, {
"user": "******",
"create_at": 1553166530805,
"emoji_name": "smile"
}, {
"user": "******",
"create_at": 1553166540953,
"emoji_name": "world_map"
}, {
"user": "******",
"create_at": 1553166540957,
"emoji_name": "world_map"
}]
zerver_realmemoji = write_emoticon_data(
realm_id=3,
custom_emoji_data=self.mattermost_data["emoji"],
data_dir=self.fixture_file_name("", "mattermost_fixtures"),
output_dir=self.make_import_output_dir("mattermost"))
# Make sure tick is present in fixture data
self.assertEqual(zerver_realmemoji[1]["name"], "tick")
tick_emoji_code = zerver_realmemoji[1]["id"]
name_to_codepoint = get_name_to_codepoint_dict()
user_id_mapper = IdMapper()
harry_id = user_id_mapper.get("harry")
ron_id = user_id_mapper.get("ron")
build_reactions(realm_id=3,
total_reactions=total_reactions,
reactions=reactions,
message_id=5,
name_to_codepoint=name_to_codepoint,
user_id_mapper=user_id_mapper,
zerver_realmemoji=zerver_realmemoji)
smile_emoji_code = name_to_codepoint["smile"]
world_map_emoji_code = name_to_codepoint["world_map"]
self.assertEqual(len(total_reactions), 4)
self.assertEqual(self.get_set(total_reactions, "reaction_type"),
set([Reaction.REALM_EMOJI, Reaction.UNICODE_EMOJI]))
self.assertEqual(self.get_set(total_reactions, "emoji_name"),
set(["tick", "smile", "world_map"]))
self.assertEqual(
self.get_set(total_reactions, "emoji_code"),
set([tick_emoji_code, smile_emoji_code, world_map_emoji_code]))
self.assertEqual(self.get_set(total_reactions, "user_profile"),
set([harry_id, ron_id]))
self.assertEqual(len(self.get_set(total_reactions, "id")), 4)
self.assertEqual(len(self.get_set(total_reactions, "message")), 1)
def test_build_reactions(self) -> None:
total_reactions = [] # type: List[Dict[str, Any]]
reactions = [
{"user": "******", "create_at": 1553165521410, "emoji_name": "tick"},
{"user": "******", "create_at": 1553166530805, "emoji_name": "smile"},
{"user": "******", "create_at": 1553166540953, "emoji_name": "world_map"},
{"user": "******", "create_at": 1553166540957, "emoji_name": "world_map"}
]
zerver_realmemoji = write_emoticon_data(
realm_id=3,
custom_emoji_data=self.mattermost_data["emoji"],
data_dir=self.fixture_file_name("", "mattermost_fixtures"),
output_dir=self.make_import_output_dir("mattermost")
)
# Make sure tick is present in fixture data
self.assertEqual(zerver_realmemoji[1]["name"], "tick")
tick_emoji_code = zerver_realmemoji[1]["id"]
name_to_codepoint = get_name_to_codepoint_dict()
user_id_mapper = IdMapper()
harry_id = user_id_mapper.get("harry")
ron_id = user_id_mapper.get("ron")
build_reactions(
realm_id=3,
total_reactions=total_reactions,
reactions=reactions,
message_id=5,
name_to_codepoint=name_to_codepoint,
user_id_mapper=user_id_mapper,
zerver_realmemoji=zerver_realmemoji
)
smile_emoji_code = name_to_codepoint["smile"]
world_map_emoji_code = name_to_codepoint["world_map"]
expected_total_reactions = [
{
'user_profile': harry_id, 'message': 5, 'id': 1, 'reaction_type': Reaction.REALM_EMOJI,
'emoji_code': tick_emoji_code, 'emoji_name': 'tick'
},
{
'user_profile': ron_id, 'message': 5, 'id': 2, 'reaction_type': Reaction.UNICODE_EMOJI,
'emoji_code': smile_emoji_code, 'emoji_name': 'smile'
},
{
'user_profile': ron_id, 'message': 5, 'id': 3, 'reaction_type': Reaction.UNICODE_EMOJI,
'emoji_code': world_map_emoji_code, 'emoji_name': 'world_map'
},
{
'user_profile': harry_id, 'message': 5, 'id': 4, 'reaction_type': Reaction.UNICODE_EMOJI,
'emoji_code': world_map_emoji_code, 'emoji_name': 'world_map'
}
]
self.assertEqual(total_reactions, expected_total_reactions)
def test_process_message_attachment(self) -> None:
fixture_dir_name = self.fixture_file_name("", "rocketchat_fixtures")
rocketchat_data = rocketchat_data_to_dict(fixture_dir_name)
output_dir = self.make_import_output_dir("mattermost")
user_id_to_user_map = map_user_id_to_user(rocketchat_data["user"])
realm_id = 3
domain_name = "zulip.com"
user_handler = UserHandler()
user_id_mapper = IdMapper()
process_users(
user_id_to_user_map=user_id_to_user_map,
realm_id=realm_id,
domain_name=domain_name,
user_handler=user_handler,
user_id_mapper=user_id_mapper,
)
zerver_attachments: List[ZerverFieldsT] = []
uploads_list: List[ZerverFieldsT] = []
upload_id_to_upload_data_map = map_upload_id_to_upload_data(
rocketchat_data["upload"])
message_with_attachment = rocketchat_data["message"][55]
process_message_attachment(
upload=message_with_attachment["file"],
realm_id=3,
message_id=1,
user_id=3,
user_handler=user_handler,
zerver_attachment=zerver_attachments,
uploads_list=uploads_list,
upload_id_to_upload_data_map=upload_id_to_upload_data_map,
output_dir=output_dir,
)
self.assert_length(zerver_attachments, 1)
self.assertEqual(zerver_attachments[0]["file_name"], "harry-ron.jpg")
self.assertEqual(zerver_attachments[0]["owner"], 3)
self.assertEqual(
user_handler.get_user(zerver_attachments[0]["owner"])["email"],
"*****@*****.**")
# TODO: Assert this for False after fixing the file permissions in PMs
self.assertTrue(zerver_attachments[0]["is_realm_public"])
self.assert_length(uploads_list, 1)
self.assertEqual(uploads_list[0]["user_profile_email"],
"*****@*****.**")
attachment_out_path = os.path.join(output_dir, "uploads",
zerver_attachments[0]["path_id"])
self.assertTrue(os.path.exists(attachment_out_path))
self.assertTrue(os.path.isfile(attachment_out_path))
def test_sender_ids(self) -> None:
realm_id = 5
user_handler = UserHandler()
user_id_mapper = IdMapper()
user_id_mapper.has = lambda key: True # type: ignore[assignment] # it's just a stub
# Simulate a "normal" user first.
user_with_id = dict(id=1,
# other fields don't matter here
)
user_handler.add_user(user=user_with_id)
normal_message: Dict[str, Any] = dict(sender=dict(id=1, ), )
sender_id = get_hipchat_sender_id(
realm_id=realm_id,
slim_mode=False,
message_dict=normal_message,
user_id_mapper=user_id_mapper,
user_handler=user_handler,
)
self.assertEqual(sender_id, 1)
bot_message = dict(sender='fred_bot', )
# Every message from fred_bot should
# return the same sender_id.
fred_bot_sender_id = 2
for i in range(3):
sender_id = get_hipchat_sender_id(
realm_id=realm_id,
slim_mode=False,
message_dict=bot_message,
user_id_mapper=user_id_mapper,
user_handler=user_handler,
)
self.assertEqual(sender_id, fred_bot_sender_id)
id_zero_message = dict(sender=dict(
id=0,
name='hal_bot',
), )
hal_bot_sender_id = 3
for i in range(3):
sender_id = get_hipchat_sender_id(
realm_id=realm_id,
slim_mode=False,
message_dict=id_zero_message,
user_id_mapper=user_id_mapper,
user_handler=user_handler,
)
self.assertEqual(sender_id, hal_bot_sender_id)
def test_convert_huddle_data(self) -> None:
fixture_file_name = self.fixture_file_name(
"export.json", "mattermost_fixtures/direct_channel")
mattermost_data = mattermost_data_file_to_dict(fixture_file_name)
username_to_user = create_username_to_user_mapping(
mattermost_data["user"])
reset_mirror_dummy_users(username_to_user)
user_handler = UserHandler()
subscriber_handler = SubscriberHandler()
huddle_id_mapper = IdMapper()
user_id_mapper = IdMapper()
team_name = "gryffindor"
convert_user_data(
user_handler=user_handler,
user_id_mapper=user_id_mapper,
user_data_map=username_to_user,
realm_id=3,
team_name=team_name,
)
zerver_huddle = convert_huddle_data(
huddle_data=mattermost_data["direct_channel"],
user_data_map=username_to_user,
subscriber_handler=subscriber_handler,
huddle_id_mapper=huddle_id_mapper,
user_id_mapper=user_id_mapper,
realm_id=3,
team_name=team_name,
)
self.assert_length(zerver_huddle, 1)
huddle_members = mattermost_data["direct_channel"][1]["members"]
huddle_name = generate_huddle_name(huddle_members)
self.assertTrue(huddle_id_mapper.has(huddle_name))
self.assertEqual(
subscriber_handler.get_users(
huddle_id=huddle_id_mapper.get(huddle_name)), {1, 2, 3})
def test_get_mentioned_user_ids(self) -> None:
user_id_mapper = IdMapper()
harry_id = user_id_mapper.get("harry")
raw_message = {"content": "Hello @harry"}
ids = get_mentioned_user_ids(raw_message, user_id_mapper)
self.assertEqual(list(ids), [harry_id])
raw_message = {"content": "Hello"}
ids = get_mentioned_user_ids(raw_message, user_id_mapper)
self.assertEqual(list(ids), [])
raw_message = {"content": "@harry How are you?"}
ids = get_mentioned_user_ids(raw_message, user_id_mapper)
self.assertEqual(list(ids), [harry_id])
raw_message = {"content": "@harry @ron Where are you folks?"}
ron_id = user_id_mapper.get("ron")
ids = get_mentioned_user_ids(raw_message, user_id_mapper)
self.assertEqual(list(ids), [harry_id, ron_id])
raw_message = {"content": "@harry.com How are you?"}
ids = get_mentioned_user_ids(raw_message, user_id_mapper)
self.assertEqual(list(ids), [])
raw_message = {"content": "[email protected] How are you?"}
ids = get_mentioned_user_ids(raw_message, user_id_mapper)
self.assertEqual(list(ids), [])
harry_id = user_id_mapper.get("harry_")
raw_message = {"content": "Hello @harry_"}
ids = get_mentioned_user_ids(raw_message, user_id_mapper)
self.assertEqual(list(ids), [harry_id])
harry_id = user_id_mapper.get("harry.")
raw_message = {"content": "Hello @harry."}
ids = get_mentioned_user_ids(raw_message, user_id_mapper)
self.assertEqual(list(ids), [harry_id])
harry_id = user_id_mapper.get("ha_rry.")
raw_message = {"content": "Hello @ha_rry."}
ids = get_mentioned_user_ids(raw_message, user_id_mapper)
self.assertEqual(list(ids), [harry_id])
ron_id = user_id_mapper.get("ron")
raw_message = {"content": "Hello @ron."}
ids = get_mentioned_user_ids(raw_message, user_id_mapper)
self.assertEqual(list(ids), [])
raw_message = {"content": "Hello @ron_"}
ids = get_mentioned_user_ids(raw_message, user_id_mapper)
self.assertEqual(list(ids), [])
def test_build_reactions(self) -> None:
fixture_file_name = self.fixture_file_name("export.json", "mattermost_fixtures")
mattermost_data = mattermost_data_file_to_dict(fixture_file_name)
total_reactions: List[Dict[str, Any]] = []
reactions = [
{"user": "******", "create_at": 1553165521410, "emoji_name": "tick"},
{"user": "******", "create_at": 1553166530805, "emoji_name": "smile"},
{"user": "******", "create_at": 1553166540953, "emoji_name": "world_map"},
{"user": "******", "create_at": 1553166540957, "emoji_name": "world_map"},
]
with self.assertLogs(level="INFO"):
zerver_realmemoji = write_emoticon_data(
realm_id=3,
custom_emoji_data=mattermost_data["emoji"],
data_dir=self.fixture_file_name("", "mattermost_fixtures"),
output_dir=self.make_import_output_dir("mattermost"),
)
# Make sure tick is present in fixture data
self.assertEqual(zerver_realmemoji[1]["name"], "tick")
tick_emoji_code = zerver_realmemoji[1]["id"]
user_id_mapper = IdMapper()
harry_id = user_id_mapper.get("harry")
ron_id = user_id_mapper.get("ron")
build_reactions(
realm_id=3,
total_reactions=total_reactions,
reactions=reactions,
message_id=5,
user_id_mapper=user_id_mapper,
zerver_realmemoji=zerver_realmemoji,
)
smile_emoji_code = name_to_codepoint["smile"]
world_map_emoji_code = name_to_codepoint["world_map"]
self.assertEqual(len(total_reactions), 4)
self.assertEqual(self.get_set(total_reactions, "reaction_type"), {Reaction.REALM_EMOJI, Reaction.UNICODE_EMOJI})
self.assertEqual(self.get_set(total_reactions, "emoji_name"), {"tick", "smile", "world_map"})
self.assertEqual(self.get_set(total_reactions, "emoji_code"), {tick_emoji_code, smile_emoji_code,
world_map_emoji_code})
self.assertEqual(self.get_set(total_reactions, "user_profile"), {harry_id, ron_id})
self.assertEqual(len(self.get_set(total_reactions, "id")), 4)
self.assertEqual(len(self.get_set(total_reactions, "message")), 1)
def test_convert_user_data(self) -> None:
user_id_mapper = IdMapper()
realm_id = 3
fixture_file_name = self.fixture_file_name("export.json",
"mattermost_fixtures")
mattermost_data = mattermost_data_file_to_dict(fixture_file_name)
username_to_user = create_username_to_user_mapping(
mattermost_data["user"])
reset_mirror_dummy_users(username_to_user)
team_name = "gryffindor"
user_handler = UserHandler()
convert_user_data(user_handler, user_id_mapper, username_to_user,
realm_id, team_name)
self.assert_length(user_handler.get_all_users(), 2)
self.assertTrue(user_id_mapper.has("harry"))
self.assertTrue(user_id_mapper.has("ron"))
self.assertEqual(
user_handler.get_user(user_id_mapper.get("harry"))["full_name"],
"Harry Potter")
self.assertEqual(
user_handler.get_user(user_id_mapper.get("ron"))["full_name"],
"Ron Weasley")
team_name = "slytherin"
user_handler = UserHandler()
convert_user_data(user_handler, user_id_mapper, username_to_user,
realm_id, team_name)
self.assert_length(user_handler.get_all_users(), 3)
self.assertTrue(user_id_mapper.has("malfoy"))
self.assertTrue(user_id_mapper.has("pansy"))
self.assertTrue(user_id_mapper.has("snape"))
team_name = "gryffindor"
# Snape is a mirror dummy user in Harry's team.
label_mirror_dummy_users(2, team_name, mattermost_data,
username_to_user)
user_handler = UserHandler()
convert_user_data(user_handler, user_id_mapper, username_to_user,
realm_id, team_name)
self.assert_length(user_handler.get_all_users(), 3)
self.assertTrue(user_id_mapper.has("snape"))
team_name = "slytherin"
user_handler = UserHandler()
convert_user_data(user_handler, user_id_mapper, username_to_user,
realm_id, team_name)
self.assert_length(user_handler.get_all_users(), 3)
def test_process_user(self) -> None:
user_id_mapper = IdMapper()
fixture_file_name = self.fixture_file_name("export.json",
"mattermost_fixtures")
mattermost_data = mattermost_data_file_to_dict(fixture_file_name)
username_to_user = create_username_to_user_mapping(
mattermost_data["user"])
reset_mirror_dummy_users(username_to_user)
harry_dict = username_to_user["harry"]
harry_dict["is_mirror_dummy"] = False
realm_id = 3
team_name = "gryffindor"
user = process_user(harry_dict, realm_id, team_name, user_id_mapper)
self.assertEqual(user["avatar_source"], "G")
self.assertEqual(user["delivery_email"], "*****@*****.**")
self.assertEqual(user["email"], "*****@*****.**")
self.assertEqual(user["full_name"], "Harry Potter")
self.assertEqual(user["id"], 1)
self.assertEqual(user["is_active"], True)
self.assertEqual(user["role"], UserProfile.ROLE_REALM_OWNER)
self.assertEqual(user["is_mirror_dummy"], False)
self.assertEqual(user["realm"], 3)
self.assertEqual(user["short_name"], "harry")
self.assertEqual(user["timezone"], "UTC")
# A user with a `null` team value shouldn't be an admin.
harry_dict["teams"] = None
user = process_user(harry_dict, realm_id, team_name, user_id_mapper)
self.assertEqual(user["role"], UserProfile.ROLE_MEMBER)
team_name = "slytherin"
snape_dict = username_to_user["snape"]
snape_dict["is_mirror_dummy"] = True
user = process_user(snape_dict, realm_id, team_name, user_id_mapper)
self.assertEqual(user["avatar_source"], "G")
self.assertEqual(user["delivery_email"], "*****@*****.**")
self.assertEqual(user["email"], "*****@*****.**")
self.assertEqual(user["full_name"], "Severus Snape")
self.assertEqual(user["id"], 2)
self.assertEqual(user["is_active"], False)
self.assertEqual(user["role"], UserProfile.ROLE_MEMBER)
self.assertEqual(user["is_mirror_dummy"], True)
self.assertEqual(user["realm"], 3)
self.assertEqual(user["short_name"], "snape")
self.assertEqual(user["timezone"], "UTC")
def test_process_user(self) -> None:
user_id_mapper = IdMapper()
harry_dict = self.username_to_user["harry"]
harry_dict["is_mirror_dummy"] = False
realm_id = 3
team_name = "gryffindor"
user = process_user(harry_dict, realm_id, team_name, user_id_mapper)
self.assertEqual(user["avatar_source"], 'G')
self.assertEqual(user["delivery_email"], "*****@*****.**")
self.assertEqual(user["email"], "*****@*****.**")
self.assertEqual(user["full_name"], "Harry Potter")
self.assertEqual(user["id"], 1)
self.assertEqual(user["is_active"], True)
self.assertEqual(user["is_realm_admin"], True)
self.assertEqual(user["is_guest"], False)
self.assertEqual(user["is_mirror_dummy"], False)
self.assertEqual(user["realm"], 3)
self.assertEqual(user["short_name"], "harry")
self.assertEqual(user["timezone"], "UTC")
# A user with a `null` team value shouldn't be an admin.
harry_dict["teams"] = None
user = process_user(harry_dict, realm_id, team_name, user_id_mapper)
self.assertEqual(user["is_realm_admin"], False)
team_name = "slytherin"
snape_dict = self.username_to_user["snape"]
snape_dict["is_mirror_dummy"] = True
user = process_user(snape_dict, realm_id, team_name, user_id_mapper)
self.assertEqual(user["avatar_source"], 'G')
self.assertEqual(user["delivery_email"], "*****@*****.**")
self.assertEqual(user["email"], "*****@*****.**")
self.assertEqual(user["full_name"], "Severus Snape")
self.assertEqual(user["id"], 2)
self.assertEqual(user["is_active"], False)
self.assertEqual(user["is_realm_admin"], False)
self.assertEqual(user["is_guest"], False)
self.assertEqual(user["is_mirror_dummy"], True)
self.assertEqual(user["realm"], 3)
self.assertEqual(user["short_name"], "snape")
self.assertEqual(user["timezone"], "UTC")
def test_convert_channel_data(self) -> None:
fixture_file_name = self.fixture_file_name("export.json", "mattermost_fixtures")
mattermost_data = mattermost_data_file_to_dict(fixture_file_name)
username_to_user = create_username_to_user_mapping(mattermost_data["user"])
reset_mirror_dummy_users(username_to_user)
user_handler = UserHandler()
subscriber_handler = SubscriberHandler()
stream_id_mapper = IdMapper()
user_id_mapper = IdMapper()
team_name = "gryffindor"
convert_user_data(
user_handler=user_handler,
user_id_mapper=user_id_mapper,
user_data_map=username_to_user,
realm_id=3,
team_name=team_name,
)
zerver_stream = convert_channel_data(
channel_data=mattermost_data["channel"],
user_data_map=username_to_user,
subscriber_handler=subscriber_handler,
stream_id_mapper=stream_id_mapper,
user_id_mapper=user_id_mapper,
realm_id=3,
team_name=team_name,
)
self.assertEqual(len(zerver_stream), 3)
self.assertEqual(zerver_stream[0]["name"], "Gryffindor common room")
self.assertEqual(zerver_stream[0]["invite_only"], False)
self.assertEqual(zerver_stream[0]["description"], "A place for talking about Gryffindor common room")
self.assertEqual(zerver_stream[0]["rendered_description"], "")
self.assertEqual(zerver_stream[0]["realm"], 3)
self.assertEqual(zerver_stream[1]["name"], "Gryffindor quidditch team")
self.assertEqual(zerver_stream[1]["invite_only"], False)
self.assertEqual(zerver_stream[1]["description"], "A place for talking about Gryffindor quidditch team")
self.assertEqual(zerver_stream[1]["rendered_description"], "")
self.assertEqual(zerver_stream[1]["realm"], 3)
self.assertEqual(zerver_stream[2]["name"], "Dumbledores army")
self.assertEqual(zerver_stream[2]["invite_only"], True)
self.assertEqual(zerver_stream[2]["description"], "A place for talking about Dumbledores army")
self.assertEqual(zerver_stream[2]["rendered_description"], "")
self.assertEqual(zerver_stream[2]["realm"], 3)
self.assertTrue(stream_id_mapper.has("gryffindor-common-room"))
self.assertTrue(stream_id_mapper.has("gryffindor-quidditch-team"))
self.assertTrue(stream_id_mapper.has("dumbledores-army"))
# TODO: Add ginny
ron_id = user_id_mapper.get("ron")
harry_id = user_id_mapper.get("harry")
self.assertEqual({ron_id, harry_id}, {1, 2})
self.assertEqual(subscriber_handler.get_users(stream_id=stream_id_mapper.get("gryffindor-common-room")), {ron_id, harry_id})
self.assertEqual(subscriber_handler.get_users(stream_id=stream_id_mapper.get("gryffindor-quidditch-team")), {ron_id, harry_id})
self.assertEqual(subscriber_handler.get_users(stream_id=stream_id_mapper.get("dumbledores-army")), {ron_id, harry_id})
# Converting channel data when a user's `teams` value is `null`.
username_to_user["ron"].update({"teams": None})
zerver_stream = convert_channel_data(
channel_data=mattermost_data["channel"],
user_data_map=username_to_user,
subscriber_handler=subscriber_handler,
stream_id_mapper=stream_id_mapper,
user_id_mapper=user_id_mapper,
realm_id=3,
team_name=team_name,
)
harry_id = user_id_mapper.get("harry")
self.assertIn(harry_id, {1, 2})
self.assertEqual(subscriber_handler.get_users(stream_id=stream_id_mapper.get("gryffindor-common-room")), {harry_id})
self.assertEqual(subscriber_handler.get_users(stream_id=stream_id_mapper.get("gryffindor-quidditch-team")), {harry_id})
self.assertEqual(subscriber_handler.get_users(stream_id=stream_id_mapper.get("dumbledores-army")), {harry_id})
team_name = "slytherin"
zerver_stream = convert_channel_data(
channel_data=mattermost_data["channel"],
user_data_map=username_to_user,
subscriber_handler=subscriber_handler,
stream_id_mapper=stream_id_mapper,
user_id_mapper=user_id_mapper,
realm_id=4,
team_name=team_name,
)
malfoy_id = user_id_mapper.get("malfoy")
pansy_id = user_id_mapper.get("pansy")
snape_id = user_id_mapper.get("snape")
self.assertEqual({malfoy_id, pansy_id, snape_id}, {3, 4, 5})
self.assertEqual(subscriber_handler.get_users(stream_id=stream_id_mapper.get("slytherin-common-room")), {malfoy_id, pansy_id, snape_id})
self.assertEqual(subscriber_handler.get_users(stream_id=stream_id_mapper.get("slytherin-quidditch-team")), {malfoy_id, pansy_id})
def do_convert_data(input_tar_file: str,
output_dir: str,
masking_content: bool,
api_token: Optional[str] = None,
slim_mode: bool = False) -> None:
input_data_dir = untar_input_file(input_tar_file)
attachment_handler = AttachmentHandler()
user_handler = UserHandler()
subscriber_handler = SubscriberHandler()
user_id_mapper = IdMapper()
stream_id_mapper = IdMapper()
realm_id = 0
realm = make_realm(realm_id=realm_id)
# users.json -> UserProfile
raw_user_data = read_user_data(data_dir=input_data_dir)
convert_user_data(
user_handler=user_handler,
slim_mode=slim_mode,
user_id_mapper=user_id_mapper,
raw_data=raw_user_data,
realm_id=realm_id,
)
normal_users = user_handler.get_normal_users()
# Don't write zerver_userprofile here, because we
# may add more users later.
# streams.json -> Stream
raw_stream_data = read_room_data(data_dir=input_data_dir)
zerver_stream = convert_room_data(
raw_data=raw_stream_data,
subscriber_handler=subscriber_handler,
stream_id_mapper=stream_id_mapper,
user_id_mapper=user_id_mapper,
realm_id=realm_id,
api_token=api_token,
)
realm['zerver_stream'] = zerver_stream
zerver_recipient = build_recipients(
zerver_userprofile=normal_users,
zerver_stream=zerver_stream,
)
realm['zerver_recipient'] = zerver_recipient
if api_token is None:
if slim_mode:
public_stream_subscriptions: List[ZerverFieldsT] = []
else:
public_stream_subscriptions = build_public_stream_subscriptions(
zerver_userprofile=normal_users,
zerver_recipient=zerver_recipient,
zerver_stream=zerver_stream,
)
private_stream_subscriptions = build_stream_subscriptions(
get_users=subscriber_handler.get_users,
zerver_recipient=zerver_recipient,
zerver_stream=[
stream_dict for stream_dict in zerver_stream
if stream_dict['invite_only']
],
)
stream_subscriptions = public_stream_subscriptions + private_stream_subscriptions
else:
stream_subscriptions = build_stream_subscriptions(
get_users=subscriber_handler.get_users,
zerver_recipient=zerver_recipient,
zerver_stream=zerver_stream,
)
personal_subscriptions = build_personal_subscriptions(
zerver_recipient=zerver_recipient, )
zerver_subscription = personal_subscriptions + stream_subscriptions
realm['zerver_subscription'] = zerver_subscription
zerver_realmemoji = write_emoticon_data(
realm_id=realm_id,
data_dir=input_data_dir,
output_dir=output_dir,
)
realm['zerver_realmemoji'] = zerver_realmemoji
subscriber_map = make_subscriber_map(
zerver_subscription=zerver_subscription, )
logging.info('Start importing message data')
for message_key in [
'UserMessage', 'NotificationMessage', 'PrivateUserMessage'
]:
write_message_data(
realm_id=realm_id,
slim_mode=slim_mode,
message_key=message_key,
zerver_recipient=zerver_recipient,
subscriber_map=subscriber_map,
data_dir=input_data_dir,
output_dir=output_dir,
masking_content=masking_content,
stream_id_mapper=stream_id_mapper,
user_id_mapper=user_id_mapper,
user_handler=user_handler,
attachment_handler=attachment_handler,
)
# Order is important here...don't write users until
# we process everything else, since we may introduce
# mirror users when processing messages.
realm['zerver_userprofile'] = user_handler.get_all_users()
realm['sort_by_date'] = True
create_converted_data_files(realm, output_dir, '/realm.json')
logging.info('Start importing avatar data')
write_avatar_data(
raw_user_data=raw_user_data,
output_dir=output_dir,
user_id_mapper=user_id_mapper,
realm_id=realm_id,
)
attachment_handler.write_info(
output_dir=output_dir,
realm_id=realm_id,
)
logging.info('Start making tarball')
subprocess.check_call(
["tar", "-czf", output_dir + '.tar.gz', output_dir, '-P'])
logging.info('Done making tarball')
def test_convert_channel_data(self) -> None:
user_handler = UserHandler()
subscriber_handler = SubscriberHandler()
stream_id_mapper = IdMapper()
user_id_mapper = IdMapper()
team_name = "gryffindor"
convert_user_data(
user_handler=user_handler,
user_id_mapper=user_id_mapper,
user_data_map=self.username_to_user,
realm_id=3,
team_name=team_name,
)
zerver_stream = convert_channel_data(
channel_data=self.mattermost_data["channel"],
user_data_map=self.username_to_user,
subscriber_handler=subscriber_handler,
stream_id_mapper=stream_id_mapper,
user_id_mapper=user_id_mapper,
realm_id=3,
team_name=team_name,
)
self.assertEqual(len(zerver_stream), 3)
self.assertEqual(zerver_stream[0]["name"], "Gryffindor common room")
self.assertEqual(zerver_stream[0]["invite_only"], False)
self.assertEqual(zerver_stream[0]["description"],
"A place for talking about Gryffindor common room")
self.assertEqual(zerver_stream[0]["rendered_description"], "")
self.assertEqual(zerver_stream[0]["realm"], 3)
self.assertEqual(zerver_stream[1]["name"], "Gryffindor quidditch team")
self.assertEqual(zerver_stream[1]["invite_only"], False)
self.assertEqual(
zerver_stream[1]["description"],
"A place for talking about Gryffindor quidditch team")
self.assertEqual(zerver_stream[1]["rendered_description"], "")
self.assertEqual(zerver_stream[1]["realm"], 3)
self.assertEqual(zerver_stream[2]["name"], "Dumbledores army")
self.assertEqual(zerver_stream[2]["invite_only"], True)
self.assertEqual(zerver_stream[2]["description"],
"A place for talking about Dumbledores army")
self.assertEqual(zerver_stream[2]["rendered_description"], "")
self.assertEqual(zerver_stream[2]["realm"], 3)
self.assertTrue(stream_id_mapper.has("gryffindor-common-room"))
self.assertTrue(stream_id_mapper.has("gryffindor-quidditch-team"))
self.assertTrue(stream_id_mapper.has("dumbledores-army"))
# TODO: Add ginny
self.assertEqual(
subscriber_handler.get_users(
stream_id_mapper.get("gryffindor-common-room")), {1, 2})
self.assertEqual(
subscriber_handler.get_users(
stream_id_mapper.get("gryffindor-quidditch-team")), {1, 2})
self.assertEqual(
subscriber_handler.get_users(
stream_id_mapper.get("dumbledores-army")), {1, 2})
team_name = "slytherin"
zerver_stream = convert_channel_data(
channel_data=self.mattermost_data["channel"],
user_data_map=self.username_to_user,
subscriber_handler=subscriber_handler,
stream_id_mapper=stream_id_mapper,
user_id_mapper=user_id_mapper,
realm_id=4,
team_name=team_name,
)
self.assertEqual(
subscriber_handler.get_users(
stream_id_mapper.get("slytherin-common-room")), {3, 4, 5})
self.assertEqual(
subscriber_handler.get_users(
stream_id_mapper.get("slytherin-quidditch-team")), {3, 4})
def test_process_users(self) -> None:
fixture_dir_name = self.fixture_file_name("", "rocketchat_fixtures")
rocketchat_data = rocketchat_data_to_dict(fixture_dir_name)
user_id_to_user_map = map_user_id_to_user(rocketchat_data["user"])
realm_id = 3
domain_name = "zulip.com"
user_handler = UserHandler()
user_id_mapper = IdMapper()
process_users(
user_id_to_user_map=user_id_to_user_map,
realm_id=realm_id,
domain_name=domain_name,
user_handler=user_handler,
user_id_mapper=user_id_mapper,
)
self.assert_length(user_handler.get_all_users(), 6)
self.assertTrue(user_id_mapper.has(rocketchat_data["user"][0]["_id"]))
self.assertTrue(user_id_mapper.has(rocketchat_data["user"][4]["_id"]))
user_id = user_id_mapper.get(rocketchat_data["user"][0]["_id"])
user = user_handler.get_user(user_id)
self.assertEqual(user["full_name"], rocketchat_data["user"][0]["name"])
self.assertEqual(user["avatar_source"], "G")
self.assertEqual(user["delivery_email"], "*****@*****.**")
self.assertEqual(user["email"], "*****@*****.**")
self.assertEqual(user["full_name"], "Rocket.Cat")
self.assertEqual(user["id"], 1)
self.assertEqual(user["is_active"], False)
self.assertEqual(user["is_mirror_dummy"], False)
self.assertEqual(user["is_bot"], True)
self.assertEqual(user["bot_type"], 1)
self.assertEqual(user["bot_owner"], 2)
self.assertEqual(user["role"], UserProfile.ROLE_MEMBER)
self.assertEqual(user["realm"], realm_id)
self.assertEqual(user["short_name"], "rocket.cat")
self.assertEqual(user["timezone"], "UTC")
user_id = user_id_mapper.get(rocketchat_data["user"][2]["_id"])
user = user_handler.get_user(user_id)
self.assertEqual(user["full_name"], rocketchat_data["user"][2]["name"])
self.assertEqual(user["avatar_source"], "G")
self.assertEqual(user["delivery_email"], "*****@*****.**")
self.assertEqual(user["email"], "*****@*****.**")
self.assertEqual(user["full_name"], "Harry Potter")
self.assertEqual(user["id"], 3)
self.assertEqual(user["is_active"], True)
self.assertEqual(user["is_mirror_dummy"], False)
self.assertEqual(user["is_bot"], False)
self.assertEqual(user["bot_type"], None)
self.assertEqual(user["bot_owner"], None)
self.assertEqual(user["role"], UserProfile.ROLE_REALM_OWNER)
self.assertEqual(user["realm"], realm_id)
self.assertEqual(user["short_name"], "harry.potter")
self.assertEqual(user["timezone"], "UTC")
# Test `is_mirror_dummy` set for users of type `unknown`
rocketchat_data["user"].append({
"_id":
"s0m34ndmID",
"createdAt":
datetime.datetime(2019, 11, 6, 0, 38, 42, 796000),
"type":
"unknown",
"roles": ["unknown"],
"name":
"Unknown user",
"username":
"******",
})
user_id_to_user_map = map_user_id_to_user(rocketchat_data["user"])
process_users(
user_id_to_user_map=user_id_to_user_map,
realm_id=realm_id,
domain_name=domain_name,
user_handler=user_handler,
user_id_mapper=user_id_mapper,
)
self.assert_length(user_handler.get_all_users(), 7)
self.assertTrue(user_id_mapper.has(rocketchat_data["user"][6]["_id"]))
user_id = user_id_mapper.get(rocketchat_data["user"][6]["_id"])
user = user_handler.get_user(user_id)
self.assertEqual(user["id"], 7)
self.assertEqual(user["is_active"], False)
self.assertEqual(user["is_mirror_dummy"], True)
self.assertEqual(user["is_bot"], False)
def do_convert_data(mattermost_data_dir: str, output_dir: str, masking_content: bool) -> None:
username_to_user: Dict[str, Dict[str, Any]] = {}
os.makedirs(output_dir, exist_ok=True)
if os.listdir(output_dir): # nocoverage
raise Exception("Output directory should be empty!")
mattermost_data_file = os.path.join(mattermost_data_dir, "export.json")
mattermost_data = mattermost_data_file_to_dict(mattermost_data_file)
username_to_user = create_username_to_user_mapping(mattermost_data["user"])
for team in mattermost_data["team"]:
realm_id = NEXT_ID("realm_id")
team_name = team["name"]
user_handler = UserHandler()
subscriber_handler = SubscriberHandler()
user_id_mapper = IdMapper()
stream_id_mapper = IdMapper()
huddle_id_mapper = IdMapper()
print("Generating data for", team_name)
realm = make_realm(realm_id, team)
realm_output_dir = os.path.join(output_dir, team_name)
reset_mirror_dummy_users(username_to_user)
label_mirror_dummy_users(
len(mattermost_data["team"]), team_name, mattermost_data, username_to_user
)
convert_user_data(
user_handler=user_handler,
user_id_mapper=user_id_mapper,
user_data_map=username_to_user,
realm_id=realm_id,
team_name=team_name,
)
zerver_stream = convert_channel_data(
channel_data=mattermost_data["channel"],
user_data_map=username_to_user,
subscriber_handler=subscriber_handler,
stream_id_mapper=stream_id_mapper,
user_id_mapper=user_id_mapper,
realm_id=realm_id,
team_name=team_name,
)
realm["zerver_stream"] = zerver_stream
zerver_huddle: List[ZerverFieldsT] = []
if len(mattermost_data["team"]) == 1:
zerver_huddle = convert_huddle_data(
huddle_data=mattermost_data["direct_channel"],
user_data_map=username_to_user,
subscriber_handler=subscriber_handler,
huddle_id_mapper=huddle_id_mapper,
user_id_mapper=user_id_mapper,
realm_id=realm_id,
team_name=team_name,
)
realm["zerver_huddle"] = zerver_huddle
all_users = user_handler.get_all_users()
zerver_recipient = build_recipients(
zerver_userprofile=all_users,
zerver_stream=zerver_stream,
zerver_huddle=zerver_huddle,
)
realm["zerver_recipient"] = zerver_recipient
stream_subscriptions = build_stream_subscriptions(
get_users=subscriber_handler.get_users,
zerver_recipient=zerver_recipient,
zerver_stream=zerver_stream,
)
huddle_subscriptions = build_huddle_subscriptions(
get_users=subscriber_handler.get_users,
zerver_recipient=zerver_recipient,
zerver_huddle=zerver_huddle,
)
personal_subscriptions = build_personal_subscriptions(
zerver_recipient=zerver_recipient,
)
# Mattermost currently supports only exporting messages from channels.
# Personal messages and huddles are not exported.
zerver_subscription = personal_subscriptions + stream_subscriptions + huddle_subscriptions
realm["zerver_subscription"] = zerver_subscription
zerver_realmemoji = write_emoticon_data(
realm_id=realm_id,
custom_emoji_data=mattermost_data["emoji"],
data_dir=mattermost_data_dir,
output_dir=realm_output_dir,
)
realm["zerver_realmemoji"] = zerver_realmemoji
subscriber_map = make_subscriber_map(
zerver_subscription=zerver_subscription,
)
total_reactions: List[Dict[str, Any]] = []
uploads_list: List[ZerverFieldsT] = []
zerver_attachment: List[ZerverFieldsT] = []
write_message_data(
num_teams=len(mattermost_data["team"]),
team_name=team_name,
realm_id=realm_id,
post_data=mattermost_data["post"],
zerver_recipient=zerver_recipient,
subscriber_map=subscriber_map,
output_dir=realm_output_dir,
masking_content=masking_content,
stream_id_mapper=stream_id_mapper,
huddle_id_mapper=huddle_id_mapper,
user_id_mapper=user_id_mapper,
user_handler=user_handler,
zerver_realmemoji=zerver_realmemoji,
total_reactions=total_reactions,
uploads_list=uploads_list,
zerver_attachment=zerver_attachment,
mattermost_data_dir=mattermost_data_dir,
)
realm["zerver_reaction"] = total_reactions
realm["zerver_userprofile"] = user_handler.get_all_users()
realm["sort_by_date"] = True
create_converted_data_files(realm, realm_output_dir, "/realm.json")
# Mattermost currently doesn't support exporting avatars
create_converted_data_files([], realm_output_dir, "/avatars/records.json")
# Export message attachments
attachment: Dict[str, List[Any]] = {"zerver_attachment": zerver_attachment}
create_converted_data_files(uploads_list, realm_output_dir, "/uploads/records.json")
create_converted_data_files(attachment, realm_output_dir, "/attachment.json")
logging.info("Start making tarball")
subprocess.check_call(["tar", "-czf", realm_output_dir + ".tar.gz", realm_output_dir, "-P"])
logging.info("Done making tarball")
def test_map_receiver_id_to_recipient_id(self) -> None:
fixture_dir_name = self.fixture_file_name("", "rocketchat_fixtures")
rocketchat_data = rocketchat_data_to_dict(fixture_dir_name)
realm_id = 3
domain_name = "zulip.com"
user_handler = UserHandler()
subscriber_handler = SubscriberHandler()
user_id_mapper = IdMapper()
stream_id_mapper = IdMapper()
huddle_id_mapper = IdMapper()
user_id_to_user_map = map_user_id_to_user(rocketchat_data["user"])
process_users(
user_id_to_user_map=user_id_to_user_map,
realm_id=realm_id,
domain_name=domain_name,
user_handler=user_handler,
user_id_mapper=user_id_mapper,
)
room_id_to_room_map: Dict[str, Dict[str, Any]] = {}
team_id_to_team_map: Dict[str, Dict[str, Any]] = {}
dsc_id_to_dsc_map: Dict[str, Dict[str, Any]] = {}
direct_id_to_direct_map: Dict[str, Dict[str, Any]] = {}
huddle_id_to_huddle_map: Dict[str, Dict[str, Any]] = {}
categorize_channels_and_map_with_id(
channel_data=rocketchat_data["room"],
room_id_to_room_map=room_id_to_room_map,
team_id_to_team_map=team_id_to_team_map,
dsc_id_to_dsc_map=dsc_id_to_dsc_map,
direct_id_to_direct_map=direct_id_to_direct_map,
huddle_id_to_huddle_map=huddle_id_to_huddle_map,
)
zerver_stream = convert_channel_data(
room_id_to_room_map=room_id_to_room_map,
team_id_to_team_map=team_id_to_team_map,
stream_id_mapper=stream_id_mapper,
realm_id=realm_id,
)
zerver_huddle = convert_huddle_data(
huddle_id_to_huddle_map=huddle_id_to_huddle_map,
huddle_id_mapper=huddle_id_mapper,
user_id_mapper=user_id_mapper,
subscriber_handler=subscriber_handler,
)
all_users = user_handler.get_all_users()
zerver_recipient = build_recipients(
zerver_userprofile=all_users,
zerver_stream=zerver_stream,
zerver_huddle=zerver_huddle,
)
stream_id_to_recipient_id: Dict[int, int] = {}
user_id_to_recipient_id: Dict[int, int] = {}
huddle_id_to_recipient_id: Dict[int, int] = {}
map_receiver_id_to_recipient_id(
zerver_recipient=zerver_recipient,
stream_id_to_recipient_id=stream_id_to_recipient_id,
user_id_to_recipient_id=user_id_to_recipient_id,
huddle_id_to_recipient_id=huddle_id_to_recipient_id,
)
# 6 for streams and 6 for users.
self.assert_length(zerver_recipient, 13)
self.assert_length(stream_id_to_recipient_id, 6)
self.assert_length(user_id_to_recipient_id, 6)
self.assert_length(huddle_id_to_recipient_id, 1)
# First user recipients are built, followed by stream recipients in `build_recipients`.
self.assertEqual(
user_id_to_recipient_id[zerver_recipient[0]["type_id"]],
zerver_recipient[0]["id"])
self.assertEqual(
user_id_to_recipient_id[zerver_recipient[1]["type_id"]],
zerver_recipient[1]["id"])
self.assertEqual(
stream_id_to_recipient_id[zerver_recipient[6]["type_id"]],
zerver_recipient[6]["id"])
self.assertEqual(
stream_id_to_recipient_id[zerver_recipient[7]["type_id"]],
zerver_recipient[7]["id"])
self.assertEqual(
huddle_id_to_recipient_id[zerver_recipient[12]["type_id"]],
zerver_recipient[12]["id"])
def test_convert_stream_subscription_data(self) -> None:
fixture_dir_name = self.fixture_file_name("", "rocketchat_fixtures")
rocketchat_data = rocketchat_data_to_dict(fixture_dir_name)
realm_id = 3
domain_name = "zulip.com"
user_handler = UserHandler()
subscriber_handler = SubscriberHandler()
user_id_mapper = IdMapper()
stream_id_mapper = IdMapper()
user_id_to_user_map = map_user_id_to_user(rocketchat_data["user"])
process_users(
user_id_to_user_map=user_id_to_user_map,
realm_id=realm_id,
domain_name=domain_name,
user_handler=user_handler,
user_id_mapper=user_id_mapper,
)
room_id_to_room_map: Dict[str, Dict[str, Any]] = {}
team_id_to_team_map: Dict[str, Dict[str, Any]] = {}
dsc_id_to_dsc_map: Dict[str, Dict[str, Any]] = {}
direct_id_to_direct_map: Dict[str, Dict[str, Any]] = {}
huddle_id_to_huddle_map: Dict[str, Dict[str, Any]] = {}
categorize_channels_and_map_with_id(
channel_data=rocketchat_data["room"],
room_id_to_room_map=room_id_to_room_map,
team_id_to_team_map=team_id_to_team_map,
dsc_id_to_dsc_map=dsc_id_to_dsc_map,
direct_id_to_direct_map=direct_id_to_direct_map,
huddle_id_to_huddle_map=huddle_id_to_huddle_map,
)
zerver_stream = convert_channel_data(
room_id_to_room_map=room_id_to_room_map,
team_id_to_team_map=team_id_to_team_map,
stream_id_mapper=stream_id_mapper,
realm_id=realm_id,
)
convert_stream_subscription_data(
user_id_to_user_map=user_id_to_user_map,
dsc_id_to_dsc_map=dsc_id_to_dsc_map,
zerver_stream=zerver_stream,
stream_id_mapper=stream_id_mapper,
user_id_mapper=user_id_mapper,
subscriber_handler=subscriber_handler,
)
priyansh_id = user_id_mapper.get(rocketchat_data["user"][1]["_id"])
harry_id = user_id_mapper.get(rocketchat_data["user"][2]["_id"])
hermione_id = user_id_mapper.get(rocketchat_data["user"][3]["_id"])
ron_id = user_id_mapper.get(rocketchat_data["user"][4]["_id"])
voldemort_id = user_id_mapper.get(rocketchat_data["user"][5]["_id"])
self.assertEqual(
subscriber_handler.get_users(stream_id=zerver_stream[0]["id"]),
{priyansh_id, harry_id, ron_id, hermione_id, voldemort_id},
)
self.assertEqual(
subscriber_handler.get_users(stream_id=zerver_stream[1]["id"]),
{priyansh_id, harry_id})
self.assertEqual(
subscriber_handler.get_users(stream_id=zerver_stream[2]["id"]),
{harry_id, hermione_id})
self.assertEqual(
subscriber_handler.get_users(stream_id=zerver_stream[3]["id"]),
{harry_id, ron_id, hermione_id},
)
self.assertEqual(
subscriber_handler.get_users(stream_id=zerver_stream[4]["id"]),
{harry_id})
self.assertEqual(
subscriber_handler.get_users(stream_id=zerver_stream[5]["id"]),
{harry_id})
# Add a new channel with no user.
no_user_channel: Dict[str, Any] = {
"_id": "rand0mID",
"ts": datetime.datetime(2021, 7, 15, 10, 58, 23, 647000),
"t": "c",
"name": "no-user-channel",
}
room_id_to_room_map[no_user_channel["_id"]] = no_user_channel
zerver_stream = convert_channel_data(
room_id_to_room_map=room_id_to_room_map,
team_id_to_team_map=team_id_to_team_map,
stream_id_mapper=stream_id_mapper,
realm_id=realm_id,
)
convert_stream_subscription_data(
user_id_to_user_map=user_id_to_user_map,
dsc_id_to_dsc_map=dsc_id_to_dsc_map,
zerver_stream=zerver_stream,
stream_id_mapper=stream_id_mapper,
user_id_mapper=user_id_mapper,
subscriber_handler=subscriber_handler,
)
self.assert_length(
subscriber_handler.get_users(stream_id=zerver_stream[6]["id"]), 0)
self.assertTrue(zerver_stream[6]["deactivated"])
def test_convert_channel_data(self) -> None:
fixture_dir_name = self.fixture_file_name("", "rocketchat_fixtures")
rocketchat_data = rocketchat_data_to_dict(fixture_dir_name)
realm_id = 3
stream_id_mapper = IdMapper()
room_id_to_room_map: Dict[str, Dict[str, Any]] = {}
team_id_to_team_map: Dict[str, Dict[str, Any]] = {}
dsc_id_to_dsc_map: Dict[str, Dict[str, Any]] = {}
direct_id_to_direct_map: Dict[str, Dict[str, Any]] = {}
huddle_id_to_huddle_map: Dict[str, Dict[str, Any]] = {}
categorize_channels_and_map_with_id(
channel_data=rocketchat_data["room"],
room_id_to_room_map=room_id_to_room_map,
team_id_to_team_map=team_id_to_team_map,
dsc_id_to_dsc_map=dsc_id_to_dsc_map,
direct_id_to_direct_map=direct_id_to_direct_map,
huddle_id_to_huddle_map=huddle_id_to_huddle_map,
)
zerver_stream = convert_channel_data(
room_id_to_room_map=room_id_to_room_map,
team_id_to_team_map=team_id_to_team_map,
stream_id_mapper=stream_id_mapper,
realm_id=realm_id,
)
# Only rooms are converted to streams.
self.assert_length(room_id_to_room_map, 6)
self.assert_length(zerver_stream, 6)
# Normal public stream
self.assertEqual(zerver_stream[0]["name"], "general")
self.assertEqual(zerver_stream[0]["invite_only"], False)
self.assertEqual(zerver_stream[0]["description"],
"This is a general channel.")
self.assertEqual(zerver_stream[0]["rendered_description"], "")
self.assertEqual(zerver_stream[0]["stream_post_policy"], 1)
self.assertEqual(zerver_stream[0]["realm"], realm_id)
# Private stream
self.assertEqual(zerver_stream[1]["name"], "random")
self.assertEqual(zerver_stream[1]["invite_only"], True)
self.assertEqual(zerver_stream[1]["description"], "")
self.assertEqual(zerver_stream[1]["rendered_description"], "")
self.assertEqual(zerver_stream[1]["stream_post_policy"], 1)
self.assertEqual(zerver_stream[1]["realm"], realm_id)
# Team main
self.assertEqual(zerver_stream[3]["name"], "[TEAM] team-harry-potter")
self.assertEqual(zerver_stream[3]["invite_only"], True)
self.assertEqual(zerver_stream[3]["description"],
"Welcome to the official Harry Potter team.")
self.assertEqual(zerver_stream[3]["rendered_description"], "")
self.assertEqual(zerver_stream[3]["stream_post_policy"], 1)
self.assertEqual(zerver_stream[3]["realm"], realm_id)
# Team channel
self.assertEqual(zerver_stream[5]["name"], "thp-channel-2")
self.assertEqual(zerver_stream[5]["invite_only"], False)
self.assertEqual(zerver_stream[5]["description"],
"[Team team-harry-potter channel]. ")
self.assertEqual(zerver_stream[5]["rendered_description"], "")
self.assertEqual(zerver_stream[5]["stream_post_policy"], 1)
self.assertEqual(zerver_stream[5]["realm"], realm_id)
def do_convert_data(rocketchat_data_dir: str, output_dir: str) -> None:
# Get all required exported data in a dictionary
rocketchat_data = rocketchat_data_to_dict(rocketchat_data_dir)
# Subdomain is set by the user while running the import command
realm_subdomain = ""
realm_id = 0
domain_name = settings.EXTERNAL_HOST
realm = make_realm(realm_id, realm_subdomain, domain_name,
rocketchat_data["instance"][0])
user_id_to_user_map: Dict[str, Dict[str, Any]] = map_user_id_to_user(
rocketchat_data["user"])
username_to_user_id_map: Dict[str, str] = map_username_to_user_id(
user_id_to_user_map)
user_handler = UserHandler()
subscriber_handler = SubscriberHandler()
user_id_mapper = IdMapper()
stream_id_mapper = IdMapper()
huddle_id_mapper = IdMapper()
process_users(
user_id_to_user_map=user_id_to_user_map,
realm_id=realm_id,
domain_name=domain_name,
user_handler=user_handler,
user_id_mapper=user_id_mapper,
)
room_id_to_room_map: Dict[str, Dict[str, Any]] = {}
team_id_to_team_map: Dict[str, Dict[str, Any]] = {}
dsc_id_to_dsc_map: Dict[str, Dict[str, Any]] = {}
direct_id_to_direct_map: Dict[str, Dict[str, Any]] = {}
huddle_id_to_huddle_map: Dict[str, Dict[str, Any]] = {}
categorize_channels_and_map_with_id(
channel_data=rocketchat_data["room"],
room_id_to_room_map=room_id_to_room_map,
team_id_to_team_map=team_id_to_team_map,
dsc_id_to_dsc_map=dsc_id_to_dsc_map,
direct_id_to_direct_map=direct_id_to_direct_map,
huddle_id_to_huddle_map=huddle_id_to_huddle_map,
)
zerver_stream = convert_channel_data(
room_id_to_room_map=room_id_to_room_map,
team_id_to_team_map=team_id_to_team_map,
stream_id_mapper=stream_id_mapper,
realm_id=realm_id,
)
realm["zerver_stream"] = zerver_stream
# Add stream subscription data to `subscriber_handler`
convert_stream_subscription_data(
user_id_to_user_map=user_id_to_user_map,
dsc_id_to_dsc_map=dsc_id_to_dsc_map,
zerver_stream=zerver_stream,
stream_id_mapper=stream_id_mapper,
user_id_mapper=user_id_mapper,
subscriber_handler=subscriber_handler,
)
zerver_huddle = convert_huddle_data(
huddle_id_to_huddle_map=huddle_id_to_huddle_map,
huddle_id_mapper=huddle_id_mapper,
user_id_mapper=user_id_mapper,
subscriber_handler=subscriber_handler,
)
realm["zerver_huddle"] = zerver_huddle
all_users = user_handler.get_all_users()
zerver_recipient = build_recipients(
zerver_userprofile=all_users,
zerver_stream=zerver_stream,
zerver_huddle=zerver_huddle,
)
realm["zerver_recipient"] = zerver_recipient
stream_subscriptions = build_stream_subscriptions(
get_users=subscriber_handler.get_users,
zerver_recipient=zerver_recipient,
zerver_stream=zerver_stream,
)
huddle_subscriptions = build_huddle_subscriptions(
get_users=subscriber_handler.get_users,
zerver_recipient=zerver_recipient,
zerver_huddle=zerver_huddle,
)
personal_subscriptions = build_personal_subscriptions(
zerver_recipient=zerver_recipient, )
zerver_subscription = personal_subscriptions + stream_subscriptions + huddle_subscriptions
realm["zerver_subscription"] = zerver_subscription
zerver_realmemoji = build_custom_emoji(
realm_id=realm_id,
custom_emoji_data=rocketchat_data["custom_emoji"],
output_dir=output_dir,
)
realm["zerver_realmemoji"] = zerver_realmemoji
subscriber_map = make_subscriber_map(
zerver_subscription=zerver_subscription, )
stream_id_to_recipient_id: Dict[int, int] = {}
huddle_id_to_recipient_id: Dict[int, int] = {}
user_id_to_recipient_id: Dict[int, int] = {}
map_receiver_id_to_recipient_id(
zerver_recipient=zerver_recipient,
stream_id_to_recipient_id=stream_id_to_recipient_id,
huddle_id_to_recipient_id=huddle_id_to_recipient_id,
user_id_to_recipient_id=user_id_to_recipient_id,
)
channel_messages: List[Dict[str, Any]] = []
private_messages: List[Dict[str, Any]] = []
separate_channel_and_private_messages(
messages=rocketchat_data["message"],
direct_id_to_direct_map=direct_id_to_direct_map,
huddle_id_to_huddle_map=huddle_id_to_huddle_map,
channel_messages=channel_messages,
private_messages=private_messages,
)
total_reactions: List[ZerverFieldsT] = []
uploads_list: List[ZerverFieldsT] = []
zerver_attachment: List[ZerverFieldsT] = []
upload_id_to_upload_data_map = map_upload_id_to_upload_data(
rocketchat_data["upload"])
# Process channel messages
process_messages(
realm_id=realm_id,
messages=channel_messages,
subscriber_map=subscriber_map,
is_pm_data=False,
username_to_user_id_map=username_to_user_id_map,
user_id_mapper=user_id_mapper,
user_handler=user_handler,
user_id_to_recipient_id=user_id_to_recipient_id,
stream_id_mapper=stream_id_mapper,
stream_id_to_recipient_id=stream_id_to_recipient_id,
huddle_id_mapper=huddle_id_mapper,
huddle_id_to_recipient_id=huddle_id_to_recipient_id,
room_id_to_room_map=room_id_to_room_map,
dsc_id_to_dsc_map=dsc_id_to_dsc_map,
direct_id_to_direct_map=direct_id_to_direct_map,
huddle_id_to_huddle_map=huddle_id_to_huddle_map,
zerver_realmemoji=zerver_realmemoji,
total_reactions=total_reactions,
uploads_list=uploads_list,
zerver_attachment=zerver_attachment,
upload_id_to_upload_data_map=upload_id_to_upload_data_map,
output_dir=output_dir,
)
# Process private messages
process_messages(
realm_id=realm_id,
messages=private_messages,
subscriber_map=subscriber_map,
is_pm_data=True,
username_to_user_id_map=username_to_user_id_map,
user_id_mapper=user_id_mapper,
user_handler=user_handler,
user_id_to_recipient_id=user_id_to_recipient_id,
stream_id_mapper=stream_id_mapper,
stream_id_to_recipient_id=stream_id_to_recipient_id,
huddle_id_mapper=huddle_id_mapper,
huddle_id_to_recipient_id=huddle_id_to_recipient_id,
room_id_to_room_map=room_id_to_room_map,
dsc_id_to_dsc_map=dsc_id_to_dsc_map,
direct_id_to_direct_map=direct_id_to_direct_map,
huddle_id_to_huddle_map=huddle_id_to_huddle_map,
zerver_realmemoji=zerver_realmemoji,
total_reactions=total_reactions,
uploads_list=uploads_list,
zerver_attachment=zerver_attachment,
upload_id_to_upload_data_map=upload_id_to_upload_data_map,
output_dir=output_dir,
)
realm["zerver_reaction"] = total_reactions
realm["zerver_userprofile"] = user_handler.get_all_users()
realm["sort_by_date"] = True
create_converted_data_files(realm, output_dir, "/realm.json")
# TODO: Add support for importing avatars
create_converted_data_files([], output_dir, "/avatars/records.json")
# Import attachments
attachment: Dict[str, List[Any]] = {"zerver_attachment": zerver_attachment}
create_converted_data_files(attachment, output_dir, "/attachment.json")
create_converted_data_files(uploads_list, output_dir,
"/uploads/records.json")
logging.info("Start making tarball")
subprocess.check_call(
["tar", "-czf", output_dir + ".tar.gz", output_dir, "-P"])
logging.info("Done making tarball")
def test_process_message_attachments(self) -> None:
mattermost_data_dir = self.fixture_file_name(
"", "mattermost_fixtures/direct_channel")
output_dir = self.make_import_output_dir("mattermost")
fixture_file_name = self.fixture_file_name(
"export.json", "mattermost_fixtures/direct_channel")
mattermost_data = mattermost_data_file_to_dict(fixture_file_name)
username_to_user = create_username_to_user_mapping(
mattermost_data["user"])
reset_mirror_dummy_users(username_to_user)
user_handler = UserHandler()
user_id_mapper = IdMapper()
team_name = "gryffindor"
convert_user_data(
user_handler=user_handler,
user_id_mapper=user_id_mapper,
user_data_map=username_to_user,
realm_id=3,
team_name=team_name,
)
zerver_attachments: List[ZerverFieldsT] = []
uploads_list: List[ZerverFieldsT] = []
process_message_attachments(
attachments=mattermost_data["post"]["direct_post"][0]
["attachments"],
realm_id=3,
message_id=1,
user_id=2,
user_handler=user_handler,
zerver_attachment=zerver_attachments,
uploads_list=uploads_list,
mattermost_data_dir=mattermost_data_dir,
output_dir=output_dir,
)
self.assert_length(zerver_attachments, 1)
self.assertEqual(zerver_attachments[0]["file_name"], "harry-ron.jpg")
self.assertEqual(zerver_attachments[0]["owner"], 2)
self.assertEqual(
user_handler.get_user(zerver_attachments[0]["owner"])["email"],
"*****@*****.**")
# TODO: Assert this for False after fixing the file permissions in PMs
self.assertTrue(zerver_attachments[0]["is_realm_public"])
self.assert_length(uploads_list, 1)
self.assertEqual(uploads_list[0]["user_profile_email"],
"*****@*****.**")
attachment_path = self.fixture_file_name(
mattermost_data["post"]["direct_post"][0]["attachments"][0]
["path"],
"mattermost_fixtures/direct_channel/data",
)
attachment_out_path = os.path.join(output_dir, "uploads",
zerver_attachments[0]["path_id"])
self.assertTrue(os.path.exists(attachment_out_path))
self.assertTrue(filecmp.cmp(attachment_path, attachment_out_path))
