def test_mattermost_data_file_to_dict(self) -> None:
fixture_file_name = self.fixture_file_name("export.json",
"mattermost_fixtures")
mattermost_data = mattermost_data_file_to_dict(fixture_file_name)
self.assert_length(mattermost_data, 7)
self.assertEqual(mattermost_data["version"], [1])
self.assert_length(mattermost_data["team"], 2)
self.assertEqual(mattermost_data["team"][0]["name"], "gryffindor")
self.assert_length(mattermost_data["channel"], 5)
self.assertEqual(mattermost_data["channel"][0]["name"],
"gryffindor-common-room")
self.assertEqual(mattermost_data["channel"][0]["team"], "gryffindor")
self.assert_length(mattermost_data["user"], 5)
self.assertEqual(mattermost_data["user"][1]["username"], "harry")
self.assert_length(mattermost_data["user"][1]["teams"], 1)
self.assert_length(mattermost_data["post"]["channel_post"], 20)
self.assertEqual(mattermost_data["post"]["channel_post"][0]["team"],
"gryffindor")
self.assertEqual(mattermost_data["post"]["channel_post"][0]["channel"],
"dumbledores-army")
self.assertEqual(mattermost_data["post"]["channel_post"][0]["user"],
"harry")
self.assert_length(
mattermost_data["post"]["channel_post"][0]["replies"], 1)
self.assert_length(mattermost_data["emoji"], 2)
self.assertEqual(mattermost_data["emoji"][0]["name"], "peerdium")
fixture_file_name = self.fixture_file_name(
"export.json", "mattermost_fixtures/direct_channel")
mattermost_data = mattermost_data_file_to_dict(fixture_file_name)
self.assert_length(mattermost_data["post"]["channel_post"], 4)
self.assertEqual(mattermost_data["post"]["channel_post"][0]["team"],
"gryffindor")
self.assertEqual(mattermost_data["post"]["channel_post"][0]["channel"],
"gryffindor-common-room")
self.assertEqual(mattermost_data["post"]["channel_post"][0]["user"],
"ron")
self.assertEqual(mattermost_data["post"]["channel_post"][0]["replies"],
None)
self.assert_length(mattermost_data["post"]["direct_post"], 7)
self.assertEqual(mattermost_data["post"]["direct_post"][0]["user"],
"ron")
self.assertEqual(mattermost_data["post"]["direct_post"][0]["replies"],
None)
self.assertEqual(mattermost_data["post"]["direct_post"][0]["message"],
"hey harry")
self.assertEqual(
mattermost_data["post"]["direct_post"][0]["channel_members"],
["ron", "harry"])
def test_write_emoticon_data(self) -> None:
fixture_file_name = self.fixture_file_name("export.json", "mattermost_fixtures")
mattermost_data = mattermost_data_file_to_dict(fixture_file_name)
output_dir = self.make_import_output_dir("mattermost")
with self.assertLogs(level="INFO"):
zerver_realm_emoji = write_emoticon_data(
realm_id=3,
custom_emoji_data=mattermost_data["emoji"],
data_dir=self.fixture_file_name("", "mattermost_fixtures"),
output_dir = output_dir,
)
self.assertEqual(len(zerver_realm_emoji), 2)
self.assertEqual(zerver_realm_emoji[0]["file_name"], "peerdium")
self.assertEqual(zerver_realm_emoji[0]["realm"], 3)
self.assertEqual(zerver_realm_emoji[0]["deactivated"], False)
self.assertEqual(zerver_realm_emoji[1]["file_name"], "tick")
self.assertEqual(zerver_realm_emoji[1]["realm"], 3)
self.assertEqual(zerver_realm_emoji[1]["deactivated"], False)
records_file = os.path.join(output_dir, "emoji", "records.json")
with open(records_file) as f:
records_json = ujson.load(f)
self.assertEqual(records_json[0]["file_name"], "peerdium")
self.assertEqual(records_json[0]["realm_id"], 3)
exported_emoji_path = self.fixture_file_name(mattermost_data["emoji"][0]["image"], "mattermost_fixtures")
self.assertTrue(filecmp.cmp(records_json[0]["path"], exported_emoji_path))
self.assertEqual(records_json[1]["file_name"], "tick")
self.assertEqual(records_json[1]["realm_id"], 3)
exported_emoji_path = self.fixture_file_name(mattermost_data["emoji"][1]["image"], "mattermost_fixtures")
self.assertTrue(filecmp.cmp(records_json[1]["path"], exported_emoji_path))
def setUp(self) -> None:
fixture_file_name = self.fixture_file_name("export.json",
"mattermost_fixtures")
self.mattermost_data = mattermost_data_file_to_dict(fixture_file_name)
self.username_to_user = create_username_to_user_mapping(
self.mattermost_data["user"])
reset_mirror_dummy_users(self.username_to_user)
def test_mattermost_data_file_to_dict(self) -> None:
fixture_file_name = self.fixture_file_name("export.json", "mattermost_fixtures")
mattermost_data = mattermost_data_file_to_dict(fixture_file_name)
self.assertEqual(len(mattermost_data), 6)
self.assertEqual(mattermost_data["version"], [1])
self.assertEqual(len(mattermost_data["team"]), 2)
self.assertEqual(mattermost_data["team"][0]["name"], "gryffindor")
self.assertEqual(len(mattermost_data["channel"]), 5)
self.assertEqual(mattermost_data["channel"][0]["name"], "gryffindor-common-room")
self.assertEqual(mattermost_data["channel"][0]["team"], "gryffindor")
self.assertEqual(len(mattermost_data["user"]), 5)
self.assertEqual(mattermost_data["user"][1]["username"], "harry")
self.assertEqual(len(mattermost_data["user"][1]["teams"]), 1)
self.assertEqual(len(mattermost_data["post"]), 20)
self.assertEqual(mattermost_data["post"][0]["team"], "gryffindor")
self.assertEqual(mattermost_data["post"][0]["channel"], "dumbledores-army")
self.assertEqual(mattermost_data["post"][0]["user"], "harry")
self.assertEqual(len(mattermost_data["post"][0]["replies"]), 1)
self.assertEqual(len(mattermost_data["emoji"]), 2)
self.assertEqual(mattermost_data["emoji"][0]["name"], "peerdium")
def test_build_reactions(self) -> None:
fixture_file_name = self.fixture_file_name("export.json", "mattermost_fixtures")
mattermost_data = mattermost_data_file_to_dict(fixture_file_name)
total_reactions = [] # type: List[Dict[str, Any]]
reactions = [
{"user": "******", "create_at": 1553165521410, "emoji_name": "tick"},
{"user": "******", "create_at": 1553166530805, "emoji_name": "smile"},
{"user": "******", "create_at": 1553166540953, "emoji_name": "world_map"},
{"user": "******", "create_at": 1553166540957, "emoji_name": "world_map"}
]
zerver_realmemoji = write_emoticon_data(
realm_id=3,
custom_emoji_data=mattermost_data["emoji"],
data_dir=self.fixture_file_name("", "mattermost_fixtures"),
output_dir=self.make_import_output_dir("mattermost")
)
# Make sure tick is present in fixture data
self.assertEqual(zerver_realmemoji[1]["name"], "tick")
tick_emoji_code = zerver_realmemoji[1]["id"]
name_to_codepoint = get_name_to_codepoint_dict()
user_id_mapper = IdMapper()
harry_id = user_id_mapper.get("harry")
ron_id = user_id_mapper.get("ron")
build_reactions(
realm_id=3,
total_reactions=total_reactions,
reactions=reactions,
message_id=5,
name_to_codepoint=name_to_codepoint,
user_id_mapper=user_id_mapper,
zerver_realmemoji=zerver_realmemoji
)
smile_emoji_code = name_to_codepoint["smile"]
world_map_emoji_code = name_to_codepoint["world_map"]
self.assertEqual(len(total_reactions), 4)
self.assertEqual(self.get_set(total_reactions, "reaction_type"), set([Reaction.REALM_EMOJI, Reaction.UNICODE_EMOJI]))
self.assertEqual(self.get_set(total_reactions, "emoji_name"), set(["tick", "smile", "world_map"]))
self.assertEqual(self.get_set(total_reactions, "emoji_code"), set([tick_emoji_code, smile_emoji_code,
world_map_emoji_code]))
self.assertEqual(self.get_set(total_reactions, "user_profile"), set([harry_id, ron_id]))
self.assertEqual(len(self.get_set(total_reactions, "id")), 4)
self.assertEqual(len(self.get_set(total_reactions, "message")), 1)
def test_process_user(self) -> None:
user_id_mapper = IdMapper()
fixture_file_name = self.fixture_file_name("export.json",
"mattermost_fixtures")
mattermost_data = mattermost_data_file_to_dict(fixture_file_name)
username_to_user = create_username_to_user_mapping(
mattermost_data["user"])
reset_mirror_dummy_users(username_to_user)
harry_dict = username_to_user["harry"]
harry_dict["is_mirror_dummy"] = False
realm_id = 3
team_name = "gryffindor"
user = process_user(harry_dict, realm_id, team_name, user_id_mapper)
self.assertEqual(user["avatar_source"], "G")
self.assertEqual(user["delivery_email"], "*****@*****.**")
self.assertEqual(user["email"], "*****@*****.**")
self.assertEqual(user["full_name"], "Harry Potter")
self.assertEqual(user["id"], 1)
self.assertEqual(user["is_active"], True)
self.assertEqual(user["role"], UserProfile.ROLE_REALM_OWNER)
self.assertEqual(user["is_mirror_dummy"], False)
self.assertEqual(user["realm"], 3)
self.assertEqual(user["short_name"], "harry")
self.assertEqual(user["timezone"], "UTC")
# A user with a `null` team value shouldn't be an admin.
harry_dict["teams"] = None
user = process_user(harry_dict, realm_id, team_name, user_id_mapper)
self.assertEqual(user["role"], UserProfile.ROLE_MEMBER)
team_name = "slytherin"
snape_dict = username_to_user["snape"]
snape_dict["is_mirror_dummy"] = True
user = process_user(snape_dict, realm_id, team_name, user_id_mapper)
self.assertEqual(user["avatar_source"], "G")
self.assertEqual(user["delivery_email"], "*****@*****.**")
self.assertEqual(user["email"], "*****@*****.**")
self.assertEqual(user["full_name"], "Severus Snape")
self.assertEqual(user["id"], 2)
self.assertEqual(user["is_active"], False)
self.assertEqual(user["role"], UserProfile.ROLE_MEMBER)
self.assertEqual(user["is_mirror_dummy"], True)
self.assertEqual(user["realm"], 3)
self.assertEqual(user["short_name"], "snape")
self.assertEqual(user["timezone"], "UTC")
def test_convert_user_data(self) -> None:
user_id_mapper = IdMapper()
realm_id = 3
fixture_file_name = self.fixture_file_name("export.json",
"mattermost_fixtures")
mattermost_data = mattermost_data_file_to_dict(fixture_file_name)
username_to_user = create_username_to_user_mapping(
mattermost_data["user"])
reset_mirror_dummy_users(username_to_user)
team_name = "gryffindor"
user_handler = UserHandler()
convert_user_data(user_handler, user_id_mapper, username_to_user,
realm_id, team_name)
self.assert_length(user_handler.get_all_users(), 2)
self.assertTrue(user_id_mapper.has("harry"))
self.assertTrue(user_id_mapper.has("ron"))
self.assertEqual(
user_handler.get_user(user_id_mapper.get("harry"))["full_name"],
"Harry Potter")
self.assertEqual(
user_handler.get_user(user_id_mapper.get("ron"))["full_name"],
"Ron Weasley")
team_name = "slytherin"
user_handler = UserHandler()
convert_user_data(user_handler, user_id_mapper, username_to_user,
realm_id, team_name)
self.assert_length(user_handler.get_all_users(), 3)
self.assertTrue(user_id_mapper.has("malfoy"))
self.assertTrue(user_id_mapper.has("pansy"))
self.assertTrue(user_id_mapper.has("snape"))
team_name = "gryffindor"
# Snape is a mirror dummy user in Harry's team.
label_mirror_dummy_users(2, team_name, mattermost_data,
username_to_user)
user_handler = UserHandler()
convert_user_data(user_handler, user_id_mapper, username_to_user,
realm_id, team_name)
self.assert_length(user_handler.get_all_users(), 3)
self.assertTrue(user_id_mapper.has("snape"))
team_name = "slytherin"
user_handler = UserHandler()
convert_user_data(user_handler, user_id_mapper, username_to_user,
realm_id, team_name)
self.assert_length(user_handler.get_all_users(), 3)
def test_check_user_in_team(self) -> None:
fixture_file_name = self.fixture_file_name("export.json", "mattermost_fixtures")
mattermost_data = mattermost_data_file_to_dict(fixture_file_name)
username_to_user = create_username_to_user_mapping(mattermost_data["user"])
reset_mirror_dummy_users(username_to_user)
harry = username_to_user["harry"]
self.assertTrue(check_user_in_team(harry, "gryffindor"))
self.assertFalse(check_user_in_team(harry, "slytherin"))
snape = username_to_user["snape"]
self.assertFalse(check_user_in_team(snape, "gryffindor"))
self.assertTrue(check_user_in_team(snape, "slytherin"))
snape.update({"teams": None})
self.assertFalse(check_user_in_team(snape, "slytherin"))
def test_label_mirror_dummy_users(self) -> None:
fixture_file_name = self.fixture_file_name("export.json", "mattermost_fixtures")
mattermost_data = mattermost_data_file_to_dict(fixture_file_name)
username_to_user = create_username_to_user_mapping(mattermost_data["user"])
reset_mirror_dummy_users(username_to_user)
label_mirror_dummy_users(
num_teams=2,
team_name="gryffindor",
mattermost_data=mattermost_data,
username_to_user=username_to_user,
)
self.assertFalse(username_to_user["harry"]["is_mirror_dummy"])
self.assertFalse(username_to_user["ron"]["is_mirror_dummy"])
self.assertFalse(username_to_user["malfoy"]["is_mirror_dummy"])
# snape is mirror dummy since the user sent a message in gryffindor and
# left the team
self.assertTrue(username_to_user["snape"]["is_mirror_dummy"])
def test_convert_huddle_data(self) -> None:
fixture_file_name = self.fixture_file_name(
"export.json", "mattermost_fixtures/direct_channel")
mattermost_data = mattermost_data_file_to_dict(fixture_file_name)
username_to_user = create_username_to_user_mapping(
mattermost_data["user"])
reset_mirror_dummy_users(username_to_user)
user_handler = UserHandler()
subscriber_handler = SubscriberHandler()
huddle_id_mapper = IdMapper()
user_id_mapper = IdMapper()
team_name = "gryffindor"
convert_user_data(
user_handler=user_handler,
user_id_mapper=user_id_mapper,
user_data_map=username_to_user,
realm_id=3,
team_name=team_name,
)
zerver_huddle = convert_huddle_data(
huddle_data=mattermost_data["direct_channel"],
user_data_map=username_to_user,
subscriber_handler=subscriber_handler,
huddle_id_mapper=huddle_id_mapper,
user_id_mapper=user_id_mapper,
realm_id=3,
team_name=team_name,
)
self.assert_length(zerver_huddle, 1)
huddle_members = mattermost_data["direct_channel"][1]["members"]
huddle_name = generate_huddle_name(huddle_members)
self.assertTrue(huddle_id_mapper.has(huddle_name))
self.assertEqual(
subscriber_handler.get_users(
huddle_id=huddle_id_mapper.get(huddle_name)), {1, 2, 3})
def test_convert_channel_data(self) -> None:
fixture_file_name = self.fixture_file_name("export.json", "mattermost_fixtures")
mattermost_data = mattermost_data_file_to_dict(fixture_file_name)
username_to_user = create_username_to_user_mapping(mattermost_data["user"])
reset_mirror_dummy_users(username_to_user)
user_handler = UserHandler()
subscriber_handler = SubscriberHandler()
stream_id_mapper = IdMapper()
user_id_mapper = IdMapper()
team_name = "gryffindor"
convert_user_data(
user_handler=user_handler,
user_id_mapper=user_id_mapper,
user_data_map=username_to_user,
realm_id=3,
team_name=team_name,
)
zerver_stream = convert_channel_data(
channel_data=mattermost_data["channel"],
user_data_map=username_to_user,
subscriber_handler=subscriber_handler,
stream_id_mapper=stream_id_mapper,
user_id_mapper=user_id_mapper,
realm_id=3,
team_name=team_name,
)
self.assertEqual(len(zerver_stream), 3)
self.assertEqual(zerver_stream[0]["name"], "Gryffindor common room")
self.assertEqual(zerver_stream[0]["invite_only"], False)
self.assertEqual(zerver_stream[0]["description"], "A place for talking about Gryffindor common room")
self.assertEqual(zerver_stream[0]["rendered_description"], "")
self.assertEqual(zerver_stream[0]["realm"], 3)
self.assertEqual(zerver_stream[1]["name"], "Gryffindor quidditch team")
self.assertEqual(zerver_stream[1]["invite_only"], False)
self.assertEqual(zerver_stream[1]["description"], "A place for talking about Gryffindor quidditch team")
self.assertEqual(zerver_stream[1]["rendered_description"], "")
self.assertEqual(zerver_stream[1]["realm"], 3)
self.assertEqual(zerver_stream[2]["name"], "Dumbledores army")
self.assertEqual(zerver_stream[2]["invite_only"], True)
self.assertEqual(zerver_stream[2]["description"], "A place for talking about Dumbledores army")
self.assertEqual(zerver_stream[2]["rendered_description"], "")
self.assertEqual(zerver_stream[2]["realm"], 3)
self.assertTrue(stream_id_mapper.has("gryffindor-common-room"))
self.assertTrue(stream_id_mapper.has("gryffindor-quidditch-team"))
self.assertTrue(stream_id_mapper.has("dumbledores-army"))
# TODO: Add ginny
ron_id = user_id_mapper.get("ron")
harry_id = user_id_mapper.get("harry")
self.assertEqual({ron_id, harry_id}, {1, 2})
self.assertEqual(subscriber_handler.get_users(stream_id=stream_id_mapper.get("gryffindor-common-room")), {ron_id, harry_id})
self.assertEqual(subscriber_handler.get_users(stream_id=stream_id_mapper.get("gryffindor-quidditch-team")), {ron_id, harry_id})
self.assertEqual(subscriber_handler.get_users(stream_id=stream_id_mapper.get("dumbledores-army")), {ron_id, harry_id})
# Converting channel data when a user's `teams` value is `null`.
username_to_user["ron"].update({"teams": None})
zerver_stream = convert_channel_data(
channel_data=mattermost_data["channel"],
user_data_map=username_to_user,
subscriber_handler=subscriber_handler,
stream_id_mapper=stream_id_mapper,
user_id_mapper=user_id_mapper,
realm_id=3,
team_name=team_name,
)
harry_id = user_id_mapper.get("harry")
self.assertIn(harry_id, {1, 2})
self.assertEqual(subscriber_handler.get_users(stream_id=stream_id_mapper.get("gryffindor-common-room")), {harry_id})
self.assertEqual(subscriber_handler.get_users(stream_id=stream_id_mapper.get("gryffindor-quidditch-team")), {harry_id})
self.assertEqual(subscriber_handler.get_users(stream_id=stream_id_mapper.get("dumbledores-army")), {harry_id})
team_name = "slytherin"
zerver_stream = convert_channel_data(
channel_data=mattermost_data["channel"],
user_data_map=username_to_user,
subscriber_handler=subscriber_handler,
stream_id_mapper=stream_id_mapper,
user_id_mapper=user_id_mapper,
realm_id=4,
team_name=team_name,
)
malfoy_id = user_id_mapper.get("malfoy")
pansy_id = user_id_mapper.get("pansy")
snape_id = user_id_mapper.get("snape")
self.assertEqual({malfoy_id, pansy_id, snape_id}, {3, 4, 5})
self.assertEqual(subscriber_handler.get_users(stream_id=stream_id_mapper.get("slytherin-common-room")), {malfoy_id, pansy_id, snape_id})
self.assertEqual(subscriber_handler.get_users(stream_id=stream_id_mapper.get("slytherin-quidditch-team")), {malfoy_id, pansy_id})
def test_process_message_attachments(self) -> None:
mattermost_data_dir = self.fixture_file_name(
"", "mattermost_fixtures/direct_channel")
output_dir = self.make_import_output_dir("mattermost")
fixture_file_name = self.fixture_file_name(
"export.json", "mattermost_fixtures/direct_channel")
mattermost_data = mattermost_data_file_to_dict(fixture_file_name)
username_to_user = create_username_to_user_mapping(
mattermost_data["user"])
reset_mirror_dummy_users(username_to_user)
user_handler = UserHandler()
user_id_mapper = IdMapper()
team_name = "gryffindor"
convert_user_data(
user_handler=user_handler,
user_id_mapper=user_id_mapper,
user_data_map=username_to_user,
realm_id=3,
team_name=team_name,
)
zerver_attachments: List[ZerverFieldsT] = []
uploads_list: List[ZerverFieldsT] = []
process_message_attachments(
attachments=mattermost_data["post"]["direct_post"][0]
["attachments"],
realm_id=3,
message_id=1,
user_id=2,
user_handler=user_handler,
zerver_attachment=zerver_attachments,
uploads_list=uploads_list,
mattermost_data_dir=mattermost_data_dir,
output_dir=output_dir,
)
self.assert_length(zerver_attachments, 1)
self.assertEqual(zerver_attachments[0]["file_name"], "harry-ron.jpg")
self.assertEqual(zerver_attachments[0]["owner"], 2)
self.assertEqual(
user_handler.get_user(zerver_attachments[0]["owner"])["email"],
"*****@*****.**")
# TODO: Assert this for False after fixing the file permissions in PMs
self.assertTrue(zerver_attachments[0]["is_realm_public"])
self.assert_length(uploads_list, 1)
self.assertEqual(uploads_list[0]["user_profile_email"],
"*****@*****.**")
attachment_path = self.fixture_file_name(
mattermost_data["post"]["direct_post"][0]["attachments"][0]
["path"],
"mattermost_fixtures/direct_channel/data",
)
attachment_out_path = os.path.join(output_dir, "uploads",
zerver_attachments[0]["path_id"])
self.assertTrue(os.path.exists(attachment_out_path))
self.assertTrue(filecmp.cmp(attachment_path, attachment_out_path))
