def test_convert_user_data(self) -> None:
user_id_mapper = IdMapper()
realm_id = 3
fixture_file_name = self.fixture_file_name("export.json",
"mattermost_fixtures")
mattermost_data = mattermost_data_file_to_dict(fixture_file_name)
username_to_user = create_username_to_user_mapping(
mattermost_data["user"])
reset_mirror_dummy_users(username_to_user)
team_name = "gryffindor"
user_handler = UserHandler()
convert_user_data(user_handler, user_id_mapper, username_to_user,
realm_id, team_name)
self.assert_length(user_handler.get_all_users(), 2)
self.assertTrue(user_id_mapper.has("harry"))
self.assertTrue(user_id_mapper.has("ron"))
self.assertEqual(
user_handler.get_user(user_id_mapper.get("harry"))["full_name"],
"Harry Potter")
self.assertEqual(
user_handler.get_user(user_id_mapper.get("ron"))["full_name"],
"Ron Weasley")
team_name = "slytherin"
user_handler = UserHandler()
convert_user_data(user_handler, user_id_mapper, username_to_user,
realm_id, team_name)
self.assert_length(user_handler.get_all_users(), 3)
self.assertTrue(user_id_mapper.has("malfoy"))
self.assertTrue(user_id_mapper.has("pansy"))
self.assertTrue(user_id_mapper.has("snape"))
team_name = "gryffindor"
# Snape is a mirror dummy user in Harry's team.
label_mirror_dummy_users(2, team_name, mattermost_data,
username_to_user)
user_handler = UserHandler()
convert_user_data(user_handler, user_id_mapper, username_to_user,
realm_id, team_name)
self.assert_length(user_handler.get_all_users(), 3)
self.assertTrue(user_id_mapper.has("snape"))
team_name = "slytherin"
user_handler = UserHandler()
convert_user_data(user_handler, user_id_mapper, username_to_user,
realm_id, team_name)
self.assert_length(user_handler.get_all_users(), 3)
def test_process_message_attachment(self) -> None:
fixture_dir_name = self.fixture_file_name("", "rocketchat_fixtures")
rocketchat_data = rocketchat_data_to_dict(fixture_dir_name)
output_dir = self.make_import_output_dir("mattermost")
user_id_to_user_map = map_user_id_to_user(rocketchat_data["user"])
realm_id = 3
domain_name = "zulip.com"
user_handler = UserHandler()
user_id_mapper = IdMapper()
process_users(
user_id_to_user_map=user_id_to_user_map,
realm_id=realm_id,
domain_name=domain_name,
user_handler=user_handler,
user_id_mapper=user_id_mapper,
)
zerver_attachments: List[ZerverFieldsT] = []
uploads_list: List[ZerverFieldsT] = []
upload_id_to_upload_data_map = map_upload_id_to_upload_data(
rocketchat_data["upload"])
message_with_attachment = rocketchat_data["message"][55]
process_message_attachment(
upload=message_with_attachment["file"],
realm_id=3,
message_id=1,
user_id=3,
user_handler=user_handler,
zerver_attachment=zerver_attachments,
uploads_list=uploads_list,
upload_id_to_upload_data_map=upload_id_to_upload_data_map,
output_dir=output_dir,
)
self.assert_length(zerver_attachments, 1)
self.assertEqual(zerver_attachments[0]["file_name"], "harry-ron.jpg")
self.assertEqual(zerver_attachments[0]["owner"], 3)
self.assertEqual(
user_handler.get_user(zerver_attachments[0]["owner"])["email"],
"*****@*****.**")
# TODO: Assert this for False after fixing the file permissions in PMs
self.assertTrue(zerver_attachments[0]["is_realm_public"])
self.assert_length(uploads_list, 1)
self.assertEqual(uploads_list[0]["user_profile_email"],
"*****@*****.**")
attachment_out_path = os.path.join(output_dir, "uploads",
zerver_attachments[0]["path_id"])
self.assertTrue(os.path.exists(attachment_out_path))
self.assertTrue(os.path.isfile(attachment_out_path))
def process_message_attachments(
attachments: List[Dict[str, Any]],
realm_id: int,
message_id: int,
user_id: int,
user_handler: UserHandler,
zerver_attachment: List[ZerverFieldsT],
uploads_list: List[ZerverFieldsT],
mattermost_data_dir: str,
output_dir: str,
) -> Tuple[str, bool]:
has_image = False
markdown_links = []
for attachment in attachments:
attachment_path = attachment["path"]
attachment_full_path = os.path.join(mattermost_data_dir, "data",
attachment_path)
file_name = attachment_path.split("/")[-1]
file_ext = f'.{file_name.split(".")[-1]}'
if file_ext.lower() in IMAGE_EXTENSIONS:
has_image = True
s3_path = "/".join([
str(realm_id),
format(random.randint(0, 255), "x"),
secrets.token_urlsafe(18),
sanitize_name(file_name),
])
content_for_link = f"[{file_name}](/user_uploads/{s3_path})"
markdown_links.append(content_for_link)
fileinfo = {
"name": file_name,
"size": os.path.getsize(attachment_full_path),
"created": os.path.getmtime(attachment_full_path),
}
upload = dict(
path=s3_path,
realm_id=realm_id,
content_type=None,
user_profile_id=user_id,
last_modified=fileinfo["created"],
user_profile_email=user_handler.get_user(user_id=user_id)["email"],
s3_path=s3_path,
size=fileinfo["size"],
)
uploads_list.append(upload)
build_attachment(
realm_id=realm_id,
message_ids={message_id},
user_id=user_id,
fileinfo=fileinfo,
s3_path=s3_path,
zerver_attachment=zerver_attachment,
)
# Copy the attachment file to output_dir
attachment_out_path = os.path.join(output_dir, "uploads", s3_path)
os.makedirs(os.path.dirname(attachment_out_path), exist_ok=True)
shutil.copyfile(attachment_full_path, attachment_out_path)
content = "\n".join(markdown_links)
return content, has_image
def test_process_users(self) -> None:
fixture_dir_name = self.fixture_file_name("", "rocketchat_fixtures")
rocketchat_data = rocketchat_data_to_dict(fixture_dir_name)
user_id_to_user_map = map_user_id_to_user(rocketchat_data["user"])
realm_id = 3
domain_name = "zulip.com"
user_handler = UserHandler()
user_id_mapper = IdMapper()
process_users(
user_id_to_user_map=user_id_to_user_map,
realm_id=realm_id,
domain_name=domain_name,
user_handler=user_handler,
user_id_mapper=user_id_mapper,
)
self.assert_length(user_handler.get_all_users(), 6)
self.assertTrue(user_id_mapper.has(rocketchat_data["user"][0]["_id"]))
self.assertTrue(user_id_mapper.has(rocketchat_data["user"][4]["_id"]))
user_id = user_id_mapper.get(rocketchat_data["user"][0]["_id"])
user = user_handler.get_user(user_id)
self.assertEqual(user["full_name"], rocketchat_data["user"][0]["name"])
self.assertEqual(user["avatar_source"], "G")
self.assertEqual(user["delivery_email"], "*****@*****.**")
self.assertEqual(user["email"], "*****@*****.**")
self.assertEqual(user["full_name"], "Rocket.Cat")
self.assertEqual(user["id"], 1)
self.assertEqual(user["is_active"], False)
self.assertEqual(user["is_mirror_dummy"], False)
self.assertEqual(user["is_bot"], True)
self.assertEqual(user["bot_type"], 1)
self.assertEqual(user["bot_owner"], 2)
self.assertEqual(user["role"], UserProfile.ROLE_MEMBER)
self.assertEqual(user["realm"], realm_id)
self.assertEqual(user["short_name"], "rocket.cat")
self.assertEqual(user["timezone"], "UTC")
user_id = user_id_mapper.get(rocketchat_data["user"][2]["_id"])
user = user_handler.get_user(user_id)
self.assertEqual(user["full_name"], rocketchat_data["user"][2]["name"])
self.assertEqual(user["avatar_source"], "G")
self.assertEqual(user["delivery_email"], "*****@*****.**")
self.assertEqual(user["email"], "*****@*****.**")
self.assertEqual(user["full_name"], "Harry Potter")
self.assertEqual(user["id"], 3)
self.assertEqual(user["is_active"], True)
self.assertEqual(user["is_mirror_dummy"], False)
self.assertEqual(user["is_bot"], False)
self.assertEqual(user["bot_type"], None)
self.assertEqual(user["bot_owner"], None)
self.assertEqual(user["role"], UserProfile.ROLE_REALM_OWNER)
self.assertEqual(user["realm"], realm_id)
self.assertEqual(user["short_name"], "harry.potter")
self.assertEqual(user["timezone"], "UTC")
# Test `is_mirror_dummy` set for users of type `unknown`
rocketchat_data["user"].append({
"_id":
"s0m34ndmID",
"createdAt":
datetime.datetime(2019, 11, 6, 0, 38, 42, 796000),
"type":
"unknown",
"roles": ["unknown"],
"name":
"Unknown user",
"username":
"******",
})
user_id_to_user_map = map_user_id_to_user(rocketchat_data["user"])
process_users(
user_id_to_user_map=user_id_to_user_map,
realm_id=realm_id,
domain_name=domain_name,
user_handler=user_handler,
user_id_mapper=user_id_mapper,
)
self.assert_length(user_handler.get_all_users(), 7)
self.assertTrue(user_id_mapper.has(rocketchat_data["user"][6]["_id"]))
user_id = user_id_mapper.get(rocketchat_data["user"][6]["_id"])
user = user_handler.get_user(user_id)
self.assertEqual(user["id"], 7)
self.assertEqual(user["is_active"], False)
self.assertEqual(user["is_mirror_dummy"], True)
self.assertEqual(user["is_bot"], False)
def test_process_message_attachments(self) -> None:
mattermost_data_dir = self.fixture_file_name(
"", "mattermost_fixtures/direct_channel")
output_dir = self.make_import_output_dir("mattermost")
fixture_file_name = self.fixture_file_name(
"export.json", "mattermost_fixtures/direct_channel")
mattermost_data = mattermost_data_file_to_dict(fixture_file_name)
username_to_user = create_username_to_user_mapping(
mattermost_data["user"])
reset_mirror_dummy_users(username_to_user)
user_handler = UserHandler()
user_id_mapper = IdMapper()
team_name = "gryffindor"
convert_user_data(
user_handler=user_handler,
user_id_mapper=user_id_mapper,
user_data_map=username_to_user,
realm_id=3,
team_name=team_name,
)
zerver_attachments: List[ZerverFieldsT] = []
uploads_list: List[ZerverFieldsT] = []
process_message_attachments(
attachments=mattermost_data["post"]["direct_post"][0]
["attachments"],
realm_id=3,
message_id=1,
user_id=2,
user_handler=user_handler,
zerver_attachment=zerver_attachments,
uploads_list=uploads_list,
mattermost_data_dir=mattermost_data_dir,
output_dir=output_dir,
)
self.assert_length(zerver_attachments, 1)
self.assertEqual(zerver_attachments[0]["file_name"], "harry-ron.jpg")
self.assertEqual(zerver_attachments[0]["owner"], 2)
self.assertEqual(
user_handler.get_user(zerver_attachments[0]["owner"])["email"],
"*****@*****.**")
# TODO: Assert this for False after fixing the file permissions in PMs
self.assertTrue(zerver_attachments[0]["is_realm_public"])
self.assert_length(uploads_list, 1)
self.assertEqual(uploads_list[0]["user_profile_email"],
"*****@*****.**")
attachment_path = self.fixture_file_name(
mattermost_data["post"]["direct_post"][0]["attachments"][0]
["path"],
"mattermost_fixtures/direct_channel/data",
)
attachment_out_path = os.path.join(output_dir, "uploads",
zerver_attachments[0]["path_id"])
self.assertTrue(os.path.exists(attachment_out_path))
self.assertTrue(filecmp.cmp(attachment_path, attachment_out_path))
def process_users(
user_id_to_user_map: Dict[str, Dict[str, Any]],
realm_id: int,
domain_name: str,
user_handler: UserHandler,
user_id_mapper: IdMapper,
) -> None:
realm_owners: List[int] = []
bots: List[int] = []
for rc_user_id in user_id_to_user_map:
user_dict = user_id_to_user_map[rc_user_id]
is_mirror_dummy = False
is_bot = False
is_active = True
# Rocket.Chat has three user types:
# "user": This is a regular user of the system.
# "bot": A special user types for bots.
# "unknown": This usually represents a livechat guest.
if user_dict["type"] != "user":
is_active = False
if user_dict["type"] == "bot":
is_bot = True
else:
is_mirror_dummy = True
if not user_dict.get("emails"):
user_dict["emails"] = [{
"address":
"{}-{}@{}".format(user_dict["username"], user_dict["type"],
domain_name)
}]
# TODO: Change this to use actual exported avatar
avatar_source = "G"
full_name = user_dict["name"]
id = user_id_mapper.get(rc_user_id)
delivery_email = user_dict["emails"][0]["address"]
email = user_dict["emails"][0]["address"]
short_name = user_dict["username"]
date_joined = float(user_dict["createdAt"].timestamp())
timezone = "UTC"
role = UserProfile.ROLE_MEMBER
if "admin" in user_dict["roles"]:
role = UserProfile.ROLE_REALM_OWNER
realm_owners.append(id)
elif "guest" in user_dict["roles"]:
role = UserProfile.ROLE_GUEST
if is_bot:
bots.append(id)
user = build_user_profile(
avatar_source=avatar_source,
date_joined=date_joined,
delivery_email=delivery_email,
email=email,
full_name=full_name,
id=id,
is_active=is_active,
role=role,
is_mirror_dummy=is_mirror_dummy,
realm_id=realm_id,
short_name=short_name,
timezone=timezone,
is_bot=is_bot,
bot_type=1 if is_bot else None,
)
user_handler.add_user(user)
# Set the first realm_owner as the owner of
# all the bots.
if realm_owners:
for bot_id in bots:
bot_user = user_handler.get_user(user_id=bot_id)
bot_user["bot_owner"] = realm_owners[0]
def process_message_attachment(
upload: Dict[str, Any],
realm_id: int,
message_id: int,
user_id: int,
user_handler: UserHandler,
zerver_attachment: List[ZerverFieldsT],
uploads_list: List[ZerverFieldsT],
upload_id_to_upload_data_map: Dict[str, Dict[str, Any]],
output_dir: str,
) -> Tuple[str, bool]:
upload_file_data = upload_id_to_upload_data_map[upload["_id"]]
file_name = upload["name"]
file_ext = f'.{upload["type"].split("/")[-1]}'
has_image = False
if file_ext.lower() in IMAGE_EXTENSIONS:
has_image = True
s3_path = "/".join([
str(realm_id),
format(random.randint(0, 255), "x"),
secrets.token_urlsafe(18),
sanitize_name(file_name),
])
# Build the attachment from chunks and save it to s3_path.
file_out_path = os.path.join(output_dir, "uploads", s3_path)
os.makedirs(os.path.dirname(file_out_path), exist_ok=True)
with open(file_out_path, "wb") as upload_file:
upload_file.write(b"".join(upload_file_data["chunk"]))
attachment_content = (
f'{upload_file_data["description"]}\n\n[{file_name}](/user_uploads/{s3_path})'
)
fileinfo = {
"name": file_name,
"size": upload_file_data["size"],
"created": float(upload_file_data["_updatedAt"].timestamp()),
}
upload = dict(
path=s3_path,
realm_id=realm_id,
content_type=upload["type"],
user_profile_id=user_id,
last_modified=fileinfo["created"],
user_profile_email=user_handler.get_user(user_id=user_id)["email"],
s3_path=s3_path,
size=fileinfo["size"],
)
uploads_list.append(upload)
build_attachment(
realm_id=realm_id,
message_ids={message_id},
user_id=user_id,
fileinfo=fileinfo,
s3_path=s3_path,
zerver_attachment=zerver_attachment,
)
return attachment_content, has_image
