def test_convert_user_data(self) -> None:
user_id_mapper = IdMapper()
realm_id = 3
team_name = "gryffindor"
user_handler = UserHandler()
convert_user_data(user_handler, user_id_mapper, self.username_to_user, realm_id, team_name)
self.assertTrue(user_id_mapper.has("harry"))
self.assertTrue(user_id_mapper.has("ron"))
self.assertEqual(user_handler.get_user(user_id_mapper.get("harry"))["full_name"], "Harry Potter")
self.assertEqual(user_handler.get_user(user_id_mapper.get("ron"))["full_name"], "Ron Weasley")
team_name = "slytherin"
user_handler = UserHandler()
convert_user_data(user_handler, user_id_mapper, self.username_to_user, realm_id, team_name)
self.assertEqual(len(user_handler.get_all_users()), 3)
self.assertTrue(user_id_mapper.has("malfoy"))
self.assertTrue(user_id_mapper.has("pansy"))
self.assertTrue(user_id_mapper.has("snape"))
team_name = "gryffindor"
# Snape is a mirror dummy user in Harry's team.
label_mirror_dummy_users(team_name, self.mattermost_data, self.username_to_user)
user_handler = UserHandler()
convert_user_data(user_handler, user_id_mapper, self.username_to_user, realm_id, team_name)
self.assertEqual(len(user_handler.get_all_users()), 3)
self.assertTrue(user_id_mapper.has("snape"))
team_name = "slytherin"
user_handler = UserHandler()
convert_user_data(user_handler, user_id_mapper, self.username_to_user, realm_id, team_name)
self.assertEqual(len(user_handler.get_all_users()), 3)
def test_convert_user_data(self) -> None:
user_id_mapper = IdMapper()
realm_id = 3
fixture_file_name = self.fixture_file_name("export.json",
"mattermost_fixtures")
mattermost_data = mattermost_data_file_to_dict(fixture_file_name)
username_to_user = create_username_to_user_mapping(
mattermost_data["user"])
reset_mirror_dummy_users(username_to_user)
team_name = "gryffindor"
user_handler = UserHandler()
convert_user_data(user_handler, user_id_mapper, username_to_user,
realm_id, team_name)
self.assertEqual(len(user_handler.get_all_users()), 2)
self.assertTrue(user_id_mapper.has("harry"))
self.assertTrue(user_id_mapper.has("ron"))
self.assertEqual(
user_handler.get_user(user_id_mapper.get("harry"))["full_name"],
"Harry Potter")
self.assertEqual(
user_handler.get_user(user_id_mapper.get("ron"))["full_name"],
"Ron Weasley")
team_name = "slytherin"
user_handler = UserHandler()
convert_user_data(user_handler, user_id_mapper, username_to_user,
realm_id, team_name)
self.assertEqual(len(user_handler.get_all_users()), 3)
self.assertTrue(user_id_mapper.has("malfoy"))
self.assertTrue(user_id_mapper.has("pansy"))
self.assertTrue(user_id_mapper.has("snape"))
team_name = "gryffindor"
# Snape is a mirror dummy user in Harry's team.
label_mirror_dummy_users(2, team_name, mattermost_data,
username_to_user)
user_handler = UserHandler()
convert_user_data(user_handler, user_id_mapper, username_to_user,
realm_id, team_name)
self.assertEqual(len(user_handler.get_all_users()), 3)
self.assertTrue(user_id_mapper.has("snape"))
team_name = "slytherin"
user_handler = UserHandler()
convert_user_data(user_handler, user_id_mapper, username_to_user,
realm_id, team_name)
self.assertEqual(len(user_handler.get_all_users()), 3)
def do_convert_data(mattermost_data_dir: str, output_dir: str, masking_content: bool) -> None:
username_to_user: Dict[str, Dict[str, Any]] = {}
os.makedirs(output_dir, exist_ok=True)
if os.listdir(output_dir): # nocoverage
raise Exception("Output directory should be empty!")
mattermost_data_file = os.path.join(mattermost_data_dir, "export.json")
mattermost_data = mattermost_data_file_to_dict(mattermost_data_file)
username_to_user = create_username_to_user_mapping(mattermost_data["user"])
for team in mattermost_data["team"]:
realm_id = NEXT_ID("realm_id")
team_name = team["name"]
user_handler = UserHandler()
subscriber_handler = SubscriberHandler()
user_id_mapper = IdMapper()
stream_id_mapper = IdMapper()
huddle_id_mapper = IdMapper()
print("Generating data for", team_name)
realm = make_realm(realm_id, team)
realm_output_dir = os.path.join(output_dir, team_name)
reset_mirror_dummy_users(username_to_user)
label_mirror_dummy_users(
len(mattermost_data["team"]), team_name, mattermost_data, username_to_user
)
convert_user_data(
user_handler=user_handler,
user_id_mapper=user_id_mapper,
user_data_map=username_to_user,
realm_id=realm_id,
team_name=team_name,
)
zerver_stream = convert_channel_data(
channel_data=mattermost_data["channel"],
user_data_map=username_to_user,
subscriber_handler=subscriber_handler,
stream_id_mapper=stream_id_mapper,
user_id_mapper=user_id_mapper,
realm_id=realm_id,
team_name=team_name,
)
realm["zerver_stream"] = zerver_stream
zerver_huddle: List[ZerverFieldsT] = []
if len(mattermost_data["team"]) == 1:
zerver_huddle = convert_huddle_data(
huddle_data=mattermost_data["direct_channel"],
user_data_map=username_to_user,
subscriber_handler=subscriber_handler,
huddle_id_mapper=huddle_id_mapper,
user_id_mapper=user_id_mapper,
realm_id=realm_id,
team_name=team_name,
)
realm["zerver_huddle"] = zerver_huddle
all_users = user_handler.get_all_users()
zerver_recipient = build_recipients(
zerver_userprofile=all_users,
zerver_stream=zerver_stream,
zerver_huddle=zerver_huddle,
)
realm["zerver_recipient"] = zerver_recipient
stream_subscriptions = build_stream_subscriptions(
get_users=subscriber_handler.get_users,
zerver_recipient=zerver_recipient,
zerver_stream=zerver_stream,
)
huddle_subscriptions = build_huddle_subscriptions(
get_users=subscriber_handler.get_users,
zerver_recipient=zerver_recipient,
zerver_huddle=zerver_huddle,
)
personal_subscriptions = build_personal_subscriptions(
zerver_recipient=zerver_recipient,
)
# Mattermost currently supports only exporting messages from channels.
# Personal messages and huddles are not exported.
zerver_subscription = personal_subscriptions + stream_subscriptions + huddle_subscriptions
realm["zerver_subscription"] = zerver_subscription
zerver_realmemoji = write_emoticon_data(
realm_id=realm_id,
custom_emoji_data=mattermost_data["emoji"],
data_dir=mattermost_data_dir,
output_dir=realm_output_dir,
)
realm["zerver_realmemoji"] = zerver_realmemoji
subscriber_map = make_subscriber_map(
zerver_subscription=zerver_subscription,
)
total_reactions: List[Dict[str, Any]] = []
uploads_list: List[ZerverFieldsT] = []
zerver_attachment: List[ZerverFieldsT] = []
write_message_data(
num_teams=len(mattermost_data["team"]),
team_name=team_name,
realm_id=realm_id,
post_data=mattermost_data["post"],
zerver_recipient=zerver_recipient,
subscriber_map=subscriber_map,
output_dir=realm_output_dir,
masking_content=masking_content,
stream_id_mapper=stream_id_mapper,
huddle_id_mapper=huddle_id_mapper,
user_id_mapper=user_id_mapper,
user_handler=user_handler,
zerver_realmemoji=zerver_realmemoji,
total_reactions=total_reactions,
uploads_list=uploads_list,
zerver_attachment=zerver_attachment,
mattermost_data_dir=mattermost_data_dir,
)
realm["zerver_reaction"] = total_reactions
realm["zerver_userprofile"] = user_handler.get_all_users()
realm["sort_by_date"] = True
create_converted_data_files(realm, realm_output_dir, "/realm.json")
# Mattermost currently doesn't support exporting avatars
create_converted_data_files([], realm_output_dir, "/avatars/records.json")
# Export message attachments
attachment: Dict[str, List[Any]] = {"zerver_attachment": zerver_attachment}
create_converted_data_files(uploads_list, realm_output_dir, "/uploads/records.json")
create_converted_data_files(attachment, realm_output_dir, "/attachment.json")
logging.info("Start making tarball")
subprocess.check_call(["tar", "-czf", realm_output_dir + ".tar.gz", realm_output_dir, "-P"])
logging.info("Done making tarball")