def convert_huddle_data(
huddle_data: List[ZerverFieldsT],
user_data_map: Dict[str, Dict[str, Any]],
subscriber_handler: SubscriberHandler,
huddle_id_mapper: IdMapper,
user_id_mapper: IdMapper,
realm_id: int,
team_name: str,
) -> List[ZerverFieldsT]:
zerver_huddle = []
for huddle in huddle_data:
if len(huddle["members"]) > 2:
huddle_name = generate_huddle_name(huddle["members"])
huddle_id = huddle_id_mapper.get(huddle_name)
huddle_dict = build_huddle(huddle_id)
huddle_user_ids = set()
for username in huddle["members"]:
huddle_user_ids.add(user_id_mapper.get(username))
subscriber_handler.set_info(
users=huddle_user_ids,
huddle_id=huddle_id,
)
zerver_huddle.append(huddle_dict)
return zerver_huddle
def convert_stream_subscription_data(
user_id_to_user_map: Dict[str, Dict[str, Any]],
dsc_id_to_dsc_map: Dict[str, Dict[str, Any]],
zerver_stream: List[ZerverFieldsT],
stream_id_mapper: IdMapper,
user_id_mapper: IdMapper,
subscriber_handler: SubscriberHandler,
) -> None:
stream_members_map: Dict[int, Set[int]] = {}
for rc_user_id in user_id_to_user_map:
user_dict = user_id_to_user_map[rc_user_id]
if not user_dict.get("__rooms"):
continue
for channel in user_dict["__rooms"]:
if channel in dsc_id_to_dsc_map:
# Ignore discussion rooms as these are not
# imported as streams, but topics.
continue
stream_id = stream_id_mapper.get(channel)
if stream_id not in stream_members_map:
stream_members_map[stream_id] = set()
stream_members_map[stream_id].add(user_id_mapper.get(rc_user_id))
for stream in zerver_stream:
if stream["id"] in stream_members_map:
users = stream_members_map[stream["id"]]
else:
users = set()
# Set the stream without any subscribers
# as deactivated.
stream["deactivated"] = True
subscriber_handler.set_info(users=users, stream_id=stream["id"])
def test_build_reactions(self) -> None:
total_reactions = [] # type: List[Dict[str, Any]]
reactions = [{
"user": "******",
"create_at": 1553165521410,
"emoji_name": "tick"
}, {
"user": "******",
"create_at": 1553166530805,
"emoji_name": "smile"
}, {
"user": "******",
"create_at": 1553166540953,
"emoji_name": "world_map"
}, {
"user": "******",
"create_at": 1553166540957,
"emoji_name": "world_map"
}]
zerver_realmemoji = write_emoticon_data(
realm_id=3,
custom_emoji_data=self.mattermost_data["emoji"],
data_dir=self.fixture_file_name("", "mattermost_fixtures"),
output_dir=self.make_import_output_dir("mattermost"))
# Make sure tick is present in fixture data
self.assertEqual(zerver_realmemoji[1]["name"], "tick")
tick_emoji_code = zerver_realmemoji[1]["id"]
name_to_codepoint = get_name_to_codepoint_dict()
user_id_mapper = IdMapper()
harry_id = user_id_mapper.get("harry")
ron_id = user_id_mapper.get("ron")
build_reactions(realm_id=3,
total_reactions=total_reactions,
reactions=reactions,
message_id=5,
name_to_codepoint=name_to_codepoint,
user_id_mapper=user_id_mapper,
zerver_realmemoji=zerver_realmemoji)
smile_emoji_code = name_to_codepoint["smile"]
world_map_emoji_code = name_to_codepoint["world_map"]
self.assertEqual(len(total_reactions), 4)
self.assertEqual(self.get_set(total_reactions, "reaction_type"),
set([Reaction.REALM_EMOJI, Reaction.UNICODE_EMOJI]))
self.assertEqual(self.get_set(total_reactions, "emoji_name"),
set(["tick", "smile", "world_map"]))
self.assertEqual(
self.get_set(total_reactions, "emoji_code"),
set([tick_emoji_code, smile_emoji_code, world_map_emoji_code]))
self.assertEqual(self.get_set(total_reactions, "user_profile"),
set([harry_id, ron_id]))
self.assertEqual(len(self.get_set(total_reactions, "id")), 4)
self.assertEqual(len(self.get_set(total_reactions, "message")), 1)
def test_build_reactions(self) -> None:
total_reactions = [] # type: List[Dict[str, Any]]
reactions = [
{"user": "******", "create_at": 1553165521410, "emoji_name": "tick"},
{"user": "******", "create_at": 1553166530805, "emoji_name": "smile"},
{"user": "******", "create_at": 1553166540953, "emoji_name": "world_map"},
{"user": "******", "create_at": 1553166540957, "emoji_name": "world_map"}
]
zerver_realmemoji = write_emoticon_data(
realm_id=3,
custom_emoji_data=self.mattermost_data["emoji"],
data_dir=self.fixture_file_name("", "mattermost_fixtures"),
output_dir=self.make_import_output_dir("mattermost")
)
# Make sure tick is present in fixture data
self.assertEqual(zerver_realmemoji[1]["name"], "tick")
tick_emoji_code = zerver_realmemoji[1]["id"]
name_to_codepoint = get_name_to_codepoint_dict()
user_id_mapper = IdMapper()
harry_id = user_id_mapper.get("harry")
ron_id = user_id_mapper.get("ron")
build_reactions(
realm_id=3,
total_reactions=total_reactions,
reactions=reactions,
message_id=5,
name_to_codepoint=name_to_codepoint,
user_id_mapper=user_id_mapper,
zerver_realmemoji=zerver_realmemoji
)
smile_emoji_code = name_to_codepoint["smile"]
world_map_emoji_code = name_to_codepoint["world_map"]
expected_total_reactions = [
{
'user_profile': harry_id, 'message': 5, 'id': 1, 'reaction_type': Reaction.REALM_EMOJI,
'emoji_code': tick_emoji_code, 'emoji_name': 'tick'
},
{
'user_profile': ron_id, 'message': 5, 'id': 2, 'reaction_type': Reaction.UNICODE_EMOJI,
'emoji_code': smile_emoji_code, 'emoji_name': 'smile'
},
{
'user_profile': ron_id, 'message': 5, 'id': 3, 'reaction_type': Reaction.UNICODE_EMOJI,
'emoji_code': world_map_emoji_code, 'emoji_name': 'world_map'
},
{
'user_profile': harry_id, 'message': 5, 'id': 4, 'reaction_type': Reaction.UNICODE_EMOJI,
'emoji_code': world_map_emoji_code, 'emoji_name': 'world_map'
}
]
self.assertEqual(total_reactions, expected_total_reactions)
def convert_room_data(raw_data: List[ZerverFieldsT],
subscriber_handler: SubscriberHandler,
stream_id_mapper: IdMapper, user_id_mapper: IdMapper,
realm_id: int) -> List[ZerverFieldsT]:
flat_data = [d['Room'] for d in raw_data]
def get_invite_only(v: str) -> bool:
if v == 'public':
return False
elif v == 'private':
return True
else:
raise Exception('unexpected value')
streams = []
for in_dict in flat_data:
now = int(timezone_now().timestamp())
stream_id = stream_id_mapper.get(in_dict['id'])
invite_only = get_invite_only(in_dict['privacy'])
stream = build_stream(
date_created=now,
realm_id=realm_id,
name=in_dict['name'],
description=in_dict['topic'],
stream_id=stream_id,
deactivated=in_dict['is_archived'],
invite_only=invite_only,
)
if invite_only:
users = {
user_id_mapper.get(key)
for key in in_dict['members'] if user_id_mapper.has(key)
} # type: Set[int]
if user_id_mapper.has(in_dict['owner']):
owner = user_id_mapper.get(in_dict['owner'])
users.add(owner)
if not users:
continue
subscriber_handler.set_info(
stream_id=stream_id,
users=users,
)
# unmapped fields:
# guest_access_url: no Zulip equivalent
# created: we just use "now"
# participants: no good sample data
streams.append(stream)
return streams
def test_sender_ids(self) -> None:
realm_id = 5
user_handler = UserHandler()
user_id_mapper = IdMapper()
user_id_mapper.has = lambda key: True # type: ignore[assignment] # it's just a stub
# Simulate a "normal" user first.
user_with_id = dict(id=1,
# other fields don't matter here
)
user_handler.add_user(user=user_with_id)
normal_message: Dict[str, Any] = dict(sender=dict(id=1, ), )
sender_id = get_hipchat_sender_id(
realm_id=realm_id,
slim_mode=False,
message_dict=normal_message,
user_id_mapper=user_id_mapper,
user_handler=user_handler,
)
self.assertEqual(sender_id, 1)
bot_message = dict(sender='fred_bot', )
# Every message from fred_bot should
# return the same sender_id.
fred_bot_sender_id = 2
for i in range(3):
sender_id = get_hipchat_sender_id(
realm_id=realm_id,
slim_mode=False,
message_dict=bot_message,
user_id_mapper=user_id_mapper,
user_handler=user_handler,
)
self.assertEqual(sender_id, fred_bot_sender_id)
id_zero_message = dict(sender=dict(
id=0,
name='hal_bot',
), )
hal_bot_sender_id = 3
for i in range(3):
sender_id = get_hipchat_sender_id(
realm_id=realm_id,
slim_mode=False,
message_dict=id_zero_message,
user_id_mapper=user_id_mapper,
user_handler=user_handler,
)
self.assertEqual(sender_id, hal_bot_sender_id)
def test_convert_user_data(self) -> None:
user_id_mapper = IdMapper()
realm_id = 3
team_name = "gryffindor"
user_handler = UserHandler()
convert_user_data(user_handler, user_id_mapper, self.username_to_user, realm_id, team_name)
self.assertTrue(user_id_mapper.has("harry"))
self.assertTrue(user_id_mapper.has("ron"))
self.assertEqual(user_handler.get_user(user_id_mapper.get("harry"))["full_name"], "Harry Potter")
self.assertEqual(user_handler.get_user(user_id_mapper.get("ron"))["full_name"], "Ron Weasley")
team_name = "slytherin"
user_handler = UserHandler()
convert_user_data(user_handler, user_id_mapper, self.username_to_user, realm_id, team_name)
self.assertEqual(len(user_handler.get_all_users()), 3)
self.assertTrue(user_id_mapper.has("malfoy"))
self.assertTrue(user_id_mapper.has("pansy"))
self.assertTrue(user_id_mapper.has("snape"))
team_name = "gryffindor"
# Snape is a mirror dummy user in Harry's team.
label_mirror_dummy_users(team_name, self.mattermost_data, self.username_to_user)
user_handler = UserHandler()
convert_user_data(user_handler, user_id_mapper, self.username_to_user, realm_id, team_name)
self.assertEqual(len(user_handler.get_all_users()), 3)
self.assertTrue(user_id_mapper.has("snape"))
team_name = "slytherin"
user_handler = UserHandler()
convert_user_data(user_handler, user_id_mapper, self.username_to_user, realm_id, team_name)
self.assertEqual(len(user_handler.get_all_users()), 3)
def test_convert_huddle_data(self) -> None:
fixture_dir_name = self.fixture_file_name("", "rocketchat_fixtures")
rocketchat_data = rocketchat_data_to_dict(fixture_dir_name)
realm_id = 3
domain_name = "zulip.com"
user_handler = UserHandler()
subscriber_handler = SubscriberHandler()
user_id_mapper = IdMapper()
huddle_id_mapper = IdMapper()
user_id_to_user_map = map_user_id_to_user(rocketchat_data["user"])
process_users(
user_id_to_user_map=user_id_to_user_map,
realm_id=realm_id,
domain_name=domain_name,
user_handler=user_handler,
user_id_mapper=user_id_mapper,
)
room_id_to_room_map: Dict[str, Dict[str, Any]] = {}
team_id_to_team_map: Dict[str, Dict[str, Any]] = {}
dsc_id_to_dsc_map: Dict[str, Dict[str, Any]] = {}
direct_id_to_direct_map: Dict[str, Dict[str, Any]] = {}
huddle_id_to_huddle_map: Dict[str, Dict[str, Any]] = {}
livechat_id_to_livechat_map: Dict[str, Dict[str, Any]] = {}
categorize_channels_and_map_with_id(
channel_data=rocketchat_data["room"],
room_id_to_room_map=room_id_to_room_map,
team_id_to_team_map=team_id_to_team_map,
dsc_id_to_dsc_map=dsc_id_to_dsc_map,
direct_id_to_direct_map=direct_id_to_direct_map,
huddle_id_to_huddle_map=huddle_id_to_huddle_map,
livechat_id_to_livechat_map=livechat_id_to_livechat_map,
)
zerver_huddle = convert_huddle_data(
huddle_id_to_huddle_map=huddle_id_to_huddle_map,
huddle_id_mapper=huddle_id_mapper,
user_id_mapper=user_id_mapper,
subscriber_handler=subscriber_handler,
)
self.assert_length(zerver_huddle, 1)
rc_huddle_id = rocketchat_data["room"][12]["_id"]
self.assertTrue(huddle_id_mapper.has(rc_huddle_id))
huddle_id = huddle_id_mapper.get(rc_huddle_id)
self.assertEqual(subscriber_handler.get_users(huddle_id=huddle_id),
{3, 4, 5})
def get_mentioned_user_ids(raw_message: Dict[str, Any], user_id_mapper: IdMapper) -> Set[int]:
user_ids = set()
content = raw_message["content"]
# usernames can be of the form user.name, user_name, username., username_, user.name_ etc
matches = re.findall("(?<=^|(?<=[^a-zA-Z0-9-_.]))@(([A-Za-z0-9]+[_.]?)+)", content)
for match in matches:
possible_username = match[0]
if user_id_mapper.has(possible_username):
user_ids.add(user_id_mapper.get(possible_username))
return user_ids
def test_build_reactions(self) -> None:
fixture_file_name = self.fixture_file_name("export.json", "mattermost_fixtures")
mattermost_data = mattermost_data_file_to_dict(fixture_file_name)
total_reactions: List[Dict[str, Any]] = []
reactions = [
{"user": "******", "create_at": 1553165521410, "emoji_name": "tick"},
{"user": "******", "create_at": 1553166530805, "emoji_name": "smile"},
{"user": "******", "create_at": 1553166540953, "emoji_name": "world_map"},
{"user": "******", "create_at": 1553166540957, "emoji_name": "world_map"},
]
with self.assertLogs(level="INFO"):
zerver_realmemoji = write_emoticon_data(
realm_id=3,
custom_emoji_data=mattermost_data["emoji"],
data_dir=self.fixture_file_name("", "mattermost_fixtures"),
output_dir=self.make_import_output_dir("mattermost"),
)
# Make sure tick is present in fixture data
self.assertEqual(zerver_realmemoji[1]["name"], "tick")
tick_emoji_code = zerver_realmemoji[1]["id"]
user_id_mapper = IdMapper()
harry_id = user_id_mapper.get("harry")
ron_id = user_id_mapper.get("ron")
build_reactions(
realm_id=3,
total_reactions=total_reactions,
reactions=reactions,
message_id=5,
user_id_mapper=user_id_mapper,
zerver_realmemoji=zerver_realmemoji,
)
smile_emoji_code = name_to_codepoint["smile"]
world_map_emoji_code = name_to_codepoint["world_map"]
self.assertEqual(len(total_reactions), 4)
self.assertEqual(self.get_set(total_reactions, "reaction_type"), {Reaction.REALM_EMOJI, Reaction.UNICODE_EMOJI})
self.assertEqual(self.get_set(total_reactions, "emoji_name"), {"tick", "smile", "world_map"})
self.assertEqual(self.get_set(total_reactions, "emoji_code"), {tick_emoji_code, smile_emoji_code,
world_map_emoji_code})
self.assertEqual(self.get_set(total_reactions, "user_profile"), {harry_id, ron_id})
self.assertEqual(len(self.get_set(total_reactions, "id")), 4)
self.assertEqual(len(self.get_set(total_reactions, "message")), 1)
def get_hipchat_sender_id(realm_id: int,
slim_mode: bool,
message_dict: Dict[str, Any],
user_id_mapper: IdMapper,
user_handler: UserHandler) -> Optional[int]:
'''
The HipChat export is inconsistent in how it renders
senders, and sometimes we don't even get an id.
'''
if isinstance(message_dict['sender'], str):
if slim_mode:
return None
# Some Hipchat instances just give us a person's
# name in the sender field for NotificationMessage.
# We turn them into a mirror user.
mirror_user = user_handler.get_mirror_user(
realm_id=realm_id,
name=message_dict['sender'],
)
sender_id = mirror_user['id']
return sender_id
raw_sender_id = message_dict['sender']['id']
if raw_sender_id == 0:
if slim_mode:
return None
mirror_user = user_handler.get_mirror_user(
realm_id=realm_id,
name=message_dict['sender']['name']
)
sender_id = mirror_user['id']
return sender_id
if not user_id_mapper.has(raw_sender_id):
if slim_mode:
return None
mirror_user = user_handler.get_mirror_user(
realm_id=realm_id,
name=message_dict['sender']['id']
)
sender_id = mirror_user['id']
return sender_id
# HAPPY PATH: Hipchat just gave us an ordinary
# sender_id.
sender_id = user_id_mapper.get(raw_sender_id)
return sender_id
def test_process_message_attachment(self) -> None:
fixture_dir_name = self.fixture_file_name("", "rocketchat_fixtures")
rocketchat_data = rocketchat_data_to_dict(fixture_dir_name)
output_dir = self.make_import_output_dir("mattermost")
user_id_to_user_map = map_user_id_to_user(rocketchat_data["user"])
realm_id = 3
domain_name = "zulip.com"
user_handler = UserHandler()
user_id_mapper = IdMapper()
process_users(
user_id_to_user_map=user_id_to_user_map,
realm_id=realm_id,
domain_name=domain_name,
user_handler=user_handler,
user_id_mapper=user_id_mapper,
)
zerver_attachments: List[ZerverFieldsT] = []
uploads_list: List[ZerverFieldsT] = []
upload_id_to_upload_data_map = map_upload_id_to_upload_data(
rocketchat_data["upload"])
message_with_attachment = rocketchat_data["message"][55]
process_message_attachment(
upload=message_with_attachment["file"],
realm_id=3,
message_id=1,
user_id=3,
user_handler=user_handler,
zerver_attachment=zerver_attachments,
uploads_list=uploads_list,
upload_id_to_upload_data_map=upload_id_to_upload_data_map,
output_dir=output_dir,
)
self.assert_length(zerver_attachments, 1)
self.assertEqual(zerver_attachments[0]["file_name"], "harry-ron.jpg")
self.assertEqual(zerver_attachments[0]["owner"], 3)
self.assertEqual(
user_handler.get_user(zerver_attachments[0]["owner"])["email"],
"*****@*****.**")
# TODO: Assert this for False after fixing the file permissions in PMs
self.assertTrue(zerver_attachments[0]["is_realm_public"])
self.assert_length(uploads_list, 1)
self.assertEqual(uploads_list[0]["user_profile_email"],
"*****@*****.**")
attachment_out_path = os.path.join(output_dir, "uploads",
zerver_attachments[0]["path_id"])
self.assertTrue(os.path.exists(attachment_out_path))
self.assertTrue(os.path.isfile(attachment_out_path))
def build_reactions(
realm_id: int,
total_reactions: List[ZerverFieldsT],
reactions: List[ZerverFieldsT],
message_id: int,
user_id_mapper: IdMapper,
zerver_realmemoji: List[ZerverFieldsT],
) -> None:
realmemoji = {}
for realm_emoji in zerver_realmemoji:
realmemoji[realm_emoji["name"]] = realm_emoji["id"]
# For the Unicode emoji codes, we use equivalent of
# function 'emoji_name_to_emoji_code' in 'zerver/lib/emoji' here
for mattermost_reaction in reactions:
emoji_name = mattermost_reaction["emoji_name"]
username = mattermost_reaction["user"]
# Check in Unicode emoji
if emoji_name in name_to_codepoint:
emoji_code = name_to_codepoint[emoji_name]
reaction_type = Reaction.UNICODE_EMOJI
# Check in realm emoji
elif emoji_name in realmemoji:
emoji_code = realmemoji[emoji_name]
reaction_type = Reaction.REALM_EMOJI
else: # nocoverage
continue
if not user_id_mapper.has(username):
continue
reaction_id = NEXT_ID("reaction")
reaction = Reaction(
id=reaction_id,
emoji_code=emoji_code,
emoji_name=emoji_name,
reaction_type=reaction_type,
)
reaction_dict = model_to_dict(reaction,
exclude=["message", "user_profile"])
reaction_dict["message"] = message_id
reaction_dict["user_profile"] = user_id_mapper.get(username)
total_reactions.append(reaction_dict)
def test_convert_user_data(self) -> None:
user_id_mapper = IdMapper()
realm_id = 3
fixture_file_name = self.fixture_file_name("export.json",
"mattermost_fixtures")
mattermost_data = mattermost_data_file_to_dict(fixture_file_name)
username_to_user = create_username_to_user_mapping(
mattermost_data["user"])
reset_mirror_dummy_users(username_to_user)
team_name = "gryffindor"
user_handler = UserHandler()
convert_user_data(user_handler, user_id_mapper, username_to_user,
realm_id, team_name)
self.assert_length(user_handler.get_all_users(), 2)
self.assertTrue(user_id_mapper.has("harry"))
self.assertTrue(user_id_mapper.has("ron"))
self.assertEqual(
user_handler.get_user(user_id_mapper.get("harry"))["full_name"],
"Harry Potter")
self.assertEqual(
user_handler.get_user(user_id_mapper.get("ron"))["full_name"],
"Ron Weasley")
team_name = "slytherin"
user_handler = UserHandler()
convert_user_data(user_handler, user_id_mapper, username_to_user,
realm_id, team_name)
self.assert_length(user_handler.get_all_users(), 3)
self.assertTrue(user_id_mapper.has("malfoy"))
self.assertTrue(user_id_mapper.has("pansy"))
self.assertTrue(user_id_mapper.has("snape"))
team_name = "gryffindor"
# Snape is a mirror dummy user in Harry's team.
label_mirror_dummy_users(2, team_name, mattermost_data,
username_to_user)
user_handler = UserHandler()
convert_user_data(user_handler, user_id_mapper, username_to_user,
realm_id, team_name)
self.assert_length(user_handler.get_all_users(), 3)
self.assertTrue(user_id_mapper.has("snape"))
team_name = "slytherin"
user_handler = UserHandler()
convert_user_data(user_handler, user_id_mapper, username_to_user,
realm_id, team_name)
self.assert_length(user_handler.get_all_users(), 3)
def test_get_mentioned_user_ids(self) -> None:
user_id_mapper = IdMapper()
harry_id = user_id_mapper.get("harry")
raw_message = {"content": "Hello @harry"}
ids = get_mentioned_user_ids(raw_message, user_id_mapper)
self.assertEqual(list(ids), [harry_id])
raw_message = {"content": "Hello"}
ids = get_mentioned_user_ids(raw_message, user_id_mapper)
self.assertEqual(list(ids), [])
raw_message = {"content": "@harry How are you?"}
ids = get_mentioned_user_ids(raw_message, user_id_mapper)
self.assertEqual(list(ids), [harry_id])
raw_message = {"content": "@harry @ron Where are you folks?"}
ron_id = user_id_mapper.get("ron")
ids = get_mentioned_user_ids(raw_message, user_id_mapper)
self.assertEqual(list(ids), [harry_id, ron_id])
raw_message = {"content": "@harry.com How are you?"}
ids = get_mentioned_user_ids(raw_message, user_id_mapper)
self.assertEqual(list(ids), [])
raw_message = {"content": "[email protected] How are you?"}
ids = get_mentioned_user_ids(raw_message, user_id_mapper)
self.assertEqual(list(ids), [])
harry_id = user_id_mapper.get("harry_")
raw_message = {"content": "Hello @harry_"}
ids = get_mentioned_user_ids(raw_message, user_id_mapper)
self.assertEqual(list(ids), [harry_id])
harry_id = user_id_mapper.get("harry.")
raw_message = {"content": "Hello @harry."}
ids = get_mentioned_user_ids(raw_message, user_id_mapper)
self.assertEqual(list(ids), [harry_id])
harry_id = user_id_mapper.get("ha_rry.")
raw_message = {"content": "Hello @ha_rry."}
ids = get_mentioned_user_ids(raw_message, user_id_mapper)
self.assertEqual(list(ids), [harry_id])
ron_id = user_id_mapper.get("ron")
raw_message = {"content": "Hello @ron."}
ids = get_mentioned_user_ids(raw_message, user_id_mapper)
self.assertEqual(list(ids), [])
raw_message = {"content": "Hello @ron_"}
ids = get_mentioned_user_ids(raw_message, user_id_mapper)
self.assertEqual(list(ids), [])
def convert_avatar_data(avatar_folder: str,
raw_data: List[ZerverFieldsT],
user_id_mapper: IdMapper,
realm_id: int) -> List[ZerverFieldsT]:
'''
This code is pretty specific to how Hipchat sends us data.
They give us the avatar payloads in base64 in users.json.
We process avatars in our own pass of that data, rather
than doing it while we're getting other user data. I
chose to keep this separate, as otherwise you have a lot
of extraneous data getting passed around.
This code has MAJOR SIDE EFFECTS--namely writing a bunch
of files to the avatars directory.
'''
avatar_records = []
for d in raw_data:
raw_user = d['User']
avatar_payload = raw_user.get('avatar')
if not avatar_payload:
continue
bits = base64.b64decode(avatar_payload)
raw_user_id = raw_user['id']
if not user_id_mapper.has(raw_user_id):
continue
user_id = user_id_mapper.get(raw_user_id)
metadata = write_avatar_png(
avatar_folder=avatar_folder,
realm_id=realm_id,
user_id=user_id,
bits=bits,
)
avatar_records.append(metadata)
return avatar_records
def test_convert_huddle_data(self) -> None:
fixture_file_name = self.fixture_file_name(
"export.json", "mattermost_fixtures/direct_channel")
mattermost_data = mattermost_data_file_to_dict(fixture_file_name)
username_to_user = create_username_to_user_mapping(
mattermost_data["user"])
reset_mirror_dummy_users(username_to_user)
user_handler = UserHandler()
subscriber_handler = SubscriberHandler()
huddle_id_mapper = IdMapper()
user_id_mapper = IdMapper()
team_name = "gryffindor"
convert_user_data(
user_handler=user_handler,
user_id_mapper=user_id_mapper,
user_data_map=username_to_user,
realm_id=3,
team_name=team_name,
)
zerver_huddle = convert_huddle_data(
huddle_data=mattermost_data["direct_channel"],
user_data_map=username_to_user,
subscriber_handler=subscriber_handler,
huddle_id_mapper=huddle_id_mapper,
user_id_mapper=user_id_mapper,
realm_id=3,
team_name=team_name,
)
self.assert_length(zerver_huddle, 1)
huddle_members = mattermost_data["direct_channel"][1]["members"]
huddle_name = generate_huddle_name(huddle_members)
self.assertTrue(huddle_id_mapper.has(huddle_name))
self.assertEqual(
subscriber_handler.get_users(
huddle_id=huddle_id_mapper.get(huddle_name)), {1, 2, 3})
def get_topic_name(
message: Dict[str, Any],
dsc_id_to_dsc_map: Dict[str, Dict[str, Any]],
thread_id_mapper: IdMapper,
is_pm_data: bool = False,
) -> str:
if is_pm_data:
return ""
elif message["rid"] in dsc_id_to_dsc_map:
dsc_channel_name = dsc_id_to_dsc_map[message["rid"]]["fname"]
return f"{dsc_channel_name} (Imported from Rocket.Chat)"
elif message.get("replies"):
# Message is the start of a thread
thread_id = thread_id_mapper.get(message["_id"])
return f"Thread {thread_id} (Imported from Rocket.Chat)"
elif message.get("tmid"):
# Message is a part of a thread
thread_id = thread_id_mapper.get(message["tmid"])
return f"Thread {thread_id} (Imported from Rocket.Chat)"
else:
# Normal channel message
return "Imported from Rocket.Chat"
def process_user(
user_dict: Dict[str, Any], realm_id: int, team_name: str, user_id_mapper: IdMapper
) -> ZerverFieldsT:
def is_team_admin(user_dict: Dict[str, Any]) -> bool:
if user_dict["teams"] is None:
return False
for team in user_dict["teams"]:
if team["name"] == team_name and "team_admin" in team["roles"]:
return True
return False
def get_full_name(user_dict: Dict[str, Any]) -> str:
full_name = "{} {}".format(user_dict["first_name"], user_dict["last_name"])
if full_name.strip():
return full_name
return user_dict["username"]
avatar_source = "G"
full_name = get_full_name(user_dict)
id = user_id_mapper.get(user_dict["username"])
delivery_email = user_dict["email"]
email = user_dict["email"]
short_name = user_dict["username"]
date_joined = int(timezone_now().timestamp())
timezone = "UTC"
role = UserProfile.ROLE_MEMBER
if is_team_admin(user_dict):
role = UserProfile.ROLE_REALM_OWNER
if user_dict["is_mirror_dummy"]:
is_active = False
is_mirror_dummy = True
else:
is_active = True
is_mirror_dummy = False
return build_user_profile(
avatar_source=avatar_source,
date_joined=date_joined,
delivery_email=delivery_email,
email=email,
full_name=full_name,
id=id,
is_active=is_active,
role=role,
is_mirror_dummy=is_mirror_dummy,
realm_id=realm_id,
short_name=short_name,
timezone=timezone,
)
def convert_huddle_data(
huddle_id_to_huddle_map: Dict[str, Dict[str, Any]],
huddle_id_mapper: IdMapper,
user_id_mapper: IdMapper,
subscriber_handler: SubscriberHandler,
) -> List[ZerverFieldsT]:
zerver_huddle: List[ZerverFieldsT] = []
for rc_huddle_id in huddle_id_to_huddle_map:
huddle_id = huddle_id_mapper.get(rc_huddle_id)
huddle = build_huddle(huddle_id)
zerver_huddle.append(huddle)
huddle_dict = huddle_id_to_huddle_map[rc_huddle_id]
huddle_user_ids = set()
for rc_user_id in huddle_dict["uids"]:
huddle_user_ids.add(user_id_mapper.get(rc_user_id))
subscriber_handler.set_info(
users=huddle_user_ids,
huddle_id=huddle_id,
)
return zerver_huddle
def convert_channel_data(
room_id_to_room_map: Dict[str, Dict[str, Any]],
team_id_to_team_map: Dict[str, Dict[str, Any]],
stream_id_mapper: IdMapper,
realm_id: int,
) -> List[ZerverFieldsT]:
streams = []
for rc_room_id in room_id_to_room_map:
channel_dict = room_id_to_room_map[rc_room_id]
date_created = float(channel_dict["ts"].timestamp())
stream_id = stream_id_mapper.get(rc_room_id)
invite_only = channel_dict["t"] == "p"
stream_name = channel_dict["name"]
stream_desc = channel_dict.get("description", "")
if channel_dict.get("teamId"):
if channel_dict.get("teamMain") is True:
# In case you change this, please also change the stream name
# used while adding the Rocket.Chat channel mention data to
# message_dict in `message_to_dict` inner-function of
# `process_messages` function below.
stream_name = "[TEAM] " + stream_name
else:
stream_desc = "[Team {} channel]. {}".format(
team_id_to_team_map[channel_dict["teamId"]]["name"],
stream_desc)
# If the channel is read-only, then only admins and moderators
# should be allowed to post in the converted Zulip stream.
# For more details: https://zulip.com/help/stream-sending-policy
#
# See `Stream` model in `zerver/models.py` to know about what each
# number represent.
stream_post_policy = 4 if channel_dict.get("ro", False) else 1
stream = build_stream(
date_created=date_created,
realm_id=realm_id,
name=stream_name,
description=stream_desc,
stream_id=stream_id,
deactivated=False,
invite_only=invite_only,
stream_post_policy=stream_post_policy,
)
streams.append(stream)
return streams
def test_process_user(self) -> None:
user_id_mapper = IdMapper()
fixture_file_name = self.fixture_file_name("export.json",
"mattermost_fixtures")
mattermost_data = mattermost_data_file_to_dict(fixture_file_name)
username_to_user = create_username_to_user_mapping(
mattermost_data["user"])
reset_mirror_dummy_users(username_to_user)
harry_dict = username_to_user["harry"]
harry_dict["is_mirror_dummy"] = False
realm_id = 3
team_name = "gryffindor"
user = process_user(harry_dict, realm_id, team_name, user_id_mapper)
self.assertEqual(user["avatar_source"], "G")
self.assertEqual(user["delivery_email"], "*****@*****.**")
self.assertEqual(user["email"], "*****@*****.**")
self.assertEqual(user["full_name"], "Harry Potter")
self.assertEqual(user["id"], 1)
self.assertEqual(user["is_active"], True)
self.assertEqual(user["role"], UserProfile.ROLE_REALM_OWNER)
self.assertEqual(user["is_mirror_dummy"], False)
self.assertEqual(user["realm"], 3)
self.assertEqual(user["short_name"], "harry")
self.assertEqual(user["timezone"], "UTC")
# A user with a `null` team value shouldn't be an admin.
harry_dict["teams"] = None
user = process_user(harry_dict, realm_id, team_name, user_id_mapper)
self.assertEqual(user["role"], UserProfile.ROLE_MEMBER)
team_name = "slytherin"
snape_dict = username_to_user["snape"]
snape_dict["is_mirror_dummy"] = True
user = process_user(snape_dict, realm_id, team_name, user_id_mapper)
self.assertEqual(user["avatar_source"], "G")
self.assertEqual(user["delivery_email"], "*****@*****.**")
self.assertEqual(user["email"], "*****@*****.**")
self.assertEqual(user["full_name"], "Severus Snape")
self.assertEqual(user["id"], 2)
self.assertEqual(user["is_active"], False)
self.assertEqual(user["role"], UserProfile.ROLE_MEMBER)
self.assertEqual(user["is_mirror_dummy"], True)
self.assertEqual(user["realm"], 3)
self.assertEqual(user["short_name"], "snape")
self.assertEqual(user["timezone"], "UTC")
def process_user(user_dict: Dict[str, Any], realm_id: int, team_name: str,
user_id_mapper: IdMapper) -> ZerverFieldsT:
def is_team_admin(user_dict: Dict[str, Any]) -> bool:
for team in user_dict["teams"]:
if team["name"] == team_name and "team_admin" in team["roles"]:
return True
return False
def get_full_name(user_dict: Dict[str, Any]) -> str:
full_name = "{} {}".format(user_dict["first_name"], user_dict["last_name"])
if full_name.strip():
return full_name
return user_dict['username']
avatar_source = 'G'
full_name = get_full_name(user_dict)
id = user_id_mapper.get(user_dict['username'])
delivery_email = user_dict['email']
email = user_dict['email']
is_realm_admin = is_team_admin(user_dict)
is_guest = False
short_name = user_dict['username']
date_joined = int(timezone_now().timestamp())
timezone = 'UTC'
if user_dict["is_mirror_dummy"]:
is_active = False
is_mirror_dummy = True
else:
is_active = True
is_mirror_dummy = False
return build_user_profile(
avatar_source=avatar_source,
date_joined=date_joined,
delivery_email=delivery_email,
email=email,
full_name=full_name,
id=id,
is_active=is_active,
is_realm_admin=is_realm_admin,
is_guest=is_guest,
is_mirror_dummy=is_mirror_dummy,
realm_id=realm_id,
short_name=short_name,
timezone=timezone,
)
def test_process_user(self) -> None:
user_id_mapper = IdMapper()
harry_dict = self.username_to_user["harry"]
harry_dict["is_mirror_dummy"] = False
realm_id = 3
team_name = "gryffindor"
user = process_user(harry_dict, realm_id, team_name, user_id_mapper)
self.assertEqual(user["avatar_source"], 'G')
self.assertEqual(user["delivery_email"], "*****@*****.**")
self.assertEqual(user["email"], "*****@*****.**")
self.assertEqual(user["full_name"], "Harry Potter")
self.assertEqual(user["id"], 1)
self.assertEqual(user["is_active"], True)
self.assertEqual(user["is_realm_admin"], True)
self.assertEqual(user["is_guest"], False)
self.assertEqual(user["is_mirror_dummy"], False)
self.assertEqual(user["realm"], 3)
self.assertEqual(user["short_name"], "harry")
self.assertEqual(user["timezone"], "UTC")
# A user with a `null` team value shouldn't be an admin.
harry_dict["teams"] = None
user = process_user(harry_dict, realm_id, team_name, user_id_mapper)
self.assertEqual(user["is_realm_admin"], False)
team_name = "slytherin"
snape_dict = self.username_to_user["snape"]
snape_dict["is_mirror_dummy"] = True
user = process_user(snape_dict, realm_id, team_name, user_id_mapper)
self.assertEqual(user["avatar_source"], 'G')
self.assertEqual(user["delivery_email"], "*****@*****.**")
self.assertEqual(user["email"], "*****@*****.**")
self.assertEqual(user["full_name"], "Severus Snape")
self.assertEqual(user["id"], 2)
self.assertEqual(user["is_active"], False)
self.assertEqual(user["is_realm_admin"], False)
self.assertEqual(user["is_guest"], False)
self.assertEqual(user["is_mirror_dummy"], True)
self.assertEqual(user["realm"], 3)
self.assertEqual(user["short_name"], "snape")
self.assertEqual(user["timezone"], "UTC")
def convert_channel_data(
room_id_to_room_map: Dict[str, Dict[str, Any]],
team_id_to_team_map: Dict[str, Dict[str, Any]],
stream_id_mapper: IdMapper,
realm_id: int,
) -> List[ZerverFieldsT]:
streams = []
for rc_room_id in room_id_to_room_map:
channel_dict = room_id_to_room_map[rc_room_id]
date_created = float(channel_dict["ts"].timestamp())
stream_id = stream_id_mapper.get(rc_room_id)
invite_only = channel_dict["t"] == "p"
stream_name = get_stream_name(channel_dict)
stream_desc = channel_dict.get("description", "")
if channel_dict.get("teamId") and not channel_dict.get("teamMain"):
stream_desc = "[Team {} channel]. {}".format(
team_id_to_team_map[channel_dict["teamId"]]["name"],
stream_desc)
# If the channel is read-only, then only admins and moderators
# should be allowed to post in the converted Zulip stream.
# For more details: https://zulip.com/help/stream-sending-policy
#
# See `Stream` model in `zerver/models.py` to know about what each
# number represent.
stream_post_policy = 4 if channel_dict.get("ro", False) else 1
stream = build_stream(
date_created=date_created,
realm_id=realm_id,
name=stream_name,
description=stream_desc,
stream_id=stream_id,
deactivated=False,
invite_only=invite_only,
stream_post_policy=stream_post_policy,
)
streams.append(stream)
return streams
def test_convert_channel_data(self) -> None:
fixture_file_name = self.fixture_file_name("export.json", "mattermost_fixtures")
mattermost_data = mattermost_data_file_to_dict(fixture_file_name)
username_to_user = create_username_to_user_mapping(mattermost_data["user"])
reset_mirror_dummy_users(username_to_user)
user_handler = UserHandler()
subscriber_handler = SubscriberHandler()
stream_id_mapper = IdMapper()
user_id_mapper = IdMapper()
team_name = "gryffindor"
convert_user_data(
user_handler=user_handler,
user_id_mapper=user_id_mapper,
user_data_map=username_to_user,
realm_id=3,
team_name=team_name,
)
zerver_stream = convert_channel_data(
channel_data=mattermost_data["channel"],
user_data_map=username_to_user,
subscriber_handler=subscriber_handler,
stream_id_mapper=stream_id_mapper,
user_id_mapper=user_id_mapper,
realm_id=3,
team_name=team_name,
)
self.assertEqual(len(zerver_stream), 3)
self.assertEqual(zerver_stream[0]["name"], "Gryffindor common room")
self.assertEqual(zerver_stream[0]["invite_only"], False)
self.assertEqual(zerver_stream[0]["description"], "A place for talking about Gryffindor common room")
self.assertEqual(zerver_stream[0]["rendered_description"], "")
self.assertEqual(zerver_stream[0]["realm"], 3)
self.assertEqual(zerver_stream[1]["name"], "Gryffindor quidditch team")
self.assertEqual(zerver_stream[1]["invite_only"], False)
self.assertEqual(zerver_stream[1]["description"], "A place for talking about Gryffindor quidditch team")
self.assertEqual(zerver_stream[1]["rendered_description"], "")
self.assertEqual(zerver_stream[1]["realm"], 3)
self.assertEqual(zerver_stream[2]["name"], "Dumbledores army")
self.assertEqual(zerver_stream[2]["invite_only"], True)
self.assertEqual(zerver_stream[2]["description"], "A place for talking about Dumbledores army")
self.assertEqual(zerver_stream[2]["rendered_description"], "")
self.assertEqual(zerver_stream[2]["realm"], 3)
self.assertTrue(stream_id_mapper.has("gryffindor-common-room"))
self.assertTrue(stream_id_mapper.has("gryffindor-quidditch-team"))
self.assertTrue(stream_id_mapper.has("dumbledores-army"))
# TODO: Add ginny
ron_id = user_id_mapper.get("ron")
harry_id = user_id_mapper.get("harry")
self.assertEqual({ron_id, harry_id}, {1, 2})
self.assertEqual(subscriber_handler.get_users(stream_id=stream_id_mapper.get("gryffindor-common-room")), {ron_id, harry_id})
self.assertEqual(subscriber_handler.get_users(stream_id=stream_id_mapper.get("gryffindor-quidditch-team")), {ron_id, harry_id})
self.assertEqual(subscriber_handler.get_users(stream_id=stream_id_mapper.get("dumbledores-army")), {ron_id, harry_id})
# Converting channel data when a user's `teams` value is `null`.
username_to_user["ron"].update({"teams": None})
zerver_stream = convert_channel_data(
channel_data=mattermost_data["channel"],
user_data_map=username_to_user,
subscriber_handler=subscriber_handler,
stream_id_mapper=stream_id_mapper,
user_id_mapper=user_id_mapper,
realm_id=3,
team_name=team_name,
)
harry_id = user_id_mapper.get("harry")
self.assertIn(harry_id, {1, 2})
self.assertEqual(subscriber_handler.get_users(stream_id=stream_id_mapper.get("gryffindor-common-room")), {harry_id})
self.assertEqual(subscriber_handler.get_users(stream_id=stream_id_mapper.get("gryffindor-quidditch-team")), {harry_id})
self.assertEqual(subscriber_handler.get_users(stream_id=stream_id_mapper.get("dumbledores-army")), {harry_id})
team_name = "slytherin"
zerver_stream = convert_channel_data(
channel_data=mattermost_data["channel"],
user_data_map=username_to_user,
subscriber_handler=subscriber_handler,
stream_id_mapper=stream_id_mapper,
user_id_mapper=user_id_mapper,
realm_id=4,
team_name=team_name,
)
malfoy_id = user_id_mapper.get("malfoy")
pansy_id = user_id_mapper.get("pansy")
snape_id = user_id_mapper.get("snape")
self.assertEqual({malfoy_id, pansy_id, snape_id}, {3, 4, 5})
self.assertEqual(subscriber_handler.get_users(stream_id=stream_id_mapper.get("slytherin-common-room")), {malfoy_id, pansy_id, snape_id})
self.assertEqual(subscriber_handler.get_users(stream_id=stream_id_mapper.get("slytherin-quidditch-team")), {malfoy_id, pansy_id})
def do_convert_data(input_tar_file: str,
output_dir: str,
masking_content: bool,
api_token: Optional[str] = None,
slim_mode: bool = False) -> None:
input_data_dir = untar_input_file(input_tar_file)
attachment_handler = AttachmentHandler()
user_handler = UserHandler()
subscriber_handler = SubscriberHandler()
user_id_mapper = IdMapper()
stream_id_mapper = IdMapper()
realm_id = 0
realm = make_realm(realm_id=realm_id)
# users.json -> UserProfile
raw_user_data = read_user_data(data_dir=input_data_dir)
convert_user_data(
user_handler=user_handler,
slim_mode=slim_mode,
user_id_mapper=user_id_mapper,
raw_data=raw_user_data,
realm_id=realm_id,
)
normal_users = user_handler.get_normal_users()
# Don't write zerver_userprofile here, because we
# may add more users later.
# streams.json -> Stream
raw_stream_data = read_room_data(data_dir=input_data_dir)
zerver_stream = convert_room_data(
raw_data=raw_stream_data,
subscriber_handler=subscriber_handler,
stream_id_mapper=stream_id_mapper,
user_id_mapper=user_id_mapper,
realm_id=realm_id,
api_token=api_token,
)
realm['zerver_stream'] = zerver_stream
zerver_recipient = build_recipients(
zerver_userprofile=normal_users,
zerver_stream=zerver_stream,
)
realm['zerver_recipient'] = zerver_recipient
if api_token is None:
if slim_mode:
public_stream_subscriptions: List[ZerverFieldsT] = []
else:
public_stream_subscriptions = build_public_stream_subscriptions(
zerver_userprofile=normal_users,
zerver_recipient=zerver_recipient,
zerver_stream=zerver_stream,
)
private_stream_subscriptions = build_stream_subscriptions(
get_users=subscriber_handler.get_users,
zerver_recipient=zerver_recipient,
zerver_stream=[
stream_dict for stream_dict in zerver_stream
if stream_dict['invite_only']
],
)
stream_subscriptions = public_stream_subscriptions + private_stream_subscriptions
else:
stream_subscriptions = build_stream_subscriptions(
get_users=subscriber_handler.get_users,
zerver_recipient=zerver_recipient,
zerver_stream=zerver_stream,
)
personal_subscriptions = build_personal_subscriptions(
zerver_recipient=zerver_recipient, )
zerver_subscription = personal_subscriptions + stream_subscriptions
realm['zerver_subscription'] = zerver_subscription
zerver_realmemoji = write_emoticon_data(
realm_id=realm_id,
data_dir=input_data_dir,
output_dir=output_dir,
)
realm['zerver_realmemoji'] = zerver_realmemoji
subscriber_map = make_subscriber_map(
zerver_subscription=zerver_subscription, )
logging.info('Start importing message data')
for message_key in [
'UserMessage', 'NotificationMessage', 'PrivateUserMessage'
]:
write_message_data(
realm_id=realm_id,
slim_mode=slim_mode,
message_key=message_key,
zerver_recipient=zerver_recipient,
subscriber_map=subscriber_map,
data_dir=input_data_dir,
output_dir=output_dir,
masking_content=masking_content,
stream_id_mapper=stream_id_mapper,
user_id_mapper=user_id_mapper,
user_handler=user_handler,
attachment_handler=attachment_handler,
)
# Order is important here...don't write users until
# we process everything else, since we may introduce
# mirror users when processing messages.
realm['zerver_userprofile'] = user_handler.get_all_users()
realm['sort_by_date'] = True
create_converted_data_files(realm, output_dir, '/realm.json')
logging.info('Start importing avatar data')
write_avatar_data(
raw_user_data=raw_user_data,
output_dir=output_dir,
user_id_mapper=user_id_mapper,
realm_id=realm_id,
)
attachment_handler.write_info(
output_dir=output_dir,
realm_id=realm_id,
)
logging.info('Start making tarball')
subprocess.check_call(
["tar", "-czf", output_dir + '.tar.gz', output_dir, '-P'])
logging.info('Done making tarball')
def process_raw_message_batch(realm_id: int, raw_messages: List[Dict[str,
Any]],
subscriber_map: Dict[int, Set[int]],
user_id_mapper: IdMapper,
user_handler: UserHandler,
attachment_handler: AttachmentHandler,
get_recipient_id: Callable[[ZerverFieldsT], int],
is_pm_data: bool, output_dir: str) -> None:
def fix_mentions(content: str, mention_user_ids: Set[int]) -> str:
for user_id in mention_user_ids:
user = user_handler.get_user(user_id=user_id)
hipchat_mention = '@{short_name}'.format(**user)
zulip_mention = '@**{full_name}**'.format(**user)
content = content.replace(hipchat_mention, zulip_mention)
content = content.replace('@here', '@**all**')
return content
mention_map: Dict[int, Set[int]] = dict()
zerver_message = []
import html2text
h = html2text.HTML2Text()
for raw_message in raw_messages:
# One side effect here:
message_id = NEXT_ID('message')
mention_user_ids = {
user_id_mapper.get(id)
for id in set(raw_message['mention_user_ids'])
if user_id_mapper.has(id)
}
mention_map[message_id] = mention_user_ids
content = fix_mentions(
content=raw_message['content'],
mention_user_ids=mention_user_ids,
)
content = h.handle(content)
if len(content) > 10000:
logging.info('skipping too-long message of length %s' %
(len(content), ))
continue
date_sent = raw_message['date_sent']
try:
recipient_id = get_recipient_id(raw_message)
except KeyError:
logging.debug(
"Could not find recipient_id for a message, skipping.")
continue
rendered_content = None
if is_pm_data:
topic_name = ''
else:
topic_name = 'imported from hipchat'
user_id = raw_message['sender_id']
# Another side effect:
extra_content = attachment_handler.handle_message_data(
realm_id=realm_id,
message_id=message_id,
sender_id=user_id,
attachment=raw_message['attachment'],
files_dir=raw_message['files_dir'],
)
if extra_content:
has_attachment = True
content += '\n' + extra_content
else:
has_attachment = False
message = build_message(
content=content,
message_id=message_id,
date_sent=date_sent,
recipient_id=recipient_id,
rendered_content=rendered_content,
topic_name=topic_name,
user_id=user_id,
has_attachment=has_attachment,
)
zerver_message.append(message)
zerver_usermessage = make_user_messages(
zerver_message=zerver_message,
subscriber_map=subscriber_map,
is_pm_data=is_pm_data,
mention_map=mention_map,
)
message_json = dict(
zerver_message=zerver_message,
zerver_usermessage=zerver_usermessage,
)
dump_file_id = NEXT_ID('dump_file_id')
message_file = "/messages-%06d.json" % (dump_file_id, )
create_converted_data_files(message_json, output_dir, message_file)
def convert_room_data(raw_data: List[ZerverFieldsT],
subscriber_handler: SubscriberHandler,
stream_id_mapper: IdMapper,
user_id_mapper: IdMapper,
realm_id: int,
api_token: Optional[str] = None) -> List[ZerverFieldsT]:
flat_data = [d['Room'] for d in raw_data]
def get_invite_only(v: str) -> bool:
if v == 'public':
return False
elif v == 'private':
return True
else:
raise Exception('unexpected value')
streams = []
for in_dict in flat_data:
now = int(timezone_now().timestamp())
stream_id = stream_id_mapper.get(in_dict['id'])
invite_only = get_invite_only(in_dict['privacy'])
stream = build_stream(
date_created=now,
realm_id=realm_id,
name=in_dict['name'],
description=in_dict['topic'],
stream_id=stream_id,
deactivated=in_dict['is_archived'],
invite_only=invite_only,
)
if invite_only:
users: Set[int] = {
user_id_mapper.get(key)
for key in in_dict['members'] if user_id_mapper.has(key)
}
if user_id_mapper.has(in_dict['owner']):
owner = user_id_mapper.get(in_dict['owner'])
users.add(owner)
else:
users = set()
if api_token is not None:
hc = hypchat.HypChat(api_token)
room_data = hc.fromurl('{}/v2/room/{}/member'.format(
hc.endpoint, in_dict['id']))
for item in room_data['items']:
hipchat_user_id = item['id']
zulip_user_id = user_id_mapper.get(hipchat_user_id)
users.add(zulip_user_id)
if users:
subscriber_handler.set_info(
stream_id=stream_id,
users=users,
)
# unmapped fields:
# guest_access_url: no Zulip equivalent
# created: we just use "now"
# participants: no good sample data
streams.append(stream)
return streams
def do_convert_data(mattermost_data_dir: str, output_dir: str, masking_content: bool) -> None:
username_to_user: Dict[str, Dict[str, Any]] = {}
os.makedirs(output_dir, exist_ok=True)
if os.listdir(output_dir): # nocoverage
raise Exception("Output directory should be empty!")
mattermost_data_file = os.path.join(mattermost_data_dir, "export.json")
mattermost_data = mattermost_data_file_to_dict(mattermost_data_file)
username_to_user = create_username_to_user_mapping(mattermost_data["user"])
for team in mattermost_data["team"]:
realm_id = NEXT_ID("realm_id")
team_name = team["name"]
user_handler = UserHandler()
subscriber_handler = SubscriberHandler()
user_id_mapper = IdMapper()
stream_id_mapper = IdMapper()
huddle_id_mapper = IdMapper()
print("Generating data for", team_name)
realm = make_realm(realm_id, team)
realm_output_dir = os.path.join(output_dir, team_name)
reset_mirror_dummy_users(username_to_user)
label_mirror_dummy_users(
len(mattermost_data["team"]), team_name, mattermost_data, username_to_user
)
convert_user_data(
user_handler=user_handler,
user_id_mapper=user_id_mapper,
user_data_map=username_to_user,
realm_id=realm_id,
team_name=team_name,
)
zerver_stream = convert_channel_data(
channel_data=mattermost_data["channel"],
user_data_map=username_to_user,
subscriber_handler=subscriber_handler,
stream_id_mapper=stream_id_mapper,
user_id_mapper=user_id_mapper,
realm_id=realm_id,
team_name=team_name,
)
realm["zerver_stream"] = zerver_stream
zerver_huddle: List[ZerverFieldsT] = []
if len(mattermost_data["team"]) == 1:
zerver_huddle = convert_huddle_data(
huddle_data=mattermost_data["direct_channel"],
user_data_map=username_to_user,
subscriber_handler=subscriber_handler,
huddle_id_mapper=huddle_id_mapper,
user_id_mapper=user_id_mapper,
realm_id=realm_id,
team_name=team_name,
)
realm["zerver_huddle"] = zerver_huddle
all_users = user_handler.get_all_users()
zerver_recipient = build_recipients(
zerver_userprofile=all_users,
zerver_stream=zerver_stream,
zerver_huddle=zerver_huddle,
)
realm["zerver_recipient"] = zerver_recipient
stream_subscriptions = build_stream_subscriptions(
get_users=subscriber_handler.get_users,
zerver_recipient=zerver_recipient,
zerver_stream=zerver_stream,
)
huddle_subscriptions = build_huddle_subscriptions(
get_users=subscriber_handler.get_users,
zerver_recipient=zerver_recipient,
zerver_huddle=zerver_huddle,
)
personal_subscriptions = build_personal_subscriptions(
zerver_recipient=zerver_recipient,
)
# Mattermost currently supports only exporting messages from channels.
# Personal messages and huddles are not exported.
zerver_subscription = personal_subscriptions + stream_subscriptions + huddle_subscriptions
realm["zerver_subscription"] = zerver_subscription
zerver_realmemoji = write_emoticon_data(
realm_id=realm_id,
custom_emoji_data=mattermost_data["emoji"],
data_dir=mattermost_data_dir,
output_dir=realm_output_dir,
)
realm["zerver_realmemoji"] = zerver_realmemoji
subscriber_map = make_subscriber_map(
zerver_subscription=zerver_subscription,
)
total_reactions: List[Dict[str, Any]] = []
uploads_list: List[ZerverFieldsT] = []
zerver_attachment: List[ZerverFieldsT] = []
write_message_data(
num_teams=len(mattermost_data["team"]),
team_name=team_name,
realm_id=realm_id,
post_data=mattermost_data["post"],
zerver_recipient=zerver_recipient,
subscriber_map=subscriber_map,
output_dir=realm_output_dir,
masking_content=masking_content,
stream_id_mapper=stream_id_mapper,
huddle_id_mapper=huddle_id_mapper,
user_id_mapper=user_id_mapper,
user_handler=user_handler,
zerver_realmemoji=zerver_realmemoji,
total_reactions=total_reactions,
uploads_list=uploads_list,
zerver_attachment=zerver_attachment,
mattermost_data_dir=mattermost_data_dir,
)
realm["zerver_reaction"] = total_reactions
realm["zerver_userprofile"] = user_handler.get_all_users()
realm["sort_by_date"] = True
create_converted_data_files(realm, realm_output_dir, "/realm.json")
# Mattermost currently doesn't support exporting avatars
create_converted_data_files([], realm_output_dir, "/avatars/records.json")
# Export message attachments
attachment: Dict[str, List[Any]] = {"zerver_attachment": zerver_attachment}
create_converted_data_files(uploads_list, realm_output_dir, "/uploads/records.json")
create_converted_data_files(attachment, realm_output_dir, "/attachment.json")
logging.info("Start making tarball")
subprocess.check_call(["tar", "-czf", realm_output_dir + ".tar.gz", realm_output_dir, "-P"])
logging.info("Done making tarball")
def test_sender_ids(self) -> None:
realm_id = 5
user_handler = UserHandler()
user_id_mapper = IdMapper()
user_id_mapper.has = lambda key: True # type: ignore # it's just a stub
# Simulate a "normal" user first.
user_with_id = dict(
id=1,
# other fields don't matter here
)
user_handler.add_user(user=user_with_id)
normal_message = dict(
sender=dict(
id=1,
)
) # type: Dict[str, Any]
sender_id = get_hipchat_sender_id(
realm_id=realm_id,
slim_mode=False,
message_dict=normal_message,
user_id_mapper=user_id_mapper,
user_handler=user_handler,
)
self.assertEqual(sender_id, 1)
bot_message = dict(
sender='fred_bot',
)
# Every message from fred_bot should
# return the same sender_id.
fred_bot_sender_id = 2
for i in range(3):
sender_id = get_hipchat_sender_id(
realm_id=realm_id,
slim_mode=False,
message_dict=bot_message,
user_id_mapper=user_id_mapper,
user_handler=user_handler,
)
self.assertEqual(sender_id, fred_bot_sender_id)
id_zero_message = dict(
sender=dict(
id=0,
name='hal_bot',
),
)
hal_bot_sender_id = 3
for i in range(3):
sender_id = get_hipchat_sender_id(
realm_id=realm_id,
slim_mode=False,
message_dict=id_zero_message,
user_id_mapper=user_id_mapper,
user_handler=user_handler,
)
self.assertEqual(sender_id, hal_bot_sender_id)
