def convert_huddle_data( huddle_data: List[ZerverFieldsT], user_data_map: Dict[str, Dict[str, Any]], subscriber_handler: SubscriberHandler, huddle_id_mapper: IdMapper, user_id_mapper: IdMapper, realm_id: int, team_name: str, ) -> List[ZerverFieldsT]: zerver_huddle = [] for huddle in huddle_data: if len(huddle["members"]) > 2: huddle_name = generate_huddle_name(huddle["members"]) huddle_id = huddle_id_mapper.get(huddle_name) huddle_dict = build_huddle(huddle_id) huddle_user_ids = set() for username in huddle["members"]: huddle_user_ids.add(user_id_mapper.get(username)) subscriber_handler.set_info( users=huddle_user_ids, huddle_id=huddle_id, ) zerver_huddle.append(huddle_dict) return zerver_huddle
def convert_stream_subscription_data( user_id_to_user_map: Dict[str, Dict[str, Any]], dsc_id_to_dsc_map: Dict[str, Dict[str, Any]], zerver_stream: List[ZerverFieldsT], stream_id_mapper: IdMapper, user_id_mapper: IdMapper, subscriber_handler: SubscriberHandler, ) -> None: stream_members_map: Dict[int, Set[int]] = {} for rc_user_id in user_id_to_user_map: user_dict = user_id_to_user_map[rc_user_id] if not user_dict.get("__rooms"): continue for channel in user_dict["__rooms"]: if channel in dsc_id_to_dsc_map: # Ignore discussion rooms as these are not # imported as streams, but topics. continue stream_id = stream_id_mapper.get(channel) if stream_id not in stream_members_map: stream_members_map[stream_id] = set() stream_members_map[stream_id].add(user_id_mapper.get(rc_user_id)) for stream in zerver_stream: if stream["id"] in stream_members_map: users = stream_members_map[stream["id"]] else: users = set() # Set the stream without any subscribers # as deactivated. stream["deactivated"] = True subscriber_handler.set_info(users=users, stream_id=stream["id"])
def test_build_reactions(self) -> None: total_reactions = [] # type: List[Dict[str, Any]] reactions = [{ "user": "******", "create_at": 1553165521410, "emoji_name": "tick" }, { "user": "******", "create_at": 1553166530805, "emoji_name": "smile" }, { "user": "******", "create_at": 1553166540953, "emoji_name": "world_map" }, { "user": "******", "create_at": 1553166540957, "emoji_name": "world_map" }] zerver_realmemoji = write_emoticon_data( realm_id=3, custom_emoji_data=self.mattermost_data["emoji"], data_dir=self.fixture_file_name("", "mattermost_fixtures"), output_dir=self.make_import_output_dir("mattermost")) # Make sure tick is present in fixture data self.assertEqual(zerver_realmemoji[1]["name"], "tick") tick_emoji_code = zerver_realmemoji[1]["id"] name_to_codepoint = get_name_to_codepoint_dict() user_id_mapper = IdMapper() harry_id = user_id_mapper.get("harry") ron_id = user_id_mapper.get("ron") build_reactions(realm_id=3, total_reactions=total_reactions, reactions=reactions, message_id=5, name_to_codepoint=name_to_codepoint, user_id_mapper=user_id_mapper, zerver_realmemoji=zerver_realmemoji) smile_emoji_code = name_to_codepoint["smile"] world_map_emoji_code = name_to_codepoint["world_map"] self.assertEqual(len(total_reactions), 4) self.assertEqual(self.get_set(total_reactions, "reaction_type"), set([Reaction.REALM_EMOJI, Reaction.UNICODE_EMOJI])) self.assertEqual(self.get_set(total_reactions, "emoji_name"), set(["tick", "smile", "world_map"])) self.assertEqual( self.get_set(total_reactions, "emoji_code"), set([tick_emoji_code, smile_emoji_code, world_map_emoji_code])) self.assertEqual(self.get_set(total_reactions, "user_profile"), set([harry_id, ron_id])) self.assertEqual(len(self.get_set(total_reactions, "id")), 4) self.assertEqual(len(self.get_set(total_reactions, "message")), 1)
def test_build_reactions(self) -> None: total_reactions = [] # type: List[Dict[str, Any]] reactions = [ {"user": "******", "create_at": 1553165521410, "emoji_name": "tick"}, {"user": "******", "create_at": 1553166530805, "emoji_name": "smile"}, {"user": "******", "create_at": 1553166540953, "emoji_name": "world_map"}, {"user": "******", "create_at": 1553166540957, "emoji_name": "world_map"} ] zerver_realmemoji = write_emoticon_data( realm_id=3, custom_emoji_data=self.mattermost_data["emoji"], data_dir=self.fixture_file_name("", "mattermost_fixtures"), output_dir=self.make_import_output_dir("mattermost") ) # Make sure tick is present in fixture data self.assertEqual(zerver_realmemoji[1]["name"], "tick") tick_emoji_code = zerver_realmemoji[1]["id"] name_to_codepoint = get_name_to_codepoint_dict() user_id_mapper = IdMapper() harry_id = user_id_mapper.get("harry") ron_id = user_id_mapper.get("ron") build_reactions( realm_id=3, total_reactions=total_reactions, reactions=reactions, message_id=5, name_to_codepoint=name_to_codepoint, user_id_mapper=user_id_mapper, zerver_realmemoji=zerver_realmemoji ) smile_emoji_code = name_to_codepoint["smile"] world_map_emoji_code = name_to_codepoint["world_map"] expected_total_reactions = [ { 'user_profile': harry_id, 'message': 5, 'id': 1, 'reaction_type': Reaction.REALM_EMOJI, 'emoji_code': tick_emoji_code, 'emoji_name': 'tick' }, { 'user_profile': ron_id, 'message': 5, 'id': 2, 'reaction_type': Reaction.UNICODE_EMOJI, 'emoji_code': smile_emoji_code, 'emoji_name': 'smile' }, { 'user_profile': ron_id, 'message': 5, 'id': 3, 'reaction_type': Reaction.UNICODE_EMOJI, 'emoji_code': world_map_emoji_code, 'emoji_name': 'world_map' }, { 'user_profile': harry_id, 'message': 5, 'id': 4, 'reaction_type': Reaction.UNICODE_EMOJI, 'emoji_code': world_map_emoji_code, 'emoji_name': 'world_map' } ] self.assertEqual(total_reactions, expected_total_reactions)
def convert_room_data(raw_data: List[ZerverFieldsT], subscriber_handler: SubscriberHandler, stream_id_mapper: IdMapper, user_id_mapper: IdMapper, realm_id: int) -> List[ZerverFieldsT]: flat_data = [d['Room'] for d in raw_data] def get_invite_only(v: str) -> bool: if v == 'public': return False elif v == 'private': return True else: raise Exception('unexpected value') streams = [] for in_dict in flat_data: now = int(timezone_now().timestamp()) stream_id = stream_id_mapper.get(in_dict['id']) invite_only = get_invite_only(in_dict['privacy']) stream = build_stream( date_created=now, realm_id=realm_id, name=in_dict['name'], description=in_dict['topic'], stream_id=stream_id, deactivated=in_dict['is_archived'], invite_only=invite_only, ) if invite_only: users = { user_id_mapper.get(key) for key in in_dict['members'] if user_id_mapper.has(key) } # type: Set[int] if user_id_mapper.has(in_dict['owner']): owner = user_id_mapper.get(in_dict['owner']) users.add(owner) if not users: continue subscriber_handler.set_info( stream_id=stream_id, users=users, ) # unmapped fields: # guest_access_url: no Zulip equivalent # created: we just use "now" # participants: no good sample data streams.append(stream) return streams
def test_sender_ids(self) -> None: realm_id = 5 user_handler = UserHandler() user_id_mapper = IdMapper() user_id_mapper.has = lambda key: True # type: ignore[assignment] # it's just a stub # Simulate a "normal" user first. user_with_id = dict(id=1, # other fields don't matter here ) user_handler.add_user(user=user_with_id) normal_message: Dict[str, Any] = dict(sender=dict(id=1, ), ) sender_id = get_hipchat_sender_id( realm_id=realm_id, slim_mode=False, message_dict=normal_message, user_id_mapper=user_id_mapper, user_handler=user_handler, ) self.assertEqual(sender_id, 1) bot_message = dict(sender='fred_bot', ) # Every message from fred_bot should # return the same sender_id. fred_bot_sender_id = 2 for i in range(3): sender_id = get_hipchat_sender_id( realm_id=realm_id, slim_mode=False, message_dict=bot_message, user_id_mapper=user_id_mapper, user_handler=user_handler, ) self.assertEqual(sender_id, fred_bot_sender_id) id_zero_message = dict(sender=dict( id=0, name='hal_bot', ), ) hal_bot_sender_id = 3 for i in range(3): sender_id = get_hipchat_sender_id( realm_id=realm_id, slim_mode=False, message_dict=id_zero_message, user_id_mapper=user_id_mapper, user_handler=user_handler, ) self.assertEqual(sender_id, hal_bot_sender_id)
def test_convert_user_data(self) -> None: user_id_mapper = IdMapper() realm_id = 3 team_name = "gryffindor" user_handler = UserHandler() convert_user_data(user_handler, user_id_mapper, self.username_to_user, realm_id, team_name) self.assertTrue(user_id_mapper.has("harry")) self.assertTrue(user_id_mapper.has("ron")) self.assertEqual(user_handler.get_user(user_id_mapper.get("harry"))["full_name"], "Harry Potter") self.assertEqual(user_handler.get_user(user_id_mapper.get("ron"))["full_name"], "Ron Weasley") team_name = "slytherin" user_handler = UserHandler() convert_user_data(user_handler, user_id_mapper, self.username_to_user, realm_id, team_name) self.assertEqual(len(user_handler.get_all_users()), 3) self.assertTrue(user_id_mapper.has("malfoy")) self.assertTrue(user_id_mapper.has("pansy")) self.assertTrue(user_id_mapper.has("snape")) team_name = "gryffindor" # Snape is a mirror dummy user in Harry's team. label_mirror_dummy_users(team_name, self.mattermost_data, self.username_to_user) user_handler = UserHandler() convert_user_data(user_handler, user_id_mapper, self.username_to_user, realm_id, team_name) self.assertEqual(len(user_handler.get_all_users()), 3) self.assertTrue(user_id_mapper.has("snape")) team_name = "slytherin" user_handler = UserHandler() convert_user_data(user_handler, user_id_mapper, self.username_to_user, realm_id, team_name) self.assertEqual(len(user_handler.get_all_users()), 3)
def test_convert_huddle_data(self) -> None: fixture_dir_name = self.fixture_file_name("", "rocketchat_fixtures") rocketchat_data = rocketchat_data_to_dict(fixture_dir_name) realm_id = 3 domain_name = "zulip.com" user_handler = UserHandler() subscriber_handler = SubscriberHandler() user_id_mapper = IdMapper() huddle_id_mapper = IdMapper() user_id_to_user_map = map_user_id_to_user(rocketchat_data["user"]) process_users( user_id_to_user_map=user_id_to_user_map, realm_id=realm_id, domain_name=domain_name, user_handler=user_handler, user_id_mapper=user_id_mapper, ) room_id_to_room_map: Dict[str, Dict[str, Any]] = {} team_id_to_team_map: Dict[str, Dict[str, Any]] = {} dsc_id_to_dsc_map: Dict[str, Dict[str, Any]] = {} direct_id_to_direct_map: Dict[str, Dict[str, Any]] = {} huddle_id_to_huddle_map: Dict[str, Dict[str, Any]] = {} livechat_id_to_livechat_map: Dict[str, Dict[str, Any]] = {} categorize_channels_and_map_with_id( channel_data=rocketchat_data["room"], room_id_to_room_map=room_id_to_room_map, team_id_to_team_map=team_id_to_team_map, dsc_id_to_dsc_map=dsc_id_to_dsc_map, direct_id_to_direct_map=direct_id_to_direct_map, huddle_id_to_huddle_map=huddle_id_to_huddle_map, livechat_id_to_livechat_map=livechat_id_to_livechat_map, ) zerver_huddle = convert_huddle_data( huddle_id_to_huddle_map=huddle_id_to_huddle_map, huddle_id_mapper=huddle_id_mapper, user_id_mapper=user_id_mapper, subscriber_handler=subscriber_handler, ) self.assert_length(zerver_huddle, 1) rc_huddle_id = rocketchat_data["room"][12]["_id"] self.assertTrue(huddle_id_mapper.has(rc_huddle_id)) huddle_id = huddle_id_mapper.get(rc_huddle_id) self.assertEqual(subscriber_handler.get_users(huddle_id=huddle_id), {3, 4, 5})
def get_mentioned_user_ids(raw_message: Dict[str, Any], user_id_mapper: IdMapper) -> Set[int]: user_ids = set() content = raw_message["content"] # usernames can be of the form user.name, user_name, username., username_, user.name_ etc matches = re.findall("(?<=^|(?<=[^a-zA-Z0-9-_.]))@(([A-Za-z0-9]+[_.]?)+)", content) for match in matches: possible_username = match[0] if user_id_mapper.has(possible_username): user_ids.add(user_id_mapper.get(possible_username)) return user_ids
def test_build_reactions(self) -> None: fixture_file_name = self.fixture_file_name("export.json", "mattermost_fixtures") mattermost_data = mattermost_data_file_to_dict(fixture_file_name) total_reactions: List[Dict[str, Any]] = [] reactions = [ {"user": "******", "create_at": 1553165521410, "emoji_name": "tick"}, {"user": "******", "create_at": 1553166530805, "emoji_name": "smile"}, {"user": "******", "create_at": 1553166540953, "emoji_name": "world_map"}, {"user": "******", "create_at": 1553166540957, "emoji_name": "world_map"}, ] with self.assertLogs(level="INFO"): zerver_realmemoji = write_emoticon_data( realm_id=3, custom_emoji_data=mattermost_data["emoji"], data_dir=self.fixture_file_name("", "mattermost_fixtures"), output_dir=self.make_import_output_dir("mattermost"), ) # Make sure tick is present in fixture data self.assertEqual(zerver_realmemoji[1]["name"], "tick") tick_emoji_code = zerver_realmemoji[1]["id"] user_id_mapper = IdMapper() harry_id = user_id_mapper.get("harry") ron_id = user_id_mapper.get("ron") build_reactions( realm_id=3, total_reactions=total_reactions, reactions=reactions, message_id=5, user_id_mapper=user_id_mapper, zerver_realmemoji=zerver_realmemoji, ) smile_emoji_code = name_to_codepoint["smile"] world_map_emoji_code = name_to_codepoint["world_map"] self.assertEqual(len(total_reactions), 4) self.assertEqual(self.get_set(total_reactions, "reaction_type"), {Reaction.REALM_EMOJI, Reaction.UNICODE_EMOJI}) self.assertEqual(self.get_set(total_reactions, "emoji_name"), {"tick", "smile", "world_map"}) self.assertEqual(self.get_set(total_reactions, "emoji_code"), {tick_emoji_code, smile_emoji_code, world_map_emoji_code}) self.assertEqual(self.get_set(total_reactions, "user_profile"), {harry_id, ron_id}) self.assertEqual(len(self.get_set(total_reactions, "id")), 4) self.assertEqual(len(self.get_set(total_reactions, "message")), 1)
def get_hipchat_sender_id(realm_id: int, slim_mode: bool, message_dict: Dict[str, Any], user_id_mapper: IdMapper, user_handler: UserHandler) -> Optional[int]: ''' The HipChat export is inconsistent in how it renders senders, and sometimes we don't even get an id. ''' if isinstance(message_dict['sender'], str): if slim_mode: return None # Some Hipchat instances just give us a person's # name in the sender field for NotificationMessage. # We turn them into a mirror user. mirror_user = user_handler.get_mirror_user( realm_id=realm_id, name=message_dict['sender'], ) sender_id = mirror_user['id'] return sender_id raw_sender_id = message_dict['sender']['id'] if raw_sender_id == 0: if slim_mode: return None mirror_user = user_handler.get_mirror_user( realm_id=realm_id, name=message_dict['sender']['name'] ) sender_id = mirror_user['id'] return sender_id if not user_id_mapper.has(raw_sender_id): if slim_mode: return None mirror_user = user_handler.get_mirror_user( realm_id=realm_id, name=message_dict['sender']['id'] ) sender_id = mirror_user['id'] return sender_id # HAPPY PATH: Hipchat just gave us an ordinary # sender_id. sender_id = user_id_mapper.get(raw_sender_id) return sender_id
def test_process_message_attachment(self) -> None: fixture_dir_name = self.fixture_file_name("", "rocketchat_fixtures") rocketchat_data = rocketchat_data_to_dict(fixture_dir_name) output_dir = self.make_import_output_dir("mattermost") user_id_to_user_map = map_user_id_to_user(rocketchat_data["user"]) realm_id = 3 domain_name = "zulip.com" user_handler = UserHandler() user_id_mapper = IdMapper() process_users( user_id_to_user_map=user_id_to_user_map, realm_id=realm_id, domain_name=domain_name, user_handler=user_handler, user_id_mapper=user_id_mapper, ) zerver_attachments: List[ZerverFieldsT] = [] uploads_list: List[ZerverFieldsT] = [] upload_id_to_upload_data_map = map_upload_id_to_upload_data( rocketchat_data["upload"]) message_with_attachment = rocketchat_data["message"][55] process_message_attachment( upload=message_with_attachment["file"], realm_id=3, message_id=1, user_id=3, user_handler=user_handler, zerver_attachment=zerver_attachments, uploads_list=uploads_list, upload_id_to_upload_data_map=upload_id_to_upload_data_map, output_dir=output_dir, ) self.assert_length(zerver_attachments, 1) self.assertEqual(zerver_attachments[0]["file_name"], "harry-ron.jpg") self.assertEqual(zerver_attachments[0]["owner"], 3) self.assertEqual( user_handler.get_user(zerver_attachments[0]["owner"])["email"], "*****@*****.**") # TODO: Assert this for False after fixing the file permissions in PMs self.assertTrue(zerver_attachments[0]["is_realm_public"]) self.assert_length(uploads_list, 1) self.assertEqual(uploads_list[0]["user_profile_email"], "*****@*****.**") attachment_out_path = os.path.join(output_dir, "uploads", zerver_attachments[0]["path_id"]) self.assertTrue(os.path.exists(attachment_out_path)) self.assertTrue(os.path.isfile(attachment_out_path))
def build_reactions( realm_id: int, total_reactions: List[ZerverFieldsT], reactions: List[ZerverFieldsT], message_id: int, user_id_mapper: IdMapper, zerver_realmemoji: List[ZerverFieldsT], ) -> None: realmemoji = {} for realm_emoji in zerver_realmemoji: realmemoji[realm_emoji["name"]] = realm_emoji["id"] # For the Unicode emoji codes, we use equivalent of # function 'emoji_name_to_emoji_code' in 'zerver/lib/emoji' here for mattermost_reaction in reactions: emoji_name = mattermost_reaction["emoji_name"] username = mattermost_reaction["user"] # Check in Unicode emoji if emoji_name in name_to_codepoint: emoji_code = name_to_codepoint[emoji_name] reaction_type = Reaction.UNICODE_EMOJI # Check in realm emoji elif emoji_name in realmemoji: emoji_code = realmemoji[emoji_name] reaction_type = Reaction.REALM_EMOJI else: # nocoverage continue if not user_id_mapper.has(username): continue reaction_id = NEXT_ID("reaction") reaction = Reaction( id=reaction_id, emoji_code=emoji_code, emoji_name=emoji_name, reaction_type=reaction_type, ) reaction_dict = model_to_dict(reaction, exclude=["message", "user_profile"]) reaction_dict["message"] = message_id reaction_dict["user_profile"] = user_id_mapper.get(username) total_reactions.append(reaction_dict)
def test_convert_user_data(self) -> None: user_id_mapper = IdMapper() realm_id = 3 fixture_file_name = self.fixture_file_name("export.json", "mattermost_fixtures") mattermost_data = mattermost_data_file_to_dict(fixture_file_name) username_to_user = create_username_to_user_mapping( mattermost_data["user"]) reset_mirror_dummy_users(username_to_user) team_name = "gryffindor" user_handler = UserHandler() convert_user_data(user_handler, user_id_mapper, username_to_user, realm_id, team_name) self.assert_length(user_handler.get_all_users(), 2) self.assertTrue(user_id_mapper.has("harry")) self.assertTrue(user_id_mapper.has("ron")) self.assertEqual( user_handler.get_user(user_id_mapper.get("harry"))["full_name"], "Harry Potter") self.assertEqual( user_handler.get_user(user_id_mapper.get("ron"))["full_name"], "Ron Weasley") team_name = "slytherin" user_handler = UserHandler() convert_user_data(user_handler, user_id_mapper, username_to_user, realm_id, team_name) self.assert_length(user_handler.get_all_users(), 3) self.assertTrue(user_id_mapper.has("malfoy")) self.assertTrue(user_id_mapper.has("pansy")) self.assertTrue(user_id_mapper.has("snape")) team_name = "gryffindor" # Snape is a mirror dummy user in Harry's team. label_mirror_dummy_users(2, team_name, mattermost_data, username_to_user) user_handler = UserHandler() convert_user_data(user_handler, user_id_mapper, username_to_user, realm_id, team_name) self.assert_length(user_handler.get_all_users(), 3) self.assertTrue(user_id_mapper.has("snape")) team_name = "slytherin" user_handler = UserHandler() convert_user_data(user_handler, user_id_mapper, username_to_user, realm_id, team_name) self.assert_length(user_handler.get_all_users(), 3)
def test_get_mentioned_user_ids(self) -> None: user_id_mapper = IdMapper() harry_id = user_id_mapper.get("harry") raw_message = {"content": "Hello @harry"} ids = get_mentioned_user_ids(raw_message, user_id_mapper) self.assertEqual(list(ids), [harry_id]) raw_message = {"content": "Hello"} ids = get_mentioned_user_ids(raw_message, user_id_mapper) self.assertEqual(list(ids), []) raw_message = {"content": "@harry How are you?"} ids = get_mentioned_user_ids(raw_message, user_id_mapper) self.assertEqual(list(ids), [harry_id]) raw_message = {"content": "@harry @ron Where are you folks?"} ron_id = user_id_mapper.get("ron") ids = get_mentioned_user_ids(raw_message, user_id_mapper) self.assertEqual(list(ids), [harry_id, ron_id]) raw_message = {"content": "@harry.com How are you?"} ids = get_mentioned_user_ids(raw_message, user_id_mapper) self.assertEqual(list(ids), []) raw_message = {"content": "[email protected] How are you?"} ids = get_mentioned_user_ids(raw_message, user_id_mapper) self.assertEqual(list(ids), []) harry_id = user_id_mapper.get("harry_") raw_message = {"content": "Hello @harry_"} ids = get_mentioned_user_ids(raw_message, user_id_mapper) self.assertEqual(list(ids), [harry_id]) harry_id = user_id_mapper.get("harry.") raw_message = {"content": "Hello @harry."} ids = get_mentioned_user_ids(raw_message, user_id_mapper) self.assertEqual(list(ids), [harry_id]) harry_id = user_id_mapper.get("ha_rry.") raw_message = {"content": "Hello @ha_rry."} ids = get_mentioned_user_ids(raw_message, user_id_mapper) self.assertEqual(list(ids), [harry_id]) ron_id = user_id_mapper.get("ron") raw_message = {"content": "Hello @ron."} ids = get_mentioned_user_ids(raw_message, user_id_mapper) self.assertEqual(list(ids), []) raw_message = {"content": "Hello @ron_"} ids = get_mentioned_user_ids(raw_message, user_id_mapper) self.assertEqual(list(ids), [])
def convert_avatar_data(avatar_folder: str, raw_data: List[ZerverFieldsT], user_id_mapper: IdMapper, realm_id: int) -> List[ZerverFieldsT]: ''' This code is pretty specific to how Hipchat sends us data. They give us the avatar payloads in base64 in users.json. We process avatars in our own pass of that data, rather than doing it while we're getting other user data. I chose to keep this separate, as otherwise you have a lot of extraneous data getting passed around. This code has MAJOR SIDE EFFECTS--namely writing a bunch of files to the avatars directory. ''' avatar_records = [] for d in raw_data: raw_user = d['User'] avatar_payload = raw_user.get('avatar') if not avatar_payload: continue bits = base64.b64decode(avatar_payload) raw_user_id = raw_user['id'] if not user_id_mapper.has(raw_user_id): continue user_id = user_id_mapper.get(raw_user_id) metadata = write_avatar_png( avatar_folder=avatar_folder, realm_id=realm_id, user_id=user_id, bits=bits, ) avatar_records.append(metadata) return avatar_records
def test_convert_huddle_data(self) -> None: fixture_file_name = self.fixture_file_name( "export.json", "mattermost_fixtures/direct_channel") mattermost_data = mattermost_data_file_to_dict(fixture_file_name) username_to_user = create_username_to_user_mapping( mattermost_data["user"]) reset_mirror_dummy_users(username_to_user) user_handler = UserHandler() subscriber_handler = SubscriberHandler() huddle_id_mapper = IdMapper() user_id_mapper = IdMapper() team_name = "gryffindor" convert_user_data( user_handler=user_handler, user_id_mapper=user_id_mapper, user_data_map=username_to_user, realm_id=3, team_name=team_name, ) zerver_huddle = convert_huddle_data( huddle_data=mattermost_data["direct_channel"], user_data_map=username_to_user, subscriber_handler=subscriber_handler, huddle_id_mapper=huddle_id_mapper, user_id_mapper=user_id_mapper, realm_id=3, team_name=team_name, ) self.assert_length(zerver_huddle, 1) huddle_members = mattermost_data["direct_channel"][1]["members"] huddle_name = generate_huddle_name(huddle_members) self.assertTrue(huddle_id_mapper.has(huddle_name)) self.assertEqual( subscriber_handler.get_users( huddle_id=huddle_id_mapper.get(huddle_name)), {1, 2, 3})
def get_topic_name( message: Dict[str, Any], dsc_id_to_dsc_map: Dict[str, Dict[str, Any]], thread_id_mapper: IdMapper, is_pm_data: bool = False, ) -> str: if is_pm_data: return "" elif message["rid"] in dsc_id_to_dsc_map: dsc_channel_name = dsc_id_to_dsc_map[message["rid"]]["fname"] return f"{dsc_channel_name} (Imported from Rocket.Chat)" elif message.get("replies"): # Message is the start of a thread thread_id = thread_id_mapper.get(message["_id"]) return f"Thread {thread_id} (Imported from Rocket.Chat)" elif message.get("tmid"): # Message is a part of a thread thread_id = thread_id_mapper.get(message["tmid"]) return f"Thread {thread_id} (Imported from Rocket.Chat)" else: # Normal channel message return "Imported from Rocket.Chat"
def process_user( user_dict: Dict[str, Any], realm_id: int, team_name: str, user_id_mapper: IdMapper ) -> ZerverFieldsT: def is_team_admin(user_dict: Dict[str, Any]) -> bool: if user_dict["teams"] is None: return False for team in user_dict["teams"]: if team["name"] == team_name and "team_admin" in team["roles"]: return True return False def get_full_name(user_dict: Dict[str, Any]) -> str: full_name = "{} {}".format(user_dict["first_name"], user_dict["last_name"]) if full_name.strip(): return full_name return user_dict["username"] avatar_source = "G" full_name = get_full_name(user_dict) id = user_id_mapper.get(user_dict["username"]) delivery_email = user_dict["email"] email = user_dict["email"] short_name = user_dict["username"] date_joined = int(timezone_now().timestamp()) timezone = "UTC" role = UserProfile.ROLE_MEMBER if is_team_admin(user_dict): role = UserProfile.ROLE_REALM_OWNER if user_dict["is_mirror_dummy"]: is_active = False is_mirror_dummy = True else: is_active = True is_mirror_dummy = False return build_user_profile( avatar_source=avatar_source, date_joined=date_joined, delivery_email=delivery_email, email=email, full_name=full_name, id=id, is_active=is_active, role=role, is_mirror_dummy=is_mirror_dummy, realm_id=realm_id, short_name=short_name, timezone=timezone, )
def convert_huddle_data( huddle_id_to_huddle_map: Dict[str, Dict[str, Any]], huddle_id_mapper: IdMapper, user_id_mapper: IdMapper, subscriber_handler: SubscriberHandler, ) -> List[ZerverFieldsT]: zerver_huddle: List[ZerverFieldsT] = [] for rc_huddle_id in huddle_id_to_huddle_map: huddle_id = huddle_id_mapper.get(rc_huddle_id) huddle = build_huddle(huddle_id) zerver_huddle.append(huddle) huddle_dict = huddle_id_to_huddle_map[rc_huddle_id] huddle_user_ids = set() for rc_user_id in huddle_dict["uids"]: huddle_user_ids.add(user_id_mapper.get(rc_user_id)) subscriber_handler.set_info( users=huddle_user_ids, huddle_id=huddle_id, ) return zerver_huddle
def convert_channel_data( room_id_to_room_map: Dict[str, Dict[str, Any]], team_id_to_team_map: Dict[str, Dict[str, Any]], stream_id_mapper: IdMapper, realm_id: int, ) -> List[ZerverFieldsT]: streams = [] for rc_room_id in room_id_to_room_map: channel_dict = room_id_to_room_map[rc_room_id] date_created = float(channel_dict["ts"].timestamp()) stream_id = stream_id_mapper.get(rc_room_id) invite_only = channel_dict["t"] == "p" stream_name = channel_dict["name"] stream_desc = channel_dict.get("description", "") if channel_dict.get("teamId"): if channel_dict.get("teamMain") is True: # In case you change this, please also change the stream name # used while adding the Rocket.Chat channel mention data to # message_dict in `message_to_dict` inner-function of # `process_messages` function below. stream_name = "[TEAM] " + stream_name else: stream_desc = "[Team {} channel]. {}".format( team_id_to_team_map[channel_dict["teamId"]]["name"], stream_desc) # If the channel is read-only, then only admins and moderators # should be allowed to post in the converted Zulip stream. # For more details: https://zulip.com/help/stream-sending-policy # # See `Stream` model in `zerver/models.py` to know about what each # number represent. stream_post_policy = 4 if channel_dict.get("ro", False) else 1 stream = build_stream( date_created=date_created, realm_id=realm_id, name=stream_name, description=stream_desc, stream_id=stream_id, deactivated=False, invite_only=invite_only, stream_post_policy=stream_post_policy, ) streams.append(stream) return streams
def test_process_user(self) -> None: user_id_mapper = IdMapper() fixture_file_name = self.fixture_file_name("export.json", "mattermost_fixtures") mattermost_data = mattermost_data_file_to_dict(fixture_file_name) username_to_user = create_username_to_user_mapping( mattermost_data["user"]) reset_mirror_dummy_users(username_to_user) harry_dict = username_to_user["harry"] harry_dict["is_mirror_dummy"] = False realm_id = 3 team_name = "gryffindor" user = process_user(harry_dict, realm_id, team_name, user_id_mapper) self.assertEqual(user["avatar_source"], "G") self.assertEqual(user["delivery_email"], "*****@*****.**") self.assertEqual(user["email"], "*****@*****.**") self.assertEqual(user["full_name"], "Harry Potter") self.assertEqual(user["id"], 1) self.assertEqual(user["is_active"], True) self.assertEqual(user["role"], UserProfile.ROLE_REALM_OWNER) self.assertEqual(user["is_mirror_dummy"], False) self.assertEqual(user["realm"], 3) self.assertEqual(user["short_name"], "harry") self.assertEqual(user["timezone"], "UTC") # A user with a `null` team value shouldn't be an admin. harry_dict["teams"] = None user = process_user(harry_dict, realm_id, team_name, user_id_mapper) self.assertEqual(user["role"], UserProfile.ROLE_MEMBER) team_name = "slytherin" snape_dict = username_to_user["snape"] snape_dict["is_mirror_dummy"] = True user = process_user(snape_dict, realm_id, team_name, user_id_mapper) self.assertEqual(user["avatar_source"], "G") self.assertEqual(user["delivery_email"], "*****@*****.**") self.assertEqual(user["email"], "*****@*****.**") self.assertEqual(user["full_name"], "Severus Snape") self.assertEqual(user["id"], 2) self.assertEqual(user["is_active"], False) self.assertEqual(user["role"], UserProfile.ROLE_MEMBER) self.assertEqual(user["is_mirror_dummy"], True) self.assertEqual(user["realm"], 3) self.assertEqual(user["short_name"], "snape") self.assertEqual(user["timezone"], "UTC")
def process_user(user_dict: Dict[str, Any], realm_id: int, team_name: str, user_id_mapper: IdMapper) -> ZerverFieldsT: def is_team_admin(user_dict: Dict[str, Any]) -> bool: for team in user_dict["teams"]: if team["name"] == team_name and "team_admin" in team["roles"]: return True return False def get_full_name(user_dict: Dict[str, Any]) -> str: full_name = "{} {}".format(user_dict["first_name"], user_dict["last_name"]) if full_name.strip(): return full_name return user_dict['username'] avatar_source = 'G' full_name = get_full_name(user_dict) id = user_id_mapper.get(user_dict['username']) delivery_email = user_dict['email'] email = user_dict['email'] is_realm_admin = is_team_admin(user_dict) is_guest = False short_name = user_dict['username'] date_joined = int(timezone_now().timestamp()) timezone = 'UTC' if user_dict["is_mirror_dummy"]: is_active = False is_mirror_dummy = True else: is_active = True is_mirror_dummy = False return build_user_profile( avatar_source=avatar_source, date_joined=date_joined, delivery_email=delivery_email, email=email, full_name=full_name, id=id, is_active=is_active, is_realm_admin=is_realm_admin, is_guest=is_guest, is_mirror_dummy=is_mirror_dummy, realm_id=realm_id, short_name=short_name, timezone=timezone, )
def test_process_user(self) -> None: user_id_mapper = IdMapper() harry_dict = self.username_to_user["harry"] harry_dict["is_mirror_dummy"] = False realm_id = 3 team_name = "gryffindor" user = process_user(harry_dict, realm_id, team_name, user_id_mapper) self.assertEqual(user["avatar_source"], 'G') self.assertEqual(user["delivery_email"], "*****@*****.**") self.assertEqual(user["email"], "*****@*****.**") self.assertEqual(user["full_name"], "Harry Potter") self.assertEqual(user["id"], 1) self.assertEqual(user["is_active"], True) self.assertEqual(user["is_realm_admin"], True) self.assertEqual(user["is_guest"], False) self.assertEqual(user["is_mirror_dummy"], False) self.assertEqual(user["realm"], 3) self.assertEqual(user["short_name"], "harry") self.assertEqual(user["timezone"], "UTC") # A user with a `null` team value shouldn't be an admin. harry_dict["teams"] = None user = process_user(harry_dict, realm_id, team_name, user_id_mapper) self.assertEqual(user["is_realm_admin"], False) team_name = "slytherin" snape_dict = self.username_to_user["snape"] snape_dict["is_mirror_dummy"] = True user = process_user(snape_dict, realm_id, team_name, user_id_mapper) self.assertEqual(user["avatar_source"], 'G') self.assertEqual(user["delivery_email"], "*****@*****.**") self.assertEqual(user["email"], "*****@*****.**") self.assertEqual(user["full_name"], "Severus Snape") self.assertEqual(user["id"], 2) self.assertEqual(user["is_active"], False) self.assertEqual(user["is_realm_admin"], False) self.assertEqual(user["is_guest"], False) self.assertEqual(user["is_mirror_dummy"], True) self.assertEqual(user["realm"], 3) self.assertEqual(user["short_name"], "snape") self.assertEqual(user["timezone"], "UTC")
def convert_channel_data( room_id_to_room_map: Dict[str, Dict[str, Any]], team_id_to_team_map: Dict[str, Dict[str, Any]], stream_id_mapper: IdMapper, realm_id: int, ) -> List[ZerverFieldsT]: streams = [] for rc_room_id in room_id_to_room_map: channel_dict = room_id_to_room_map[rc_room_id] date_created = float(channel_dict["ts"].timestamp()) stream_id = stream_id_mapper.get(rc_room_id) invite_only = channel_dict["t"] == "p" stream_name = get_stream_name(channel_dict) stream_desc = channel_dict.get("description", "") if channel_dict.get("teamId") and not channel_dict.get("teamMain"): stream_desc = "[Team {} channel]. {}".format( team_id_to_team_map[channel_dict["teamId"]]["name"], stream_desc) # If the channel is read-only, then only admins and moderators # should be allowed to post in the converted Zulip stream. # For more details: https://zulip.com/help/stream-sending-policy # # See `Stream` model in `zerver/models.py` to know about what each # number represent. stream_post_policy = 4 if channel_dict.get("ro", False) else 1 stream = build_stream( date_created=date_created, realm_id=realm_id, name=stream_name, description=stream_desc, stream_id=stream_id, deactivated=False, invite_only=invite_only, stream_post_policy=stream_post_policy, ) streams.append(stream) return streams
def test_convert_channel_data(self) -> None: fixture_file_name = self.fixture_file_name("export.json", "mattermost_fixtures") mattermost_data = mattermost_data_file_to_dict(fixture_file_name) username_to_user = create_username_to_user_mapping(mattermost_data["user"]) reset_mirror_dummy_users(username_to_user) user_handler = UserHandler() subscriber_handler = SubscriberHandler() stream_id_mapper = IdMapper() user_id_mapper = IdMapper() team_name = "gryffindor" convert_user_data( user_handler=user_handler, user_id_mapper=user_id_mapper, user_data_map=username_to_user, realm_id=3, team_name=team_name, ) zerver_stream = convert_channel_data( channel_data=mattermost_data["channel"], user_data_map=username_to_user, subscriber_handler=subscriber_handler, stream_id_mapper=stream_id_mapper, user_id_mapper=user_id_mapper, realm_id=3, team_name=team_name, ) self.assertEqual(len(zerver_stream), 3) self.assertEqual(zerver_stream[0]["name"], "Gryffindor common room") self.assertEqual(zerver_stream[0]["invite_only"], False) self.assertEqual(zerver_stream[0]["description"], "A place for talking about Gryffindor common room") self.assertEqual(zerver_stream[0]["rendered_description"], "") self.assertEqual(zerver_stream[0]["realm"], 3) self.assertEqual(zerver_stream[1]["name"], "Gryffindor quidditch team") self.assertEqual(zerver_stream[1]["invite_only"], False) self.assertEqual(zerver_stream[1]["description"], "A place for talking about Gryffindor quidditch team") self.assertEqual(zerver_stream[1]["rendered_description"], "") self.assertEqual(zerver_stream[1]["realm"], 3) self.assertEqual(zerver_stream[2]["name"], "Dumbledores army") self.assertEqual(zerver_stream[2]["invite_only"], True) self.assertEqual(zerver_stream[2]["description"], "A place for talking about Dumbledores army") self.assertEqual(zerver_stream[2]["rendered_description"], "") self.assertEqual(zerver_stream[2]["realm"], 3) self.assertTrue(stream_id_mapper.has("gryffindor-common-room")) self.assertTrue(stream_id_mapper.has("gryffindor-quidditch-team")) self.assertTrue(stream_id_mapper.has("dumbledores-army")) # TODO: Add ginny ron_id = user_id_mapper.get("ron") harry_id = user_id_mapper.get("harry") self.assertEqual({ron_id, harry_id}, {1, 2}) self.assertEqual(subscriber_handler.get_users(stream_id=stream_id_mapper.get("gryffindor-common-room")), {ron_id, harry_id}) self.assertEqual(subscriber_handler.get_users(stream_id=stream_id_mapper.get("gryffindor-quidditch-team")), {ron_id, harry_id}) self.assertEqual(subscriber_handler.get_users(stream_id=stream_id_mapper.get("dumbledores-army")), {ron_id, harry_id}) # Converting channel data when a user's `teams` value is `null`. username_to_user["ron"].update({"teams": None}) zerver_stream = convert_channel_data( channel_data=mattermost_data["channel"], user_data_map=username_to_user, subscriber_handler=subscriber_handler, stream_id_mapper=stream_id_mapper, user_id_mapper=user_id_mapper, realm_id=3, team_name=team_name, ) harry_id = user_id_mapper.get("harry") self.assertIn(harry_id, {1, 2}) self.assertEqual(subscriber_handler.get_users(stream_id=stream_id_mapper.get("gryffindor-common-room")), {harry_id}) self.assertEqual(subscriber_handler.get_users(stream_id=stream_id_mapper.get("gryffindor-quidditch-team")), {harry_id}) self.assertEqual(subscriber_handler.get_users(stream_id=stream_id_mapper.get("dumbledores-army")), {harry_id}) team_name = "slytherin" zerver_stream = convert_channel_data( channel_data=mattermost_data["channel"], user_data_map=username_to_user, subscriber_handler=subscriber_handler, stream_id_mapper=stream_id_mapper, user_id_mapper=user_id_mapper, realm_id=4, team_name=team_name, ) malfoy_id = user_id_mapper.get("malfoy") pansy_id = user_id_mapper.get("pansy") snape_id = user_id_mapper.get("snape") self.assertEqual({malfoy_id, pansy_id, snape_id}, {3, 4, 5}) self.assertEqual(subscriber_handler.get_users(stream_id=stream_id_mapper.get("slytherin-common-room")), {malfoy_id, pansy_id, snape_id}) self.assertEqual(subscriber_handler.get_users(stream_id=stream_id_mapper.get("slytherin-quidditch-team")), {malfoy_id, pansy_id})
def do_convert_data(input_tar_file: str, output_dir: str, masking_content: bool, api_token: Optional[str] = None, slim_mode: bool = False) -> None: input_data_dir = untar_input_file(input_tar_file) attachment_handler = AttachmentHandler() user_handler = UserHandler() subscriber_handler = SubscriberHandler() user_id_mapper = IdMapper() stream_id_mapper = IdMapper() realm_id = 0 realm = make_realm(realm_id=realm_id) # users.json -> UserProfile raw_user_data = read_user_data(data_dir=input_data_dir) convert_user_data( user_handler=user_handler, slim_mode=slim_mode, user_id_mapper=user_id_mapper, raw_data=raw_user_data, realm_id=realm_id, ) normal_users = user_handler.get_normal_users() # Don't write zerver_userprofile here, because we # may add more users later. # streams.json -> Stream raw_stream_data = read_room_data(data_dir=input_data_dir) zerver_stream = convert_room_data( raw_data=raw_stream_data, subscriber_handler=subscriber_handler, stream_id_mapper=stream_id_mapper, user_id_mapper=user_id_mapper, realm_id=realm_id, api_token=api_token, ) realm['zerver_stream'] = zerver_stream zerver_recipient = build_recipients( zerver_userprofile=normal_users, zerver_stream=zerver_stream, ) realm['zerver_recipient'] = zerver_recipient if api_token is None: if slim_mode: public_stream_subscriptions: List[ZerverFieldsT] = [] else: public_stream_subscriptions = build_public_stream_subscriptions( zerver_userprofile=normal_users, zerver_recipient=zerver_recipient, zerver_stream=zerver_stream, ) private_stream_subscriptions = build_stream_subscriptions( get_users=subscriber_handler.get_users, zerver_recipient=zerver_recipient, zerver_stream=[ stream_dict for stream_dict in zerver_stream if stream_dict['invite_only'] ], ) stream_subscriptions = public_stream_subscriptions + private_stream_subscriptions else: stream_subscriptions = build_stream_subscriptions( get_users=subscriber_handler.get_users, zerver_recipient=zerver_recipient, zerver_stream=zerver_stream, ) personal_subscriptions = build_personal_subscriptions( zerver_recipient=zerver_recipient, ) zerver_subscription = personal_subscriptions + stream_subscriptions realm['zerver_subscription'] = zerver_subscription zerver_realmemoji = write_emoticon_data( realm_id=realm_id, data_dir=input_data_dir, output_dir=output_dir, ) realm['zerver_realmemoji'] = zerver_realmemoji subscriber_map = make_subscriber_map( zerver_subscription=zerver_subscription, ) logging.info('Start importing message data') for message_key in [ 'UserMessage', 'NotificationMessage', 'PrivateUserMessage' ]: write_message_data( realm_id=realm_id, slim_mode=slim_mode, message_key=message_key, zerver_recipient=zerver_recipient, subscriber_map=subscriber_map, data_dir=input_data_dir, output_dir=output_dir, masking_content=masking_content, stream_id_mapper=stream_id_mapper, user_id_mapper=user_id_mapper, user_handler=user_handler, attachment_handler=attachment_handler, ) # Order is important here...don't write users until # we process everything else, since we may introduce # mirror users when processing messages. realm['zerver_userprofile'] = user_handler.get_all_users() realm['sort_by_date'] = True create_converted_data_files(realm, output_dir, '/realm.json') logging.info('Start importing avatar data') write_avatar_data( raw_user_data=raw_user_data, output_dir=output_dir, user_id_mapper=user_id_mapper, realm_id=realm_id, ) attachment_handler.write_info( output_dir=output_dir, realm_id=realm_id, ) logging.info('Start making tarball') subprocess.check_call( ["tar", "-czf", output_dir + '.tar.gz', output_dir, '-P']) logging.info('Done making tarball')
def process_raw_message_batch(realm_id: int, raw_messages: List[Dict[str, Any]], subscriber_map: Dict[int, Set[int]], user_id_mapper: IdMapper, user_handler: UserHandler, attachment_handler: AttachmentHandler, get_recipient_id: Callable[[ZerverFieldsT], int], is_pm_data: bool, output_dir: str) -> None: def fix_mentions(content: str, mention_user_ids: Set[int]) -> str: for user_id in mention_user_ids: user = user_handler.get_user(user_id=user_id) hipchat_mention = '@{short_name}'.format(**user) zulip_mention = '@**{full_name}**'.format(**user) content = content.replace(hipchat_mention, zulip_mention) content = content.replace('@here', '@**all**') return content mention_map: Dict[int, Set[int]] = dict() zerver_message = [] import html2text h = html2text.HTML2Text() for raw_message in raw_messages: # One side effect here: message_id = NEXT_ID('message') mention_user_ids = { user_id_mapper.get(id) for id in set(raw_message['mention_user_ids']) if user_id_mapper.has(id) } mention_map[message_id] = mention_user_ids content = fix_mentions( content=raw_message['content'], mention_user_ids=mention_user_ids, ) content = h.handle(content) if len(content) > 10000: logging.info('skipping too-long message of length %s' % (len(content), )) continue date_sent = raw_message['date_sent'] try: recipient_id = get_recipient_id(raw_message) except KeyError: logging.debug( "Could not find recipient_id for a message, skipping.") continue rendered_content = None if is_pm_data: topic_name = '' else: topic_name = 'imported from hipchat' user_id = raw_message['sender_id'] # Another side effect: extra_content = attachment_handler.handle_message_data( realm_id=realm_id, message_id=message_id, sender_id=user_id, attachment=raw_message['attachment'], files_dir=raw_message['files_dir'], ) if extra_content: has_attachment = True content += '\n' + extra_content else: has_attachment = False message = build_message( content=content, message_id=message_id, date_sent=date_sent, recipient_id=recipient_id, rendered_content=rendered_content, topic_name=topic_name, user_id=user_id, has_attachment=has_attachment, ) zerver_message.append(message) zerver_usermessage = make_user_messages( zerver_message=zerver_message, subscriber_map=subscriber_map, is_pm_data=is_pm_data, mention_map=mention_map, ) message_json = dict( zerver_message=zerver_message, zerver_usermessage=zerver_usermessage, ) dump_file_id = NEXT_ID('dump_file_id') message_file = "/messages-%06d.json" % (dump_file_id, ) create_converted_data_files(message_json, output_dir, message_file)
def convert_room_data(raw_data: List[ZerverFieldsT], subscriber_handler: SubscriberHandler, stream_id_mapper: IdMapper, user_id_mapper: IdMapper, realm_id: int, api_token: Optional[str] = None) -> List[ZerverFieldsT]: flat_data = [d['Room'] for d in raw_data] def get_invite_only(v: str) -> bool: if v == 'public': return False elif v == 'private': return True else: raise Exception('unexpected value') streams = [] for in_dict in flat_data: now = int(timezone_now().timestamp()) stream_id = stream_id_mapper.get(in_dict['id']) invite_only = get_invite_only(in_dict['privacy']) stream = build_stream( date_created=now, realm_id=realm_id, name=in_dict['name'], description=in_dict['topic'], stream_id=stream_id, deactivated=in_dict['is_archived'], invite_only=invite_only, ) if invite_only: users: Set[int] = { user_id_mapper.get(key) for key in in_dict['members'] if user_id_mapper.has(key) } if user_id_mapper.has(in_dict['owner']): owner = user_id_mapper.get(in_dict['owner']) users.add(owner) else: users = set() if api_token is not None: hc = hypchat.HypChat(api_token) room_data = hc.fromurl('{}/v2/room/{}/member'.format( hc.endpoint, in_dict['id'])) for item in room_data['items']: hipchat_user_id = item['id'] zulip_user_id = user_id_mapper.get(hipchat_user_id) users.add(zulip_user_id) if users: subscriber_handler.set_info( stream_id=stream_id, users=users, ) # unmapped fields: # guest_access_url: no Zulip equivalent # created: we just use "now" # participants: no good sample data streams.append(stream) return streams
def do_convert_data(mattermost_data_dir: str, output_dir: str, masking_content: bool) -> None: username_to_user: Dict[str, Dict[str, Any]] = {} os.makedirs(output_dir, exist_ok=True) if os.listdir(output_dir): # nocoverage raise Exception("Output directory should be empty!") mattermost_data_file = os.path.join(mattermost_data_dir, "export.json") mattermost_data = mattermost_data_file_to_dict(mattermost_data_file) username_to_user = create_username_to_user_mapping(mattermost_data["user"]) for team in mattermost_data["team"]: realm_id = NEXT_ID("realm_id") team_name = team["name"] user_handler = UserHandler() subscriber_handler = SubscriberHandler() user_id_mapper = IdMapper() stream_id_mapper = IdMapper() huddle_id_mapper = IdMapper() print("Generating data for", team_name) realm = make_realm(realm_id, team) realm_output_dir = os.path.join(output_dir, team_name) reset_mirror_dummy_users(username_to_user) label_mirror_dummy_users( len(mattermost_data["team"]), team_name, mattermost_data, username_to_user ) convert_user_data( user_handler=user_handler, user_id_mapper=user_id_mapper, user_data_map=username_to_user, realm_id=realm_id, team_name=team_name, ) zerver_stream = convert_channel_data( channel_data=mattermost_data["channel"], user_data_map=username_to_user, subscriber_handler=subscriber_handler, stream_id_mapper=stream_id_mapper, user_id_mapper=user_id_mapper, realm_id=realm_id, team_name=team_name, ) realm["zerver_stream"] = zerver_stream zerver_huddle: List[ZerverFieldsT] = [] if len(mattermost_data["team"]) == 1: zerver_huddle = convert_huddle_data( huddle_data=mattermost_data["direct_channel"], user_data_map=username_to_user, subscriber_handler=subscriber_handler, huddle_id_mapper=huddle_id_mapper, user_id_mapper=user_id_mapper, realm_id=realm_id, team_name=team_name, ) realm["zerver_huddle"] = zerver_huddle all_users = user_handler.get_all_users() zerver_recipient = build_recipients( zerver_userprofile=all_users, zerver_stream=zerver_stream, zerver_huddle=zerver_huddle, ) realm["zerver_recipient"] = zerver_recipient stream_subscriptions = build_stream_subscriptions( get_users=subscriber_handler.get_users, zerver_recipient=zerver_recipient, zerver_stream=zerver_stream, ) huddle_subscriptions = build_huddle_subscriptions( get_users=subscriber_handler.get_users, zerver_recipient=zerver_recipient, zerver_huddle=zerver_huddle, ) personal_subscriptions = build_personal_subscriptions( zerver_recipient=zerver_recipient, ) # Mattermost currently supports only exporting messages from channels. # Personal messages and huddles are not exported. zerver_subscription = personal_subscriptions + stream_subscriptions + huddle_subscriptions realm["zerver_subscription"] = zerver_subscription zerver_realmemoji = write_emoticon_data( realm_id=realm_id, custom_emoji_data=mattermost_data["emoji"], data_dir=mattermost_data_dir, output_dir=realm_output_dir, ) realm["zerver_realmemoji"] = zerver_realmemoji subscriber_map = make_subscriber_map( zerver_subscription=zerver_subscription, ) total_reactions: List[Dict[str, Any]] = [] uploads_list: List[ZerverFieldsT] = [] zerver_attachment: List[ZerverFieldsT] = [] write_message_data( num_teams=len(mattermost_data["team"]), team_name=team_name, realm_id=realm_id, post_data=mattermost_data["post"], zerver_recipient=zerver_recipient, subscriber_map=subscriber_map, output_dir=realm_output_dir, masking_content=masking_content, stream_id_mapper=stream_id_mapper, huddle_id_mapper=huddle_id_mapper, user_id_mapper=user_id_mapper, user_handler=user_handler, zerver_realmemoji=zerver_realmemoji, total_reactions=total_reactions, uploads_list=uploads_list, zerver_attachment=zerver_attachment, mattermost_data_dir=mattermost_data_dir, ) realm["zerver_reaction"] = total_reactions realm["zerver_userprofile"] = user_handler.get_all_users() realm["sort_by_date"] = True create_converted_data_files(realm, realm_output_dir, "/realm.json") # Mattermost currently doesn't support exporting avatars create_converted_data_files([], realm_output_dir, "/avatars/records.json") # Export message attachments attachment: Dict[str, List[Any]] = {"zerver_attachment": zerver_attachment} create_converted_data_files(uploads_list, realm_output_dir, "/uploads/records.json") create_converted_data_files(attachment, realm_output_dir, "/attachment.json") logging.info("Start making tarball") subprocess.check_call(["tar", "-czf", realm_output_dir + ".tar.gz", realm_output_dir, "-P"]) logging.info("Done making tarball")
def test_sender_ids(self) -> None: realm_id = 5 user_handler = UserHandler() user_id_mapper = IdMapper() user_id_mapper.has = lambda key: True # type: ignore # it's just a stub # Simulate a "normal" user first. user_with_id = dict( id=1, # other fields don't matter here ) user_handler.add_user(user=user_with_id) normal_message = dict( sender=dict( id=1, ) ) # type: Dict[str, Any] sender_id = get_hipchat_sender_id( realm_id=realm_id, slim_mode=False, message_dict=normal_message, user_id_mapper=user_id_mapper, user_handler=user_handler, ) self.assertEqual(sender_id, 1) bot_message = dict( sender='fred_bot', ) # Every message from fred_bot should # return the same sender_id. fred_bot_sender_id = 2 for i in range(3): sender_id = get_hipchat_sender_id( realm_id=realm_id, slim_mode=False, message_dict=bot_message, user_id_mapper=user_id_mapper, user_handler=user_handler, ) self.assertEqual(sender_id, fred_bot_sender_id) id_zero_message = dict( sender=dict( id=0, name='hal_bot', ), ) hal_bot_sender_id = 3 for i in range(3): sender_id = get_hipchat_sender_id( realm_id=realm_id, slim_mode=False, message_dict=id_zero_message, user_id_mapper=user_id_mapper, user_handler=user_handler, ) self.assertEqual(sender_id, hal_bot_sender_id)