def test_convert_user_data(self) -> None:
user_id_mapper = IdMapper()
realm_id = 3
team_name = "gryffindor"
user_handler = UserHandler()
convert_user_data(user_handler, user_id_mapper, self.username_to_user, realm_id, team_name)
self.assertTrue(user_id_mapper.has("harry"))
self.assertTrue(user_id_mapper.has("ron"))
self.assertEqual(user_handler.get_user(user_id_mapper.get("harry"))["full_name"], "Harry Potter")
self.assertEqual(user_handler.get_user(user_id_mapper.get("ron"))["full_name"], "Ron Weasley")
team_name = "slytherin"
user_handler = UserHandler()
convert_user_data(user_handler, user_id_mapper, self.username_to_user, realm_id, team_name)
self.assertEqual(len(user_handler.get_all_users()), 3)
self.assertTrue(user_id_mapper.has("malfoy"))
self.assertTrue(user_id_mapper.has("pansy"))
self.assertTrue(user_id_mapper.has("snape"))
team_name = "gryffindor"
# Snape is a mirror dummy user in Harry's team.
label_mirror_dummy_users(team_name, self.mattermost_data, self.username_to_user)
user_handler = UserHandler()
convert_user_data(user_handler, user_id_mapper, self.username_to_user, realm_id, team_name)
self.assertEqual(len(user_handler.get_all_users()), 3)
self.assertTrue(user_id_mapper.has("snape"))
team_name = "slytherin"
user_handler = UserHandler()
convert_user_data(user_handler, user_id_mapper, self.username_to_user, realm_id, team_name)
self.assertEqual(len(user_handler.get_all_users()), 3)
def convert_room_data(raw_data: List[ZerverFieldsT],
subscriber_handler: SubscriberHandler,
stream_id_mapper: IdMapper, user_id_mapper: IdMapper,
realm_id: int) -> List[ZerverFieldsT]:
flat_data = [d['Room'] for d in raw_data]
def get_invite_only(v: str) -> bool:
if v == 'public':
return False
elif v == 'private':
return True
else:
raise Exception('unexpected value')
streams = []
for in_dict in flat_data:
now = int(timezone_now().timestamp())
stream_id = stream_id_mapper.get(in_dict['id'])
invite_only = get_invite_only(in_dict['privacy'])
stream = build_stream(
date_created=now,
realm_id=realm_id,
name=in_dict['name'],
description=in_dict['topic'],
stream_id=stream_id,
deactivated=in_dict['is_archived'],
invite_only=invite_only,
)
if invite_only:
users = {
user_id_mapper.get(key)
for key in in_dict['members'] if user_id_mapper.has(key)
} # type: Set[int]
if user_id_mapper.has(in_dict['owner']):
owner = user_id_mapper.get(in_dict['owner'])
users.add(owner)
if not users:
continue
subscriber_handler.set_info(
stream_id=stream_id,
users=users,
)
# unmapped fields:
# guest_access_url: no Zulip equivalent
# created: we just use "now"
# participants: no good sample data
streams.append(stream)
return streams
def test_convert_user_data(self) -> None:
user_id_mapper = IdMapper()
realm_id = 3
fixture_file_name = self.fixture_file_name("export.json",
"mattermost_fixtures")
mattermost_data = mattermost_data_file_to_dict(fixture_file_name)
username_to_user = create_username_to_user_mapping(
mattermost_data["user"])
reset_mirror_dummy_users(username_to_user)
team_name = "gryffindor"
user_handler = UserHandler()
convert_user_data(user_handler, user_id_mapper, username_to_user,
realm_id, team_name)
self.assert_length(user_handler.get_all_users(), 2)
self.assertTrue(user_id_mapper.has("harry"))
self.assertTrue(user_id_mapper.has("ron"))
self.assertEqual(
user_handler.get_user(user_id_mapper.get("harry"))["full_name"],
"Harry Potter")
self.assertEqual(
user_handler.get_user(user_id_mapper.get("ron"))["full_name"],
"Ron Weasley")
team_name = "slytherin"
user_handler = UserHandler()
convert_user_data(user_handler, user_id_mapper, username_to_user,
realm_id, team_name)
self.assert_length(user_handler.get_all_users(), 3)
self.assertTrue(user_id_mapper.has("malfoy"))
self.assertTrue(user_id_mapper.has("pansy"))
self.assertTrue(user_id_mapper.has("snape"))
team_name = "gryffindor"
# Snape is a mirror dummy user in Harry's team.
label_mirror_dummy_users(2, team_name, mattermost_data,
username_to_user)
user_handler = UserHandler()
convert_user_data(user_handler, user_id_mapper, username_to_user,
realm_id, team_name)
self.assert_length(user_handler.get_all_users(), 3)
self.assertTrue(user_id_mapper.has("snape"))
team_name = "slytherin"
user_handler = UserHandler()
convert_user_data(user_handler, user_id_mapper, username_to_user,
realm_id, team_name)
self.assert_length(user_handler.get_all_users(), 3)
def test_sender_ids(self) -> None:
realm_id = 5
user_handler = UserHandler()
user_id_mapper = IdMapper()
user_id_mapper.has = lambda key: True # type: ignore[assignment] # it's just a stub
# Simulate a "normal" user first.
user_with_id = dict(id=1,
# other fields don't matter here
)
user_handler.add_user(user=user_with_id)
normal_message: Dict[str, Any] = dict(sender=dict(id=1, ), )
sender_id = get_hipchat_sender_id(
realm_id=realm_id,
slim_mode=False,
message_dict=normal_message,
user_id_mapper=user_id_mapper,
user_handler=user_handler,
)
self.assertEqual(sender_id, 1)
bot_message = dict(sender='fred_bot', )
# Every message from fred_bot should
# return the same sender_id.
fred_bot_sender_id = 2
for i in range(3):
sender_id = get_hipchat_sender_id(
realm_id=realm_id,
slim_mode=False,
message_dict=bot_message,
user_id_mapper=user_id_mapper,
user_handler=user_handler,
)
self.assertEqual(sender_id, fred_bot_sender_id)
id_zero_message = dict(sender=dict(
id=0,
name='hal_bot',
), )
hal_bot_sender_id = 3
for i in range(3):
sender_id = get_hipchat_sender_id(
realm_id=realm_id,
slim_mode=False,
message_dict=id_zero_message,
user_id_mapper=user_id_mapper,
user_handler=user_handler,
)
self.assertEqual(sender_id, hal_bot_sender_id)
def test_convert_huddle_data(self) -> None:
fixture_dir_name = self.fixture_file_name("", "rocketchat_fixtures")
rocketchat_data = rocketchat_data_to_dict(fixture_dir_name)
realm_id = 3
domain_name = "zulip.com"
user_handler = UserHandler()
subscriber_handler = SubscriberHandler()
user_id_mapper = IdMapper()
huddle_id_mapper = IdMapper()
user_id_to_user_map = map_user_id_to_user(rocketchat_data["user"])
process_users(
user_id_to_user_map=user_id_to_user_map,
realm_id=realm_id,
domain_name=domain_name,
user_handler=user_handler,
user_id_mapper=user_id_mapper,
)
room_id_to_room_map: Dict[str, Dict[str, Any]] = {}
team_id_to_team_map: Dict[str, Dict[str, Any]] = {}
dsc_id_to_dsc_map: Dict[str, Dict[str, Any]] = {}
direct_id_to_direct_map: Dict[str, Dict[str, Any]] = {}
huddle_id_to_huddle_map: Dict[str, Dict[str, Any]] = {}
livechat_id_to_livechat_map: Dict[str, Dict[str, Any]] = {}
categorize_channels_and_map_with_id(
channel_data=rocketchat_data["room"],
room_id_to_room_map=room_id_to_room_map,
team_id_to_team_map=team_id_to_team_map,
dsc_id_to_dsc_map=dsc_id_to_dsc_map,
direct_id_to_direct_map=direct_id_to_direct_map,
huddle_id_to_huddle_map=huddle_id_to_huddle_map,
livechat_id_to_livechat_map=livechat_id_to_livechat_map,
)
zerver_huddle = convert_huddle_data(
huddle_id_to_huddle_map=huddle_id_to_huddle_map,
huddle_id_mapper=huddle_id_mapper,
user_id_mapper=user_id_mapper,
subscriber_handler=subscriber_handler,
)
self.assert_length(zerver_huddle, 1)
rc_huddle_id = rocketchat_data["room"][12]["_id"]
self.assertTrue(huddle_id_mapper.has(rc_huddle_id))
huddle_id = huddle_id_mapper.get(rc_huddle_id)
self.assertEqual(subscriber_handler.get_users(huddle_id=huddle_id),
{3, 4, 5})
def get_mentioned_user_ids(raw_message: Dict[str, Any], user_id_mapper: IdMapper) -> Set[int]:
user_ids = set()
content = raw_message["content"]
# usernames can be of the form user.name, user_name, username., username_, user.name_ etc
matches = re.findall("(?<=^|(?<=[^a-zA-Z0-9-_.]))@(([A-Za-z0-9]+[_.]?)+)", content)
for match in matches:
possible_username = match[0]
if user_id_mapper.has(possible_username):
user_ids.add(user_id_mapper.get(possible_username))
return user_ids
def get_hipchat_sender_id(realm_id: int,
slim_mode: bool,
message_dict: Dict[str, Any],
user_id_mapper: IdMapper,
user_handler: UserHandler) -> Optional[int]:
'''
The HipChat export is inconsistent in how it renders
senders, and sometimes we don't even get an id.
'''
if isinstance(message_dict['sender'], str):
if slim_mode:
return None
# Some Hipchat instances just give us a person's
# name in the sender field for NotificationMessage.
# We turn them into a mirror user.
mirror_user = user_handler.get_mirror_user(
realm_id=realm_id,
name=message_dict['sender'],
)
sender_id = mirror_user['id']
return sender_id
raw_sender_id = message_dict['sender']['id']
if raw_sender_id == 0:
if slim_mode:
return None
mirror_user = user_handler.get_mirror_user(
realm_id=realm_id,
name=message_dict['sender']['name']
)
sender_id = mirror_user['id']
return sender_id
if not user_id_mapper.has(raw_sender_id):
if slim_mode:
return None
mirror_user = user_handler.get_mirror_user(
realm_id=realm_id,
name=message_dict['sender']['id']
)
sender_id = mirror_user['id']
return sender_id
# HAPPY PATH: Hipchat just gave us an ordinary
# sender_id.
sender_id = user_id_mapper.get(raw_sender_id)
return sender_id
def build_reactions(
realm_id: int,
total_reactions: List[ZerverFieldsT],
reactions: List[ZerverFieldsT],
message_id: int,
user_id_mapper: IdMapper,
zerver_realmemoji: List[ZerverFieldsT],
) -> None:
realmemoji = {}
for realm_emoji in zerver_realmemoji:
realmemoji[realm_emoji["name"]] = realm_emoji["id"]
# For the Unicode emoji codes, we use equivalent of
# function 'emoji_name_to_emoji_code' in 'zerver/lib/emoji' here
for mattermost_reaction in reactions:
emoji_name = mattermost_reaction["emoji_name"]
username = mattermost_reaction["user"]
# Check in Unicode emoji
if emoji_name in name_to_codepoint:
emoji_code = name_to_codepoint[emoji_name]
reaction_type = Reaction.UNICODE_EMOJI
# Check in realm emoji
elif emoji_name in realmemoji:
emoji_code = realmemoji[emoji_name]
reaction_type = Reaction.REALM_EMOJI
else: # nocoverage
continue
if not user_id_mapper.has(username):
continue
reaction_id = NEXT_ID("reaction")
reaction = Reaction(
id=reaction_id,
emoji_code=emoji_code,
emoji_name=emoji_name,
reaction_type=reaction_type,
)
reaction_dict = model_to_dict(reaction,
exclude=["message", "user_profile"])
reaction_dict["message"] = message_id
reaction_dict["user_profile"] = user_id_mapper.get(username)
total_reactions.append(reaction_dict)
def convert_avatar_data(avatar_folder: str,
raw_data: List[ZerverFieldsT],
user_id_mapper: IdMapper,
realm_id: int) -> List[ZerverFieldsT]:
'''
This code is pretty specific to how Hipchat sends us data.
They give us the avatar payloads in base64 in users.json.
We process avatars in our own pass of that data, rather
than doing it while we're getting other user data. I
chose to keep this separate, as otherwise you have a lot
of extraneous data getting passed around.
This code has MAJOR SIDE EFFECTS--namely writing a bunch
of files to the avatars directory.
'''
avatar_records = []
for d in raw_data:
raw_user = d['User']
avatar_payload = raw_user.get('avatar')
if not avatar_payload:
continue
bits = base64.b64decode(avatar_payload)
raw_user_id = raw_user['id']
if not user_id_mapper.has(raw_user_id):
continue
user_id = user_id_mapper.get(raw_user_id)
metadata = write_avatar_png(
avatar_folder=avatar_folder,
realm_id=realm_id,
user_id=user_id,
bits=bits,
)
avatar_records.append(metadata)
return avatar_records
def test_convert_huddle_data(self) -> None:
fixture_file_name = self.fixture_file_name(
"export.json", "mattermost_fixtures/direct_channel")
mattermost_data = mattermost_data_file_to_dict(fixture_file_name)
username_to_user = create_username_to_user_mapping(
mattermost_data["user"])
reset_mirror_dummy_users(username_to_user)
user_handler = UserHandler()
subscriber_handler = SubscriberHandler()
huddle_id_mapper = IdMapper()
user_id_mapper = IdMapper()
team_name = "gryffindor"
convert_user_data(
user_handler=user_handler,
user_id_mapper=user_id_mapper,
user_data_map=username_to_user,
realm_id=3,
team_name=team_name,
)
zerver_huddle = convert_huddle_data(
huddle_data=mattermost_data["direct_channel"],
user_data_map=username_to_user,
subscriber_handler=subscriber_handler,
huddle_id_mapper=huddle_id_mapper,
user_id_mapper=user_id_mapper,
realm_id=3,
team_name=team_name,
)
self.assert_length(zerver_huddle, 1)
huddle_members = mattermost_data["direct_channel"][1]["members"]
huddle_name = generate_huddle_name(huddle_members)
self.assertTrue(huddle_id_mapper.has(huddle_name))
self.assertEqual(
subscriber_handler.get_users(
huddle_id=huddle_id_mapper.get(huddle_name)), {1, 2, 3})
def test_convert_channel_data(self) -> None:
fixture_file_name = self.fixture_file_name("export.json", "mattermost_fixtures")
mattermost_data = mattermost_data_file_to_dict(fixture_file_name)
username_to_user = create_username_to_user_mapping(mattermost_data["user"])
reset_mirror_dummy_users(username_to_user)
user_handler = UserHandler()
subscriber_handler = SubscriberHandler()
stream_id_mapper = IdMapper()
user_id_mapper = IdMapper()
team_name = "gryffindor"
convert_user_data(
user_handler=user_handler,
user_id_mapper=user_id_mapper,
user_data_map=username_to_user,
realm_id=3,
team_name=team_name,
)
zerver_stream = convert_channel_data(
channel_data=mattermost_data["channel"],
user_data_map=username_to_user,
subscriber_handler=subscriber_handler,
stream_id_mapper=stream_id_mapper,
user_id_mapper=user_id_mapper,
realm_id=3,
team_name=team_name,
)
self.assertEqual(len(zerver_stream), 3)
self.assertEqual(zerver_stream[0]["name"], "Gryffindor common room")
self.assertEqual(zerver_stream[0]["invite_only"], False)
self.assertEqual(zerver_stream[0]["description"], "A place for talking about Gryffindor common room")
self.assertEqual(zerver_stream[0]["rendered_description"], "")
self.assertEqual(zerver_stream[0]["realm"], 3)
self.assertEqual(zerver_stream[1]["name"], "Gryffindor quidditch team")
self.assertEqual(zerver_stream[1]["invite_only"], False)
self.assertEqual(zerver_stream[1]["description"], "A place for talking about Gryffindor quidditch team")
self.assertEqual(zerver_stream[1]["rendered_description"], "")
self.assertEqual(zerver_stream[1]["realm"], 3)
self.assertEqual(zerver_stream[2]["name"], "Dumbledores army")
self.assertEqual(zerver_stream[2]["invite_only"], True)
self.assertEqual(zerver_stream[2]["description"], "A place for talking about Dumbledores army")
self.assertEqual(zerver_stream[2]["rendered_description"], "")
self.assertEqual(zerver_stream[2]["realm"], 3)
self.assertTrue(stream_id_mapper.has("gryffindor-common-room"))
self.assertTrue(stream_id_mapper.has("gryffindor-quidditch-team"))
self.assertTrue(stream_id_mapper.has("dumbledores-army"))
# TODO: Add ginny
ron_id = user_id_mapper.get("ron")
harry_id = user_id_mapper.get("harry")
self.assertEqual({ron_id, harry_id}, {1, 2})
self.assertEqual(subscriber_handler.get_users(stream_id=stream_id_mapper.get("gryffindor-common-room")), {ron_id, harry_id})
self.assertEqual(subscriber_handler.get_users(stream_id=stream_id_mapper.get("gryffindor-quidditch-team")), {ron_id, harry_id})
self.assertEqual(subscriber_handler.get_users(stream_id=stream_id_mapper.get("dumbledores-army")), {ron_id, harry_id})
# Converting channel data when a user's `teams` value is `null`.
username_to_user["ron"].update({"teams": None})
zerver_stream = convert_channel_data(
channel_data=mattermost_data["channel"],
user_data_map=username_to_user,
subscriber_handler=subscriber_handler,
stream_id_mapper=stream_id_mapper,
user_id_mapper=user_id_mapper,
realm_id=3,
team_name=team_name,
)
harry_id = user_id_mapper.get("harry")
self.assertIn(harry_id, {1, 2})
self.assertEqual(subscriber_handler.get_users(stream_id=stream_id_mapper.get("gryffindor-common-room")), {harry_id})
self.assertEqual(subscriber_handler.get_users(stream_id=stream_id_mapper.get("gryffindor-quidditch-team")), {harry_id})
self.assertEqual(subscriber_handler.get_users(stream_id=stream_id_mapper.get("dumbledores-army")), {harry_id})
team_name = "slytherin"
zerver_stream = convert_channel_data(
channel_data=mattermost_data["channel"],
user_data_map=username_to_user,
subscriber_handler=subscriber_handler,
stream_id_mapper=stream_id_mapper,
user_id_mapper=user_id_mapper,
realm_id=4,
team_name=team_name,
)
malfoy_id = user_id_mapper.get("malfoy")
pansy_id = user_id_mapper.get("pansy")
snape_id = user_id_mapper.get("snape")
self.assertEqual({malfoy_id, pansy_id, snape_id}, {3, 4, 5})
self.assertEqual(subscriber_handler.get_users(stream_id=stream_id_mapper.get("slytherin-common-room")), {malfoy_id, pansy_id, snape_id})
self.assertEqual(subscriber_handler.get_users(stream_id=stream_id_mapper.get("slytherin-quidditch-team")), {malfoy_id, pansy_id})
def process_raw_message_batch(realm_id: int, raw_messages: List[Dict[str,
Any]],
subscriber_map: Dict[int, Set[int]],
user_id_mapper: IdMapper,
user_handler: UserHandler,
attachment_handler: AttachmentHandler,
get_recipient_id: Callable[[ZerverFieldsT], int],
is_pm_data: bool, output_dir: str) -> None:
def fix_mentions(content: str, mention_user_ids: Set[int]) -> str:
for user_id in mention_user_ids:
user = user_handler.get_user(user_id=user_id)
hipchat_mention = '@{short_name}'.format(**user)
zulip_mention = '@**{full_name}**'.format(**user)
content = content.replace(hipchat_mention, zulip_mention)
content = content.replace('@here', '@**all**')
return content
mention_map: Dict[int, Set[int]] = dict()
zerver_message = []
import html2text
h = html2text.HTML2Text()
for raw_message in raw_messages:
# One side effect here:
message_id = NEXT_ID('message')
mention_user_ids = {
user_id_mapper.get(id)
for id in set(raw_message['mention_user_ids'])
if user_id_mapper.has(id)
}
mention_map[message_id] = mention_user_ids
content = fix_mentions(
content=raw_message['content'],
mention_user_ids=mention_user_ids,
)
content = h.handle(content)
if len(content) > 10000:
logging.info('skipping too-long message of length %s' %
(len(content), ))
continue
date_sent = raw_message['date_sent']
try:
recipient_id = get_recipient_id(raw_message)
except KeyError:
logging.debug(
"Could not find recipient_id for a message, skipping.")
continue
rendered_content = None
if is_pm_data:
topic_name = ''
else:
topic_name = 'imported from hipchat'
user_id = raw_message['sender_id']
# Another side effect:
extra_content = attachment_handler.handle_message_data(
realm_id=realm_id,
message_id=message_id,
sender_id=user_id,
attachment=raw_message['attachment'],
files_dir=raw_message['files_dir'],
)
if extra_content:
has_attachment = True
content += '\n' + extra_content
else:
has_attachment = False
message = build_message(
content=content,
message_id=message_id,
date_sent=date_sent,
recipient_id=recipient_id,
rendered_content=rendered_content,
topic_name=topic_name,
user_id=user_id,
has_attachment=has_attachment,
)
zerver_message.append(message)
zerver_usermessage = make_user_messages(
zerver_message=zerver_message,
subscriber_map=subscriber_map,
is_pm_data=is_pm_data,
mention_map=mention_map,
)
message_json = dict(
zerver_message=zerver_message,
zerver_usermessage=zerver_usermessage,
)
dump_file_id = NEXT_ID('dump_file_id')
message_file = "/messages-%06d.json" % (dump_file_id, )
create_converted_data_files(message_json, output_dir, message_file)
def convert_room_data(raw_data: List[ZerverFieldsT],
subscriber_handler: SubscriberHandler,
stream_id_mapper: IdMapper,
user_id_mapper: IdMapper,
realm_id: int,
api_token: Optional[str] = None) -> List[ZerverFieldsT]:
flat_data = [d['Room'] for d in raw_data]
def get_invite_only(v: str) -> bool:
if v == 'public':
return False
elif v == 'private':
return True
else:
raise Exception('unexpected value')
streams = []
for in_dict in flat_data:
now = int(timezone_now().timestamp())
stream_id = stream_id_mapper.get(in_dict['id'])
invite_only = get_invite_only(in_dict['privacy'])
stream = build_stream(
date_created=now,
realm_id=realm_id,
name=in_dict['name'],
description=in_dict['topic'],
stream_id=stream_id,
deactivated=in_dict['is_archived'],
invite_only=invite_only,
)
if invite_only:
users: Set[int] = {
user_id_mapper.get(key)
for key in in_dict['members'] if user_id_mapper.has(key)
}
if user_id_mapper.has(in_dict['owner']):
owner = user_id_mapper.get(in_dict['owner'])
users.add(owner)
else:
users = set()
if api_token is not None:
hc = hypchat.HypChat(api_token)
room_data = hc.fromurl('{}/v2/room/{}/member'.format(
hc.endpoint, in_dict['id']))
for item in room_data['items']:
hipchat_user_id = item['id']
zulip_user_id = user_id_mapper.get(hipchat_user_id)
users.add(zulip_user_id)
if users:
subscriber_handler.set_info(
stream_id=stream_id,
users=users,
)
# unmapped fields:
# guest_access_url: no Zulip equivalent
# created: we just use "now"
# participants: no good sample data
streams.append(stream)
return streams
def test_process_users(self) -> None:
fixture_dir_name = self.fixture_file_name("", "rocketchat_fixtures")
rocketchat_data = rocketchat_data_to_dict(fixture_dir_name)
user_id_to_user_map = map_user_id_to_user(rocketchat_data["user"])
realm_id = 3
domain_name = "zulip.com"
user_handler = UserHandler()
user_id_mapper = IdMapper()
process_users(
user_id_to_user_map=user_id_to_user_map,
realm_id=realm_id,
domain_name=domain_name,
user_handler=user_handler,
user_id_mapper=user_id_mapper,
)
self.assert_length(user_handler.get_all_users(), 6)
self.assertTrue(user_id_mapper.has(rocketchat_data["user"][0]["_id"]))
self.assertTrue(user_id_mapper.has(rocketchat_data["user"][4]["_id"]))
user_id = user_id_mapper.get(rocketchat_data["user"][0]["_id"])
user = user_handler.get_user(user_id)
self.assertEqual(user["full_name"], rocketchat_data["user"][0]["name"])
self.assertEqual(user["avatar_source"], "G")
self.assertEqual(user["delivery_email"], "*****@*****.**")
self.assertEqual(user["email"], "*****@*****.**")
self.assertEqual(user["full_name"], "Rocket.Cat")
self.assertEqual(user["id"], 1)
self.assertEqual(user["is_active"], False)
self.assertEqual(user["is_mirror_dummy"], False)
self.assertEqual(user["is_bot"], True)
self.assertEqual(user["bot_type"], 1)
self.assertEqual(user["bot_owner"], 2)
self.assertEqual(user["role"], UserProfile.ROLE_MEMBER)
self.assertEqual(user["realm"], realm_id)
self.assertEqual(user["short_name"], "rocket.cat")
self.assertEqual(user["timezone"], "UTC")
user_id = user_id_mapper.get(rocketchat_data["user"][2]["_id"])
user = user_handler.get_user(user_id)
self.assertEqual(user["full_name"], rocketchat_data["user"][2]["name"])
self.assertEqual(user["avatar_source"], "G")
self.assertEqual(user["delivery_email"], "*****@*****.**")
self.assertEqual(user["email"], "*****@*****.**")
self.assertEqual(user["full_name"], "Harry Potter")
self.assertEqual(user["id"], 3)
self.assertEqual(user["is_active"], True)
self.assertEqual(user["is_mirror_dummy"], False)
self.assertEqual(user["is_bot"], False)
self.assertEqual(user["bot_type"], None)
self.assertEqual(user["bot_owner"], None)
self.assertEqual(user["role"], UserProfile.ROLE_REALM_OWNER)
self.assertEqual(user["realm"], realm_id)
self.assertEqual(user["short_name"], "harry.potter")
self.assertEqual(user["timezone"], "UTC")
# Test `is_mirror_dummy` set for users of type `unknown`
rocketchat_data["user"].append({
"_id":
"s0m34ndmID",
"createdAt":
datetime.datetime(2019, 11, 6, 0, 38, 42, 796000),
"type":
"unknown",
"roles": ["unknown"],
"name":
"Unknown user",
"username":
"******",
})
user_id_to_user_map = map_user_id_to_user(rocketchat_data["user"])
process_users(
user_id_to_user_map=user_id_to_user_map,
realm_id=realm_id,
domain_name=domain_name,
user_handler=user_handler,
user_id_mapper=user_id_mapper,
)
self.assert_length(user_handler.get_all_users(), 7)
self.assertTrue(user_id_mapper.has(rocketchat_data["user"][6]["_id"]))
user_id = user_id_mapper.get(rocketchat_data["user"][6]["_id"])
user = user_handler.get_user(user_id)
self.assertEqual(user["id"], 7)
self.assertEqual(user["is_active"], False)
self.assertEqual(user["is_mirror_dummy"], True)
self.assertEqual(user["is_bot"], False)
def test_sender_ids(self) -> None:
realm_id = 5
user_handler = UserHandler()
user_id_mapper = IdMapper()
user_id_mapper.has = lambda key: True # type: ignore # it's just a stub
# Simulate a "normal" user first.
user_with_id = dict(
id=1,
# other fields don't matter here
)
user_handler.add_user(user=user_with_id)
normal_message = dict(
sender=dict(
id=1,
)
) # type: Dict[str, Any]
sender_id = get_hipchat_sender_id(
realm_id=realm_id,
slim_mode=False,
message_dict=normal_message,
user_id_mapper=user_id_mapper,
user_handler=user_handler,
)
self.assertEqual(sender_id, 1)
bot_message = dict(
sender='fred_bot',
)
# Every message from fred_bot should
# return the same sender_id.
fred_bot_sender_id = 2
for i in range(3):
sender_id = get_hipchat_sender_id(
realm_id=realm_id,
slim_mode=False,
message_dict=bot_message,
user_id_mapper=user_id_mapper,
user_handler=user_handler,
)
self.assertEqual(sender_id, fred_bot_sender_id)
id_zero_message = dict(
sender=dict(
id=0,
name='hal_bot',
),
)
hal_bot_sender_id = 3
for i in range(3):
sender_id = get_hipchat_sender_id(
realm_id=realm_id,
slim_mode=False,
message_dict=id_zero_message,
user_id_mapper=user_id_mapper,
user_handler=user_handler,
)
self.assertEqual(sender_id, hal_bot_sender_id)
def test_convert_channel_data(self) -> None:
user_handler = UserHandler()
subscriber_handler = SubscriberHandler()
stream_id_mapper = IdMapper()
user_id_mapper = IdMapper()
team_name = "gryffindor"
convert_user_data(
user_handler=user_handler,
user_id_mapper=user_id_mapper,
user_data_map=self.username_to_user,
realm_id=3,
team_name=team_name,
)
zerver_stream = convert_channel_data(
channel_data=self.mattermost_data["channel"],
user_data_map=self.username_to_user,
subscriber_handler=subscriber_handler,
stream_id_mapper=stream_id_mapper,
user_id_mapper=user_id_mapper,
realm_id=3,
team_name=team_name,
)
self.assertEqual(len(zerver_stream), 3)
self.assertEqual(zerver_stream[0]["name"], "Gryffindor common room")
self.assertEqual(zerver_stream[0]["invite_only"], False)
self.assertEqual(zerver_stream[0]["description"],
"A place for talking about Gryffindor common room")
self.assertEqual(zerver_stream[0]["rendered_description"], "")
self.assertEqual(zerver_stream[0]["realm"], 3)
self.assertEqual(zerver_stream[1]["name"], "Gryffindor quidditch team")
self.assertEqual(zerver_stream[1]["invite_only"], False)
self.assertEqual(
zerver_stream[1]["description"],
"A place for talking about Gryffindor quidditch team")
self.assertEqual(zerver_stream[1]["rendered_description"], "")
self.assertEqual(zerver_stream[1]["realm"], 3)
self.assertEqual(zerver_stream[2]["name"], "Dumbledores army")
self.assertEqual(zerver_stream[2]["invite_only"], True)
self.assertEqual(zerver_stream[2]["description"],
"A place for talking about Dumbledores army")
self.assertEqual(zerver_stream[2]["rendered_description"], "")
self.assertEqual(zerver_stream[2]["realm"], 3)
self.assertTrue(stream_id_mapper.has("gryffindor-common-room"))
self.assertTrue(stream_id_mapper.has("gryffindor-quidditch-team"))
self.assertTrue(stream_id_mapper.has("dumbledores-army"))
# TODO: Add ginny
self.assertEqual(
subscriber_handler.get_users(
stream_id_mapper.get("gryffindor-common-room")), {1, 2})
self.assertEqual(
subscriber_handler.get_users(
stream_id_mapper.get("gryffindor-quidditch-team")), {1, 2})
self.assertEqual(
subscriber_handler.get_users(
stream_id_mapper.get("dumbledores-army")), {1, 2})
team_name = "slytherin"
zerver_stream = convert_channel_data(
channel_data=self.mattermost_data["channel"],
user_data_map=self.username_to_user,
subscriber_handler=subscriber_handler,
stream_id_mapper=stream_id_mapper,
user_id_mapper=user_id_mapper,
realm_id=4,
team_name=team_name,
)
self.assertEqual(
subscriber_handler.get_users(
stream_id_mapper.get("slytherin-common-room")), {3, 4, 5})
self.assertEqual(
subscriber_handler.get_users(
stream_id_mapper.get("slytherin-quidditch-team")), {3, 4})
