def test_mentioned_data(self) -> None:
slack_user_map = {"U08RGD1RD": 540, "U0CBK5KAT": 554, "U09TYF5SK": 571}
# For this test, only relevant keys are 'id', 'name', 'deleted'
# and 'real_name'
users = [
{
"id": "U0CBK5KAT",
"name": "aaron.anzalone",
"deleted": False,
"is_mirror_dummy": False,
"real_name": "",
},
{
"id": "U08RGD1RD",
"name": "john",
"deleted": False,
"is_mirror_dummy": False,
"real_name": "John Doe",
},
{
"id": "U09TYF5Sk",
"name": "Jane",
"is_mirror_dummy": False,
"deleted":
True, # Deleted users don't have 'real_name' key in Slack
},
]
channel_map = {"general": ("C5Z73A7RA", 137)}
message = "Hi <@U08RGD1RD|john>: How are you? <#C5Z73A7RA|general>"
text, mentioned_users, has_link = convert_to_zulip_markdown(
message, users, channel_map, slack_user_map)
full_name = get_user_full_name(users[1])
self.assertEqual(full_name, "John Doe")
self.assertEqual(get_user_full_name(users[2]), "Jane")
self.assertEqual(text,
f"Hi @**{full_name}**: How are you? #**general**")
self.assertEqual(mentioned_users, [540])
# multiple mentioning
message = "Hi <@U08RGD1RD|john>: How are you?<@U0CBK5KAT> asked."
text, mentioned_users, has_link = convert_to_zulip_markdown(
message, users, channel_map, slack_user_map)
self.assertEqual(
text, "Hi @**John Doe**: How are you?@**aaron.anzalone** asked.")
self.assertEqual(mentioned_users, [540, 554])
# Check wrong mentioning
message = "Hi <@U08RGD1RD|jon>: How are you?"
text, mentioned_users, has_link = convert_to_zulip_markdown(
message, users, channel_map, slack_user_map)
self.assertEqual(text, message)
self.assertEqual(mentioned_users, [])
def test_mentioned_data(self) -> None:
slack_user_map = {'U08RGD1RD': 540, 'U0CBK5KAT': 554, 'U09TYF5SK': 571}
# For this test, only relevant keys are 'id', 'name', 'deleted'
# and 'real_name'
users = [{
"id": "U0CBK5KAT",
"name": "aaron.anzalone",
"deleted": False,
"is_mirror_dummy": False,
"real_name": ""
}, {
"id": "U08RGD1RD",
"name": "john",
"deleted": False,
"is_mirror_dummy": False,
"real_name": "John Doe"
}, {
"id": "U09TYF5Sk",
"name": "Jane",
"is_mirror_dummy": False,
"deleted": True
}] # Deleted users don't have 'real_name' key in Slack
channel_map = {'general': ('C5Z73A7RA', 137)}
message = 'Hi <@U08RGD1RD|john>: How are you? <#C5Z73A7RA|general>'
text, mentioned_users, has_link = convert_to_zulip_markdown(
message, users, channel_map, slack_user_map)
full_name = get_user_full_name(users[1])
self.assertEqual(full_name, 'John Doe')
self.assertEqual(get_user_full_name(users[2]), 'Jane')
self.assertEqual(
text, 'Hi @**%s**: How are you? #**general**' % (full_name, ))
self.assertEqual(mentioned_users, [540])
# multiple mentioning
message = 'Hi <@U08RGD1RD|john>: How are you?<@U0CBK5KAT> asked.'
text, mentioned_users, has_link = convert_to_zulip_markdown(
message, users, channel_map, slack_user_map)
self.assertEqual(
text, 'Hi @**%s**: How are you?@**%s** asked.' %
('John Doe', 'aaron.anzalone'))
self.assertEqual(mentioned_users, [540, 554])
# Check wrong mentioning
message = 'Hi <@U08RGD1RD|jon>: How are you?'
text, mentioned_users, has_link = convert_to_zulip_markdown(
message, users, channel_map, slack_user_map)
self.assertEqual(text, message)
self.assertEqual(mentioned_users, [])
def test_mentioned_data(self) -> None:
slack_user_map = {'U08RGD1RD': 540,
'U0CBK5KAT': 554,
'U09TYF5SK': 571}
# For this test, only relevant keys are 'id', 'name', 'deleted'
# and 'real_name'
users = [{"id": "U0CBK5KAT",
"name": "aaron.anzalone",
"deleted": False,
"real_name": ""},
{"id": "U08RGD1RD",
"name": "john",
"deleted": False,
"real_name": "John Doe"},
{"id": "U09TYF5Sk",
"name": "Jane",
"deleted": True}] # Deleted users don't have 'real_name' key in Slack
channel_map = {'general': ('C5Z73A7RA', 137)}
message = 'Hi <@U08RGD1RD|john>: How are you? <#C5Z73A7RA|general>'
text, mentioned_users, has_link = convert_to_zulip_markdown(message, users, channel_map, slack_user_map)
full_name = get_user_full_name(users[1])
self.assertEqual(full_name, 'John Doe')
self.assertEqual(get_user_full_name(users[2]), 'Jane')
self.assertEqual(text, 'Hi @**%s**: How are you? #**general**' % (full_name))
self.assertEqual(mentioned_users, [540])
# multiple mentioning
message = 'Hi <@U08RGD1RD|john>: How are you?<@U0CBK5KAT> asked.'
text, mentioned_users, has_link = convert_to_zulip_markdown(message, users, channel_map, slack_user_map)
self.assertEqual(text, 'Hi @**%s**: How are you?@**%s** asked.' %
('John Doe', 'aaron.anzalone'))
self.assertEqual(mentioned_users, [540, 554])
# Check wrong mentioning
message = 'Hi <@U08RGD1RD|jon>: How are you?'
text, mentioned_users, has_link = convert_to_zulip_markdown(message, users, channel_map, slack_user_map)
self.assertEqual(text, message)
self.assertEqual(mentioned_users, [])
def get_user_email(user: ZerverFieldsT, domain_name: str) -> str:
if 'email' in user['profile']:
return user['profile']['email']
if 'bot_id' in user['profile']:
if 'real_name_normalized' in user['profile']:
slack_bot_name = user['profile']['real_name_normalized']
elif 'first_name' in user['profile']:
slack_bot_name = user['profile']['first_name']
else:
raise AssertionError("Could not identify bot type")
return slack_bot_name.replace("Bot", "").replace(" ", "") + "-bot@%s" % (domain_name,)
if get_user_full_name(user) == "slackbot":
return "imported-slackbot-bot@%s" % (domain_name,)
raise AssertionError("Could not find email address for Slack user %s" % (user,))
def get_user_email(user: ZerverFieldsT, domain_name: str) -> str:
if 'email' in user['profile']:
return user['profile']['email']
if 'bot_id' in user['profile']:
if 'real_name_normalized' in user['profile']:
slack_bot_name = user['profile']['real_name_normalized']
elif 'first_name' in user['profile']:
slack_bot_name = user['profile']['first_name']
else:
raise AssertionError("Could not identify bot type")
return slack_bot_name.replace("Bot", "").replace(" ", "") + "-bot@%s" % (domain_name,)
if get_user_full_name(user).lower() == "slackbot":
return "imported-slackbot-bot@%s" % (domain_name,)
raise AssertionError("Could not find email address for Slack user %s" % (user,))
def get_user_email(user: ZerverFieldsT, domain_name: str) -> str:
if "email" in user["profile"]:
return user["profile"]["email"]
if user["is_mirror_dummy"]:
return "{}@{}.slack.com".format(user["name"], user["team_domain"])
if "bot_id" in user["profile"]:
if "real_name_normalized" in user["profile"]:
slack_bot_name = user["profile"]["real_name_normalized"]
elif "first_name" in user["profile"]:
slack_bot_name = user["profile"]["first_name"]
else:
raise AssertionError("Could not identify bot type")
return slack_bot_name.replace("Bot", "").replace(" ", "") + f"-bot@{domain_name}"
if get_user_full_name(user).lower() == "slackbot":
return f"imported-slackbot-bot@{domain_name}"
raise AssertionError(f"Could not find email address for Slack user {user}")
def users_to_zerver_userprofile(slack_data_dir: str, users: List[ZerverFieldsT], realm_id: int,
timestamp: Any, domain_name: str) -> Tuple[List[ZerverFieldsT],
List[ZerverFieldsT],
AddedUsersT,
List[ZerverFieldsT],
List[ZerverFieldsT]]:
"""
Returns:
1. zerver_userprofile, which is a list of user profile
2. avatar_list, which is list to map avatars to zulip avatard records.json
3. added_users, which is a dictionary to map from slack user id to zulip
user id
4. zerver_customprofilefield, which is a list of all custom profile fields
5. zerver_customprofilefield_values, which is a list of user profile fields
"""
logging.info('######### IMPORTING USERS STARTED #########\n')
zerver_userprofile = []
zerver_customprofilefield = [] # type: List[ZerverFieldsT]
zerver_customprofilefield_values = [] # type: List[ZerverFieldsT]
avatar_list = [] # type: List[ZerverFieldsT]
added_users = {}
# The user data we get from the slack api does not contain custom profile data
# Hence we get it from the slack zip file
slack_data_file_user_list = get_data_file(slack_data_dir + '/users.json')
# To map user id with the custom profile fields of the corresponding user
slack_user_custom_field_map = {} # type: ZerverFieldsT
# To store custom fields corresponding to their ids
custom_field_map = {} # type: ZerverFieldsT
for user in slack_data_file_user_list:
process_slack_custom_fields(user, slack_user_custom_field_map)
# We have only one primary owner in slack, see link
# https://get.slack.help/hc/en-us/articles/201912948-Owners-and-Administrators
# This is to import the primary owner first from all the users
user_id_count = custom_field_id_count = customprofilefield_id = 0
primary_owner_id = user_id_count
user_id_count += 1
for user in users:
slack_user_id = user['id']
if user.get('is_primary_owner', False):
user_id = primary_owner_id
else:
user_id = user_id_count
# email
email = get_user_email(user, domain_name)
# avatar
# ref: https://chat.zulip.org/help/change-your-avatar
avatar_url = build_avatar_url(slack_user_id, user['team_id'],
user['profile']['avatar_hash'])
build_avatar(user_id, realm_id, email, avatar_url, timestamp, avatar_list)
# check if user is the admin
realm_admin = get_admin(user)
# timezone
timezone = get_user_timezone(user)
# Check for custom profile fields
if slack_user_id in slack_user_custom_field_map:
# For processing the fields
custom_field_map, customprofilefield_id = build_customprofile_field(
zerver_customprofilefield, slack_user_custom_field_map[slack_user_id],
customprofilefield_id, realm_id, custom_field_map)
# Store the custom field values for the corresponding user
custom_field_id_count = build_customprofilefields_values(
custom_field_map, slack_user_custom_field_map[slack_user_id], user_id,
custom_field_id_count, zerver_customprofilefield_values)
userprofile = UserProfile(
full_name=get_user_full_name(user),
short_name=user['name'],
is_active=not user['deleted'],
id=user_id,
email=email,
delivery_email=email,
avatar_source='U',
is_bot=user.get('is_bot', False),
pointer=-1,
is_realm_admin=realm_admin,
bot_type=1 if user.get('is_bot', False) else None,
date_joined=timestamp,
timezone=timezone,
last_login=timestamp)
userprofile_dict = model_to_dict(userprofile)
# Set realm id separately as the corresponding realm is not yet a Realm model instance
userprofile_dict['realm'] = realm_id
zerver_userprofile.append(userprofile_dict)
added_users[slack_user_id] = user_id
if not user.get('is_primary_owner', False):
user_id_count += 1
logging.info(u"{} -> {}".format(user['name'], userprofile_dict['email']))
process_customprofilefields(zerver_customprofilefield, zerver_customprofilefield_values)
logging.info('######### IMPORTING USERS FINISHED #########\n')
return zerver_userprofile, avatar_list, added_users, zerver_customprofilefield, \
zerver_customprofilefield_values
def users_to_zerver_userprofile(slack_data_dir: str, users: List[ZerverFieldsT], realm_id: int,
timestamp: Any, domain_name: str) -> Tuple[List[ZerverFieldsT],
List[ZerverFieldsT],
AddedUsersT,
List[ZerverFieldsT],
List[ZerverFieldsT]]:
"""
Returns:
1. zerver_userprofile, which is a list of user profile
2. avatar_list, which is list to map avatars to zulip avatard records.json
3. added_users, which is a dictionary to map from slack user id to zulip
user id
4. zerver_customprofilefield, which is a list of all custom profile fields
5. zerver_customprofilefield_values, which is a list of user profile fields
"""
logging.info('######### IMPORTING USERS STARTED #########\n')
zerver_userprofile = []
zerver_customprofilefield = [] # type: List[ZerverFieldsT]
zerver_customprofilefield_values = [] # type: List[ZerverFieldsT]
avatar_list = [] # type: List[ZerverFieldsT]
added_users = {}
# The user data we get from the slack api does not contain custom profile data
# Hence we get it from the slack zip file
slack_data_file_user_list = get_data_file(slack_data_dir + '/users.json')
# To map user id with the custom profile fields of the corresponding user
slack_user_custom_field_map = {} # type: ZerverFieldsT
# To store custom fields corresponding to their ids
custom_field_map = {} # type: ZerverFieldsT
for user in slack_data_file_user_list:
process_slack_custom_fields(user, slack_user_custom_field_map)
# We have only one primary owner in slack, see link
# https://get.slack.help/hc/en-us/articles/201912948-Owners-and-Administrators
# This is to import the primary owner first from all the users
user_id_count = custom_field_id_count = customprofilefield_id = 0
primary_owner_id = user_id_count
user_id_count += 1
for user in users:
slack_user_id = user['id']
if user.get('is_primary_owner', False):
user_id = primary_owner_id
else:
user_id = user_id_count
# email
email = get_user_email(user, domain_name)
# avatar
# ref: https://chat.zulip.org/help/change-your-avatar
avatar_url = build_avatar_url(slack_user_id, user['team_id'],
user['profile']['avatar_hash'])
build_avatar(user_id, realm_id, email, avatar_url, timestamp, avatar_list)
# check if user is the admin
realm_admin = get_admin(user)
# timezone
timezone = get_user_timezone(user)
# Check for custom profile fields
if slack_user_id in slack_user_custom_field_map:
# For processing the fields
custom_field_map, customprofilefield_id = build_customprofile_field(
zerver_customprofilefield, slack_user_custom_field_map[slack_user_id],
customprofilefield_id, realm_id, custom_field_map)
# Store the custom field values for the corresponding user
custom_field_id_count = build_customprofilefields_values(
custom_field_map, slack_user_custom_field_map[slack_user_id], user_id,
custom_field_id_count, zerver_customprofilefield_values)
userprofile = UserProfile(
full_name=get_user_full_name(user),
short_name=user['name'],
is_active=not user['deleted'],
id=user_id,
email=email,
delivery_email=email,
avatar_source='U',
is_bot=user.get('is_bot', False),
pointer=-1,
is_realm_admin=realm_admin,
bot_type=1 if user.get('is_bot', False) else None,
date_joined=timestamp,
timezone=timezone,
last_login=timestamp)
userprofile_dict = model_to_dict(userprofile)
# Set realm id separately as the corresponding realm is not yet a Realm model instance
userprofile_dict['realm'] = realm_id
zerver_userprofile.append(userprofile_dict)
added_users[slack_user_id] = user_id
if not user.get('is_primary_owner', False):
user_id_count += 1
logging.info(u"{} -> {}".format(user['name'], userprofile_dict['email']))
process_customprofilefields(zerver_customprofilefield, zerver_customprofilefield_values)
logging.info('######### IMPORTING USERS FINISHED #########\n')
return zerver_userprofile, avatar_list, added_users, zerver_customprofilefield, \
zerver_customprofilefield_values
def users_to_zerver_userprofile(
slack_data_dir: str, users: List[ZerverFieldsT], realm_id: int,
timestamp: Any, domain_name: str
) -> Tuple[List[ZerverFieldsT], List[ZerverFieldsT], SlackToZulipUserIDT,
List[ZerverFieldsT], List[ZerverFieldsT]]:
"""
Returns:
1. zerver_userprofile, which is a list of user profile
2. avatar_list, which is list to map avatars to zulip avatard records.json
3. slack_user_id_to_zulip_user_id, which is a dictionary to map from slack user id to zulip
user id
4. zerver_customprofilefield, which is a list of all custom profile fields
5. zerver_customprofilefield_values, which is a list of user profile fields
"""
logging.info('######### IMPORTING USERS STARTED #########\n')
zerver_userprofile = []
zerver_customprofilefield = [] # type: List[ZerverFieldsT]
zerver_customprofilefield_values = [] # type: List[ZerverFieldsT]
avatar_list = [] # type: List[ZerverFieldsT]
slack_user_id_to_zulip_user_id = {}
# The user data we get from the slack api does not contain custom profile data
# Hence we get it from the slack zip file
slack_data_file_user_list = get_data_file(slack_data_dir + '/users.json')
slack_user_id_to_custom_profile_fields = {} # type: ZerverFieldsT
slack_custom_field_name_to_zulip_custom_field_id = {
} # type: ZerverFieldsT
for user in slack_data_file_user_list:
process_slack_custom_fields(user,
slack_user_id_to_custom_profile_fields)
# We have only one primary owner in slack, see link
# https://get.slack.help/hc/en-us/articles/201912948-Owners-and-Administrators
# This is to import the primary owner first from all the users
user_id_count = custom_profile_field_value_id_count = custom_profile_field_id_count = 0
primary_owner_id = user_id_count
user_id_count += 1
for user in users:
slack_user_id = user['id']
if user.get('is_primary_owner', False):
user_id = primary_owner_id
else:
user_id = user_id_count
email = get_user_email(user, domain_name)
# ref: https://chat.zulip.org/help/set-your-profile-picture
avatar_url = build_avatar_url(slack_user_id, user['team_id'],
user['profile']['avatar_hash'])
build_avatar(user_id, realm_id, email, avatar_url, timestamp,
avatar_list)
role = UserProfile.ROLE_MEMBER
if get_admin(user):
role = UserProfile.ROLE_REALM_ADMINISTRATOR
timezone = get_user_timezone(user)
if slack_user_id in slack_user_id_to_custom_profile_fields:
slack_custom_field_name_to_zulip_custom_field_id, custom_profile_field_id_count = \
build_customprofile_field(zerver_customprofilefield,
slack_user_id_to_custom_profile_fields[slack_user_id],
custom_profile_field_id_count, realm_id,
slack_custom_field_name_to_zulip_custom_field_id)
custom_profile_field_value_id_count = build_customprofilefields_values(
slack_custom_field_name_to_zulip_custom_field_id,
slack_user_id_to_custom_profile_fields[slack_user_id], user_id,
custom_profile_field_value_id_count,
zerver_customprofilefield_values)
userprofile = UserProfile(
full_name=get_user_full_name(user),
short_name=user['name'],
is_active=not user.get('deleted', False)
and not user["is_mirror_dummy"],
is_mirror_dummy=user["is_mirror_dummy"],
id=user_id,
email=email,
delivery_email=email,
avatar_source='U',
is_bot=user.get('is_bot', False),
pointer=-1,
role=role,
bot_type=1 if user.get('is_bot', False) else None,
date_joined=timestamp,
timezone=timezone,
last_login=timestamp)
userprofile_dict = model_to_dict(userprofile)
# Set realm id separately as the corresponding realm is not yet a Realm model instance
userprofile_dict['realm'] = realm_id
zerver_userprofile.append(userprofile_dict)
slack_user_id_to_zulip_user_id[slack_user_id] = user_id
if not user.get('is_primary_owner', False):
user_id_count += 1
logging.info(u"{} -> {}".format(user['name'],
userprofile_dict['email']))
process_customprofilefields(zerver_customprofilefield,
zerver_customprofilefield_values)
logging.info('######### IMPORTING USERS FINISHED #########\n')
return zerver_userprofile, avatar_list, slack_user_id_to_zulip_user_id, zerver_customprofilefield, \
zerver_customprofilefield_values
def users_to_zerver_userprofile(
slack_data_dir: str, users: List[ZerverFieldsT], realm_id: int, timestamp: Any, domain_name: str
) -> Tuple[
List[ZerverFieldsT],
List[ZerverFieldsT],
SlackToZulipUserIDT,
List[ZerverFieldsT],
List[ZerverFieldsT],
]:
"""
Returns:
1. zerver_userprofile, which is a list of user profile
2. avatar_list, which is list to map avatars to Zulip avatard records.json
3. slack_user_id_to_zulip_user_id, which is a dictionary to map from Slack user ID to Zulip
user id
4. zerver_customprofilefield, which is a list of all custom profile fields
5. zerver_customprofilefield_values, which is a list of user profile fields
"""
logging.info("######### IMPORTING USERS STARTED #########\n")
zerver_userprofile = []
zerver_customprofilefield: List[ZerverFieldsT] = []
zerver_customprofilefield_values: List[ZerverFieldsT] = []
avatar_list: List[ZerverFieldsT] = []
slack_user_id_to_zulip_user_id = {}
# The user data we get from the Slack API does not contain custom profile data
# Hence we get it from the Slack zip file
slack_data_file_user_list = get_data_file(slack_data_dir + "/users.json")
slack_user_id_to_custom_profile_fields: ZerverFieldsT = {}
slack_custom_field_name_to_zulip_custom_field_id: ZerverFieldsT = {}
for user in slack_data_file_user_list:
process_slack_custom_fields(user, slack_user_id_to_custom_profile_fields)
# We have only one primary owner in Slack, see link
# https://get.slack.help/hc/en-us/articles/201912948-Owners-and-Administrators
# This is to import the primary owner first from all the users
user_id_count = custom_profile_field_value_id_count = custom_profile_field_id_count = 0
primary_owner_id = user_id_count
user_id_count += 1
for user in users:
slack_user_id = user["id"]
if user.get("is_primary_owner", False):
user_id = primary_owner_id
else:
user_id = user_id_count
email = get_user_email(user, domain_name)
# ref: https://zulip.com/help/change-your-profile-picture
avatar_url = build_avatar_url(
slack_user_id, user["team_id"], user["profile"]["avatar_hash"]
)
build_avatar(user_id, realm_id, email, avatar_url, timestamp, avatar_list)
role = UserProfile.ROLE_MEMBER
if get_owner(user):
role = UserProfile.ROLE_REALM_OWNER
elif get_admin(user):
role = UserProfile.ROLE_REALM_ADMINISTRATOR
if get_guest(user):
role = UserProfile.ROLE_GUEST
timezone = get_user_timezone(user)
if slack_user_id in slack_user_id_to_custom_profile_fields:
(
slack_custom_field_name_to_zulip_custom_field_id,
custom_profile_field_id_count,
) = build_customprofile_field(
zerver_customprofilefield,
slack_user_id_to_custom_profile_fields[slack_user_id],
custom_profile_field_id_count,
realm_id,
slack_custom_field_name_to_zulip_custom_field_id,
)
custom_profile_field_value_id_count = build_customprofilefields_values(
slack_custom_field_name_to_zulip_custom_field_id,
slack_user_id_to_custom_profile_fields[slack_user_id],
user_id,
custom_profile_field_value_id_count,
zerver_customprofilefield_values,
)
userprofile = UserProfile(
full_name=get_user_full_name(user),
is_active=not user.get("deleted", False) and not user["is_mirror_dummy"],
is_mirror_dummy=user["is_mirror_dummy"],
id=user_id,
email=email,
delivery_email=email,
avatar_source="U",
is_bot=user.get("is_bot", False),
role=role,
bot_type=1 if user.get("is_bot", False) else None,
date_joined=timestamp,
timezone=timezone,
last_login=timestamp,
)
userprofile_dict = model_to_dict(userprofile)
# Set realm id separately as the corresponding realm is not yet a Realm model instance
userprofile_dict["realm"] = realm_id
zerver_userprofile.append(userprofile_dict)
slack_user_id_to_zulip_user_id[slack_user_id] = user_id
if not user.get("is_primary_owner", False):
user_id_count += 1
logging.info("%s -> %s", user["name"], userprofile_dict["email"])
process_customprofilefields(zerver_customprofilefield, zerver_customprofilefield_values)
logging.info("######### IMPORTING USERS FINISHED #########\n")
return (
zerver_userprofile,
avatar_list,
slack_user_id_to_zulip_user_id,
zerver_customprofilefield,
zerver_customprofilefield_values,
)
