def test_has_link(self) -> None:
slack_user_map = {} # type: Dict[str, int]
message = '<http://journals.plos.org/plosone/article>'
text, mentioned_users, has_link = convert_to_zulip_markdown(message, [], slack_user_map)
self.assertEqual(text, 'http://journals.plos.org/plosone/article')
self.assertEqual(has_link, True)
message = '<mailto:[email protected]>'
text, mentioned_users, has_link = convert_to_zulip_markdown(message, [], slack_user_map)
self.assertEqual(text, 'mailto:[email protected]')
self.assertEqual(has_link, True)
message = 'random message'
text, mentioned_users, has_link = convert_to_zulip_markdown(message, [], slack_user_map)
self.assertEqual(has_link, False)
def channel_message_to_zerver_message(
REALM_ID: int, users: List[ZerverFieldsT], added_users: AddedUsersT,
added_recipient: AddedRecipientsT, all_messages: List[ZerverFieldsT],
zerver_subscription: List[ZerverFieldsT],
ids: List[Any]) -> Tuple[List[ZerverFieldsT], List[ZerverFieldsT]]:
"""
Returns:
1. zerver_message, which is a list of the messages
2. zerver_usermessage, which is a list of the usermessages
"""
message_id_count = usermessage_id_count = 0
message_id_list, usermessage_id_list = ids
zerver_message = []
zerver_usermessage = [] # type: List[ZerverFieldsT]
for message in all_messages:
user = get_message_sending_user(message)
if not user:
# Ignore messages without user names
# These are Sometimes produced by slack
continue
has_attachment = False
content, mentioned_users_id, has_link = convert_to_zulip_markdown(
message['text'], users, added_users)
rendered_content = None
if 'subtype' in message.keys():
subtype = message['subtype']
if subtype in ["channel_join", "channel_leave", "channel_name"]:
continue
recipient_id = added_recipient[message['channel_name']]
message_id = message_id_list[message_id_count]
# construct message
zulip_message = dict(
sending_client=1,
rendered_content_version=1, # This is Zulip-specific
has_image=message.get('has_image', False),
subject='from slack', # This is Zulip-specific
pub_date=float(message['ts']),
id=message_id,
has_attachment=
has_attachment, # attachment will be posted in the subsequent message;
# this is how Slack does it, i.e. less like email
edit_history=None,
sender=added_users[user], # map slack id to zulip id
content=content,
rendered_content=rendered_content, # slack doesn't cache this
recipient=recipient_id,
last_edit_time=None,
has_link=has_link)
zerver_message.append(zulip_message)
# construct usermessages
zerver_usermessage, usermessage_id_count = build_zerver_usermessage(
zerver_usermessage, usermessage_id_count, usermessage_id_list,
zerver_subscription, recipient_id, mentioned_users_id, message_id)
message_id_count += 1
return zerver_message, zerver_usermessage
def channel_message_to_zerver_message(
constants: List[Any], channel: str, added_users: AddedUsersT,
added_recipient: AddedRecipientsT,
zerver_subscription: List[ZerverFieldsT],
ids: List[int]) -> Tuple[List[ZerverFieldsT], List[ZerverFieldsT]]:
"""
Returns:
1. zerver_message, which is a list of the messages
2. zerver_usermessage, which is a list of the usermessages
"""
slack_data_dir, REALM_ID = constants
message_id, usermessage_id = ids
json_names = os.listdir(slack_data_dir + '/' + channel)
users = get_data_file(slack_data_dir + '/users.json')
zerver_message = []
zerver_usermessage = [] # type: List[ZerverFieldsT]
for json_name in json_names:
messages = get_data_file(slack_data_dir + '/%s/%s' %
(channel, json_name))
for message in messages:
has_attachment = False
content, mentioned_users_id, has_link = convert_to_zulip_markdown(
message['text'], users, added_users)
rendered_content = None
if 'subtype' in message.keys():
subtype = message['subtype']
if subtype in [
"channel_join", "channel_leave", "channel_name"
]:
continue
recipient_id = added_recipient[channel]
# construct message
zulip_message = dict(
sending_client=1,
rendered_content_version=1, # This is Zulip-specific
has_image=message.get('has_image', False),
subject='from slack', # This is Zulip-specific
pub_date=float(message['ts']),
id=message_id,
has_attachment=
has_attachment, # attachment will be posted in the subsequent message;
# this is how Slack does it, i.e. less like email
edit_history=None,
sender=added_users[get_message_sending_user(
message)], # map slack id to zulip id
content=content,
rendered_content=rendered_content, # slack doesn't cache this
recipient=recipient_id,
last_edit_time=None,
has_link=has_link)
zerver_message.append(zulip_message)
# construct usermessages
zerver_usermessage, usermessage_id = build_zerver_usermessage(
zerver_usermessage, usermessage_id, zerver_subscription,
recipient_id, mentioned_users_id, message_id)
message_id += 1
return zerver_message, zerver_usermessage
def test_mentioned_data(self) -> None:
slack_user_map = {'U08RGD1RD': 540, 'U0CBK5KAT': 554, 'U09TYF5SK': 571}
# For this test, only relevant keys are 'id', 'name', 'deleted'
# and 'real_name'
users = [{
"id": "U0CBK5KAT",
"name": "aaron.anzalone",
"deleted": False,
"real_name": ""
}, {
"id": "U08RGD1RD",
"name": "john",
"deleted": False,
"real_name": "John Doe"
}, {
"id": "U09TYF5Sk",
"name": "Jane",
"deleted": True
}] # Deleted users don't have 'real_name' key in Slack
channel_map = {'general': ('C5Z73A7RA', 137)}
message = 'Hi <@U08RGD1RD|john>: How are you? <#C5Z73A7RA|general>'
text, mentioned_users, has_link = convert_to_zulip_markdown(
message, users, channel_map, slack_user_map)
full_name = get_user_full_name(users[1])
self.assertEqual(full_name, 'John Doe')
self.assertEqual(get_user_full_name(users[2]), 'Jane')
self.assertEqual(text,
'Hi @**%s**: How are you? #**general**' % (full_name))
self.assertEqual(mentioned_users, [540])
# multiple mentioning
message = 'Hi <@U08RGD1RD|john>: How are you?<@U0CBK5KAT> asked.'
text, mentioned_users, has_link = convert_to_zulip_markdown(
message, users, channel_map, slack_user_map)
self.assertEqual(
text, 'Hi @**%s**: How are you?@**%s** asked.' %
('John Doe', 'aaron.anzalone'))
self.assertEqual(mentioned_users, [540, 554])
# Check wrong mentioning
message = 'Hi <@U08RGD1RD|jon>: How are you?'
text, mentioned_users, has_link = convert_to_zulip_markdown(
message, users, channel_map, slack_user_map)
self.assertEqual(text, message)
self.assertEqual(mentioned_users, [])
def test_has_link(self) -> None:
slack_user_map = {} # type: Dict[str, int]
message = '<http://journals.plos.org/plosone/article>'
text, mentioned_users, has_link = convert_to_zulip_markdown(
message, [], slack_user_map)
self.assertEqual(text, 'http://journals.plos.org/plosone/article')
self.assertEqual(has_link, True)
message = '<mailto:[email protected]>'
text, mentioned_users, has_link = convert_to_zulip_markdown(
message, [], slack_user_map)
self.assertEqual(text, 'mailto:[email protected]')
self.assertEqual(has_link, True)
message = 'random message'
text, mentioned_users, has_link = convert_to_zulip_markdown(
message, [], slack_user_map)
self.assertEqual(has_link, False)
def test_message_conversion_fixtures(self) -> None:
format_tests = self.load_slack_message_conversion_tests()
valid_keys = set(['name', "input", "conversion_output"])
for name, test in format_tests.items():
# Check that there aren't any unexpected keys as those are often typos
self.assertEqual(len(set(test.keys()) - valid_keys), 0)
slack_user_map = {} # type: Dict[str, int]
users = [{}] # type: List[Dict[str, Any]]
converted = convert_to_zulip_markdown(test['input'], users, slack_user_map)
converted_text = converted[0]
print("Running Slack Message Conversion test: %s" % (name,))
self.assertEqual(converted_text, test['conversion_output'])
def test_mentioned_data(self) -> None:
slack_user_map = {'U08RGD1RD': 540,
'U0CBK5KAT': 554,
'U09TYF5SK': 571}
# For this test, only relevant keys are 'id', 'name', 'deleted'
# and 'real_name'
users = [{"id": "U0CBK5KAT",
"name": "aaron.anzalone",
"deleted": False,
"real_name": ""},
{"id": "U08RGD1RD",
"name": "john",
"deleted": False,
"real_name": "John Doe"},
{"id": "U09TYF5Sk",
"name": "Jane",
"deleted": True}] # Deleted users don't have 'real_name' key in Slack
channel_map = {'general': ('C5Z73A7RA', 137)}
message = 'Hi <@U08RGD1RD|john>: How are you? <#C5Z73A7RA|general>'
text, mentioned_users, has_link = convert_to_zulip_markdown(message, users, channel_map, slack_user_map)
full_name = get_user_full_name(users[1])
self.assertEqual(full_name, 'John Doe')
self.assertEqual(get_user_full_name(users[2]), 'Jane')
self.assertEqual(text, 'Hi @**%s**: How are you? #**general**' % (full_name))
self.assertEqual(mentioned_users, [540])
# multiple mentioning
message = 'Hi <@U08RGD1RD|john>: How are you?<@U0CBK5KAT> asked.'
text, mentioned_users, has_link = convert_to_zulip_markdown(message, users, channel_map, slack_user_map)
self.assertEqual(text, 'Hi @**%s**: How are you?@**%s** asked.' %
('John Doe', 'aaron.anzalone'))
self.assertEqual(mentioned_users, [540, 554])
# Check wrong mentioning
message = 'Hi <@U08RGD1RD|jon>: How are you?'
text, mentioned_users, has_link = convert_to_zulip_markdown(message, users, channel_map, slack_user_map)
self.assertEqual(text, message)
self.assertEqual(mentioned_users, [])
def test_message_conversion_fixtures(self) -> None:
format_tests = self.load_slack_message_conversion_tests()
valid_keys = set(['name', "input", "conversion_output"])
for name, test in format_tests.items():
# Check that there aren't any unexpected keys as those are often typos
self.assertEqual(len(set(test.keys()) - valid_keys), 0)
slack_user_map = {} # type: Dict[str, int]
users = [{}] # type: List[Dict[str, Any]]
converted = convert_to_zulip_markdown(test['input'], users,
slack_user_map)
converted_text = converted[0]
print("Running Slack Message Conversion test: %s" % (name, ))
self.assertEqual(converted_text, test['conversion_output'])
def channel_message_to_zerver_message(realm_id: int, users: List[ZerverFieldsT],
added_users: AddedUsersT,
added_recipient: AddedRecipientsT,
all_messages: List[ZerverFieldsT],
zerver_realmemoji: List[ZerverFieldsT],
zerver_subscription: List[ZerverFieldsT],
added_channels: AddedChannelsT,
domain_name: str) -> Tuple[List[ZerverFieldsT],
List[ZerverFieldsT],
List[ZerverFieldsT],
List[ZerverFieldsT],
List[ZerverFieldsT]]:
"""
Returns:
1. zerver_message, which is a list of the messages
2. zerver_usermessage, which is a list of the usermessages
3. zerver_attachment, which is a list of the attachments
4. uploads_list, which is a list of uploads to be mapped in uploads records.json
5. reaction_list, which is a list of all user reactions
"""
message_id_count = usermessage_id_count = attachment_id_count = reaction_id_count = 0
zerver_message = []
zerver_usermessage = [] # type: List[ZerverFieldsT]
uploads_list = [] # type: List[ZerverFieldsT]
zerver_attachment = [] # type: List[ZerverFieldsT]
reaction_list = [] # type: List[ZerverFieldsT]
# For unicode emoji
with open(NAME_TO_CODEPOINT_PATH) as fp:
name_to_codepoint = ujson.load(fp)
for message in all_messages:
user = get_message_sending_user(message)
if not user:
# Ignore messages without user names
# These are Sometimes produced by slack
continue
has_attachment = has_image = False
content, mentioned_users_id, has_link = convert_to_zulip_markdown(message['text'],
users,
added_channels,
added_users)
rendered_content = None
recipient_id = added_recipient[message['channel_name']]
message_id = message_id_count
# Process message reactions
if 'reactions' in message.keys():
reaction_id_count = build_reactions(reaction_list, message['reactions'], added_users,
message_id, reaction_id_count, name_to_codepoint,
zerver_realmemoji)
# Process different subtypes of slack messages
if 'subtype' in message.keys():
subtype = message['subtype']
if subtype in ["channel_join", "channel_leave", "channel_name"]:
continue
# Subtypes which have only the action in the message should
# be rendered with '/me' in the content initially
# For example "sh_room_created" has the message 'started a call'
# which should be displayed as '/me started a call'
elif subtype in ["bot_add", "sh_room_created", "me_message"]:
content = ('/me %s' % (content))
# For attachments with slack download link
elif subtype == "file_share" and 'files.slack.com' in message['file']['url_private']:
fileinfo = message['file']
has_attachment = has_link = True
has_image = True if 'image' in fileinfo['mimetype'] else False
file_user = [iterate_user for iterate_user in users if message['user'] == user]
file_user_email = get_user_email(file_user[0], domain_name)
s3_path, content = get_attachment_path_and_content(fileinfo, realm_id)
# construct attachments
build_uploads(added_users[user], realm_id, file_user_email, fileinfo, s3_path,
uploads_list)
attachment_id = attachment_id_count
build_zerver_attachment(realm_id, message_id, attachment_id, added_users[user],
fileinfo, s3_path, zerver_attachment)
attachment_id_count += 1
# For attachments with link not from slack
# Example: Google drive integration
elif subtype == "file_share":
fileinfo = message['file']
has_link = True
if 'title' in fileinfo:
file_name = fileinfo['title']
else:
file_name = fileinfo['name']
content = '[%s](%s)' % (file_name, fileinfo['url_private'])
# construct message
zulip_message = dict(
sending_client=1,
rendered_content_version=1, # This is Zulip-specific
has_image=has_image,
subject='imported from slack', # This is Zulip-specific
pub_date=float(message['ts']),
id=message_id,
has_attachment=has_attachment, # attachment will be posted in the subsequent message;
# this is how Slack does it, i.e. less like email
edit_history=None,
sender=added_users[user], # map slack id to zulip id
content=content,
rendered_content=rendered_content, # slack doesn't cache this
recipient=recipient_id,
last_edit_time=None,
has_link=has_link)
zerver_message.append(zulip_message)
# construct usermessages
usermessage_id_count = build_zerver_usermessage(
zerver_usermessage, usermessage_id_count, zerver_subscription,
recipient_id, mentioned_users_id, message_id)
message_id_count += 1
return zerver_message, zerver_usermessage, zerver_attachment, uploads_list, reaction_list
def channel_message_to_zerver_message(realm_id: int, users: List[ZerverFieldsT],
added_users: AddedUsersT,
added_recipient: AddedRecipientsT,
all_messages: List[ZerverFieldsT],
zerver_realmemoji: List[ZerverFieldsT],
zerver_subscription: List[ZerverFieldsT],
added_channels: AddedChannelsT,
domain_name: str) -> Tuple[List[ZerverFieldsT],
List[ZerverFieldsT],
List[ZerverFieldsT],
List[ZerverFieldsT],
List[ZerverFieldsT]]:
"""
Returns:
1. zerver_message, which is a list of the messages
2. zerver_usermessage, which is a list of the usermessages
3. zerver_attachment, which is a list of the attachments
4. uploads_list, which is a list of uploads to be mapped in uploads records.json
5. reaction_list, which is a list of all user reactions
"""
message_id_count = usermessage_id_count = attachment_id_count = reaction_id_count = 0
zerver_message = []
zerver_usermessage = [] # type: List[ZerverFieldsT]
uploads_list = [] # type: List[ZerverFieldsT]
zerver_attachment = [] # type: List[ZerverFieldsT]
reaction_list = [] # type: List[ZerverFieldsT]
# For unicode emoji
with open(NAME_TO_CODEPOINT_PATH) as fp:
name_to_codepoint = ujson.load(fp)
for message in all_messages:
user = get_message_sending_user(message)
if not user:
# Ignore messages without user names
# These are Sometimes produced by slack
continue
if message.get('subtype') in [
# Zulip doesn't have a pinned_item concept
"pinned_item",
"unpinned_item",
# Slack's channel join/leave notices are spammy
"channel_join",
"channel_leave",
"channel_name"
]:
continue
has_attachment = has_image = False
try:
content, mentioned_users_id, has_link = convert_to_zulip_markdown(
message['text'], users, added_channels, added_users)
except Exception:
print("Slack message unexpectedly missing text representation:")
print(json.dumps(message, indent=4))
continue
rendered_content = None
recipient_id = added_recipient[message['channel_name']]
message_id = message_id_count
# Process message reactions
if 'reactions' in message.keys():
reaction_id_count = build_reactions(reaction_list, message['reactions'], added_users,
message_id, reaction_id_count, name_to_codepoint,
zerver_realmemoji)
# Process different subtypes of slack messages
if 'subtype' in message.keys():
subtype = message['subtype']
# Subtypes which have only the action in the message should
# be rendered with '/me' in the content initially
# For example "sh_room_created" has the message 'started a call'
# which should be displayed as '/me started a call'
if subtype in ["bot_add", "sh_room_created", "me_message"]:
content = ('/me %s' % (content))
# For attachments with slack download link
elif subtype == "file_share" and 'files.slack.com' in message['file']['url_private']:
fileinfo = message['file']
has_attachment = has_link = True
has_image = True if 'image' in fileinfo['mimetype'] else False
file_user = [iterate_user for iterate_user in users if message['user'] == user]
file_user_email = get_user_email(file_user[0], domain_name)
s3_path, content = get_attachment_path_and_content(fileinfo, realm_id)
# construct attachments
build_uploads(added_users[user], realm_id, file_user_email, fileinfo, s3_path,
uploads_list)
attachment_id = attachment_id_count
build_zerver_attachment(realm_id, message_id, attachment_id, added_users[user],
fileinfo, s3_path, zerver_attachment)
attachment_id_count += 1
# For attachments with link not from slack
# Example: Google drive integration
elif subtype == "file_share":
fileinfo = message['file']
has_link = True
if 'title' in fileinfo:
file_name = fileinfo['title']
else:
file_name = fileinfo['name']
content = '[%s](%s)' % (file_name, fileinfo['url_private'])
# construct message
zulip_message = dict(
sending_client=1,
rendered_content_version=1, # This is Zulip-specific
has_image=has_image,
subject='imported from slack', # This is Zulip-specific
pub_date=float(message['ts']),
id=message_id,
has_attachment=has_attachment, # attachment will be posted in the subsequent message;
# this is how Slack does it, i.e. less like email
edit_history=None,
sender=added_users[user], # map slack id to zulip id
content=content,
rendered_content=rendered_content, # slack doesn't cache this
recipient=recipient_id,
last_edit_time=None,
has_link=has_link)
zerver_message.append(zulip_message)
# construct usermessages
usermessage_id_count = build_zerver_usermessage(
zerver_usermessage, usermessage_id_count, zerver_subscription,
recipient_id, mentioned_users_id, message_id)
message_id_count += 1
return zerver_message, zerver_usermessage, zerver_attachment, uploads_list, reaction_list
def channel_message_to_zerver_message(
constants: List[Any], channel: str, added_users: AddedUsersT,
added_recipient: AddedRecipientsT,
zerver_userprofile: List[ZerverFieldsT],
zerver_subscription: List[ZerverFieldsT],
ids: List[int]) -> Tuple[List[ZerverFieldsT], List[ZerverFieldsT]]:
"""
Returns:
1. zerver_message, which is a list of the messages
2. zerver_usermessage, which is a list of the usermessages
"""
slack_data_dir, REALM_ID = constants
message_id, usermessage_id = ids
json_names = os.listdir(slack_data_dir + '/' + channel)
users = json.load(open(slack_data_dir + '/users.json'))
zerver_message = []
zerver_usermessage = []
for json_name in json_names:
messages = json.load(
open(slack_data_dir + '/%s/%s' % (channel, json_name)))
for message in messages:
has_attachment = False
content, mentioned_users_id, has_link = convert_to_zulip_markdown(
message['text'], users, added_users)
rendered_content = None
if 'subtype' in message.keys():
subtype = message['subtype']
if subtype in [
"channel_join", "channel_leave", "channel_name"
]:
continue
try:
user = message.get('user', message['file']['user'])
except KeyError:
user = message['user']
recipient_id = added_recipient[channel]
# construct message
zulip_message = dict(
sending_client=1,
rendered_content_version=1, # This is Zulip-specific
has_image=message.get('has_image', False),
subject=channel, # This is Zulip-specific
pub_date=float(message['ts']),
id=message_id,
has_attachment=
has_attachment, # attachment will be posted in the subsequent message;
# this is how Slack does it, i.e. less like email
edit_history=None,
sender=added_users[user], # map slack id to zulip id
content=content,
rendered_content=rendered_content, # slack doesn't cache this
recipient=recipient_id,
last_edit_time=None,
has_link=has_link)
zerver_message.append(zulip_message)
# construct usermessages
for subscription in zerver_subscription:
if subscription['recipient'] == recipient_id:
flags_mask = 1 # For read
if subscription['user_profile'] in mentioned_users_id:
flags_mask = 9 # For read and mentioned
usermessage = dict(
user_profile=subscription['user_profile'],
id=usermessage_id,
flags_mask=flags_mask,
message=zulip_message['id'])
usermessage_id += 1
zerver_usermessage.append(usermessage)
message_id += 1
return zerver_message, zerver_usermessage
def channel_message_to_zerver_message(constants: List[Any], channel: str,
added_users: AddedUsersT, added_recipient: AddedRecipientsT,
zerver_userprofile: List[ZerverFieldsT],
zerver_subscription: List[ZerverFieldsT],
ids: List[int]) -> Tuple[List[ZerverFieldsT],
List[ZerverFieldsT]]:
"""
Returns:
1. zerver_message, which is a list of the messages
2. zerver_usermessage, which is a list of the usermessages
"""
slack_data_dir, REALM_ID = constants
message_id, usermessage_id = ids
json_names = os.listdir(slack_data_dir + '/' + channel)
users = json.load(open(slack_data_dir + '/users.json'))
zerver_message = []
zerver_usermessage = []
for json_name in json_names:
messages = json.load(open(slack_data_dir + '/%s/%s' % (channel, json_name)))
for message in messages:
has_attachment = False
content, mentioned_users_id, has_link = convert_to_zulip_markdown(message['text'],
users,
added_users)
rendered_content = None
if 'subtype' in message.keys():
subtype = message['subtype']
if subtype in ["channel_join", "channel_leave", "channel_name"]:
continue
try:
user = message.get('user', message['file']['user'])
except KeyError:
user = message['user']
recipient_id = added_recipient[channel]
# construct message
zulip_message = dict(
sending_client=1,
rendered_content_version=1, # This is Zulip-specific
has_image=message.get('has_image', False),
subject=channel, # This is Zulip-specific
pub_date=float(message['ts']),
id=message_id,
has_attachment=has_attachment, # attachment will be posted in the subsequent message;
# this is how Slack does it, i.e. less like email
edit_history=None,
sender=added_users[user], # map slack id to zulip id
content=content,
rendered_content=rendered_content, # slack doesn't cache this
recipient=recipient_id,
last_edit_time=None,
has_link=has_link)
zerver_message.append(zulip_message)
# construct usermessages
for subscription in zerver_subscription:
if subscription['recipient'] == zulip_message['recipient']:
flags_mask = 1 # For read
if subscription['user_profile'] in mentioned_users_id:
flags_mask = 9 # For read and mentioned
usermessage = dict(
user_profile=subscription['user_profile'],
id=usermessage_id,
flags_mask=flags_mask,
message=zulip_message['id'])
usermessage_id += 1
zerver_usermessage.append(usermessage)
message_id += 1
return zerver_message, zerver_usermessage
def channel_message_to_zerver_message(realm_id: int, users: List[ZerverFieldsT],
added_users: AddedUsersT,
added_recipient: AddedRecipientsT,
all_messages: List[ZerverFieldsT],
zerver_subscription: List[ZerverFieldsT],
domain_name: str,
ids: List[Any]) -> Tuple[List[ZerverFieldsT],
List[ZerverFieldsT],
List[ZerverFieldsT],
List[ZerverFieldsT]]:
"""
Returns:
1. zerver_message, which is a list of the messages
2. zerver_usermessage, which is a list of the usermessages
3. zerver_attachment, which is a list of the attachments
4. uploads_list, which is a list of uploads to be mapped in uploads records.json
"""
message_id_count = usermessage_id_count = attachment_id_count = 0
message_id_list, usermessage_id_list, attachment_id_list = ids
zerver_message = []
zerver_usermessage = [] # type: List[ZerverFieldsT]
uploads_list = [] # type: List[ZerverFieldsT]
zerver_attachment = [] # type: List[ZerverFieldsT]
for message in all_messages:
user = get_message_sending_user(message)
if not user:
# Ignore messages without user names
# These are Sometimes produced by slack
continue
has_attachment = has_image = False
content, mentioned_users_id, has_link = convert_to_zulip_markdown(message['text'],
users,
added_users)
rendered_content = None
recipient_id = added_recipient[message['channel_name']]
message_id = message_id_list[message_id_count]
if 'subtype' in message.keys():
subtype = message['subtype']
if subtype in ["channel_join", "channel_leave", "channel_name"]:
continue
# For attachments with slack download link
elif subtype == "file_share" and 'files.slack.com' in message['file']['url_private']:
fileinfo = message['file']
has_attachment = has_link = True
has_image = True if 'image' in fileinfo['mimetype'] else False
file_user = [iterate_user for iterate_user in users if message['user'] == user]
file_user_email = get_user_email(file_user[0], domain_name)
s3_path, content = get_attachment_path_and_content(fileinfo, realm_id)
# construct attachments
build_uploads(added_users[user], realm_id, file_user_email, fileinfo, s3_path,
uploads_list)
attachment_id = attachment_id_list[attachment_id_count]
build_zerver_attachment(realm_id, message_id, attachment_id, added_users[user],
fileinfo, s3_path, zerver_attachment)
attachment_id_count += 1
# For attachments with link not from slack
# Example: Google drive integration
elif subtype == "file_share":
fileinfo = message['file']
has_link = True
if 'title' in fileinfo:
file_name = fileinfo['title']
else:
file_name = fileinfo['name']
content = '[%s](%s)' % (file_name, fileinfo['url_private'])
# construct message
zulip_message = dict(
sending_client=1,
rendered_content_version=1, # This is Zulip-specific
has_image=has_image,
subject='imported from slack', # This is Zulip-specific
pub_date=float(message['ts']),
id=message_id,
has_attachment=has_attachment, # attachment will be posted in the subsequent message;
# this is how Slack does it, i.e. less like email
edit_history=None,
sender=added_users[user], # map slack id to zulip id
content=content,
rendered_content=rendered_content, # slack doesn't cache this
recipient=recipient_id,
last_edit_time=None,
has_link=has_link)
zerver_message.append(zulip_message)
# construct usermessages
usermessage_id_count = build_zerver_usermessage(
zerver_usermessage, usermessage_id_count, usermessage_id_list,
zerver_subscription, recipient_id, mentioned_users_id, message_id)
message_id_count += 1
return zerver_message, zerver_usermessage, zerver_attachment, uploads_list
