def process(in_dict: ZerverFieldsT) -> ZerverFieldsT:
now = int(timezone_now().timestamp())
out_dict = build_stream(
date_created=now,
realm_id=realm_id,
name=in_dict['name'],
description=in_dict['topic'],
stream_id=in_dict['id'],
deactivated=in_dict['is_archived'],
invite_only=invite_only(in_dict['privacy']),
)
subscriber_handler.set_info(
stream_id=in_dict['id'],
owner=in_dict['owner'],
members=set(in_dict['members']),
)
# unmapped fields:
# guest_access_url: no Zulip equivalent
# created: we just use "now"
# members: no good sample data
# owners: no good sample data
# participants: no good sample data
return out_dict
def convert_room_data(raw_data: List[ZerverFieldsT],
subscriber_handler: SubscriberHandler,
stream_id_mapper: IdMapper, user_id_mapper: IdMapper,
realm_id: int) -> List[ZerverFieldsT]:
flat_data = [d['Room'] for d in raw_data]
def get_invite_only(v: str) -> bool:
if v == 'public':
return False
elif v == 'private':
return True
else:
raise Exception('unexpected value')
streams = []
for in_dict in flat_data:
now = int(timezone_now().timestamp())
stream_id = stream_id_mapper.get(in_dict['id'])
invite_only = get_invite_only(in_dict['privacy'])
stream = build_stream(
date_created=now,
realm_id=realm_id,
name=in_dict['name'],
description=in_dict['topic'],
stream_id=stream_id,
deactivated=in_dict['is_archived'],
invite_only=invite_only,
)
if invite_only:
users = {
user_id_mapper.get(key)
for key in in_dict['members'] if user_id_mapper.has(key)
} # type: Set[int]
if user_id_mapper.has(in_dict['owner']):
owner = user_id_mapper.get(in_dict['owner'])
users.add(owner)
if not users:
continue
subscriber_handler.set_info(
stream_id=stream_id,
users=users,
)
# unmapped fields:
# guest_access_url: no Zulip equivalent
# created: we just use "now"
# participants: no good sample data
streams.append(stream)
return streams
def process_channels(channels: List[Dict[str, Any]],
invite_only: bool = False) -> None:
nonlocal stream_id_count
nonlocal recipient_id_count
nonlocal defaultstream_id
nonlocal subscription_id_count
for channel in channels:
# slack_channel_id = channel['id']
# map Slack's topic and purpose content into Zulip's stream description.
# WARN This mapping is lossy since the topic.creator, topic.last_set,
# purpose.creator, purpose.last_set fields are not preserved.
description = channel["purpose"]["value"]
stream_id = stream_id_count
recipient_id = recipient_id_count
# construct the stream object and append it to zerver_stream
stream = build_stream(float(channel["created"]), realm_id,
channel["name"], description, stream_id,
channel["is_archived"], invite_only)
zerver_stream.append(stream)
# construct defaultstream object
# slack has the default channel 'general' and 'random'
# where every user is subscribed
default_channels = ['general', 'random'] # Slack specific
if channel['name'] in default_channels:
defaultstream = build_defaultstream(realm_id, stream_id,
defaultstream_id)
zerver_defaultstream.append(defaultstream)
defaultstream_id += 1
added_channels[stream['name']] = (channel['id'], stream_id)
recipient = build_recipient(stream_id, recipient_id,
Recipient.STREAM)
zerver_recipient.append(recipient)
added_recipient[stream['name']] = recipient_id
# TODO add recipients for private message and huddles
# construct the subscription object and append it to zerver_subscription
subscription_id_count = get_subscription(channel['members'],
zerver_subscription,
recipient_id, added_users,
subscription_id_count)
# TODO add zerver_subscription which correspond to
# huddles type recipient
# For huddles:
# sub['recipient']=recipient['id'] where recipient['type_id']=added_users[member]
stream_id_count += 1
recipient_id_count += 1
logging.info(u"{} -> created".format(channel['name']))
def convert_channel_data(
room_id_to_room_map: Dict[str, Dict[str, Any]],
team_id_to_team_map: Dict[str, Dict[str, Any]],
stream_id_mapper: IdMapper,
realm_id: int,
) -> List[ZerverFieldsT]:
streams = []
for rc_room_id in room_id_to_room_map:
channel_dict = room_id_to_room_map[rc_room_id]
date_created = float(channel_dict["ts"].timestamp())
stream_id = stream_id_mapper.get(rc_room_id)
invite_only = channel_dict["t"] == "p"
stream_name = channel_dict["name"]
stream_desc = channel_dict.get("description", "")
if channel_dict.get("teamId"):
if channel_dict.get("teamMain") is True:
# In case you change this, please also change the stream name
# used while adding the Rocket.Chat channel mention data to
# message_dict in `message_to_dict` inner-function of
# `process_messages` function below.
stream_name = "[TEAM] " + stream_name
else:
stream_desc = "[Team {} channel]. {}".format(
team_id_to_team_map[channel_dict["teamId"]]["name"],
stream_desc)
# If the channel is read-only, then only admins and moderators
# should be allowed to post in the converted Zulip stream.
# For more details: https://zulip.com/help/stream-sending-policy
#
# See `Stream` model in `zerver/models.py` to know about what each
# number represent.
stream_post_policy = 4 if channel_dict.get("ro", False) else 1
stream = build_stream(
date_created=date_created,
realm_id=realm_id,
name=stream_name,
description=stream_desc,
stream_id=stream_id,
deactivated=False,
invite_only=invite_only,
stream_post_policy=stream_post_policy,
)
streams.append(stream)
return streams
def build_stream_and_defaultstream(timestamp: Any) -> Tuple[List[ZerverFieldsT],
List[ZerverFieldsT]]:
logging.info('######### IMPORTING STREAM STARTED #########\n')
# We have only one stream for gitter export
stream_name = 'from gitter'
stream_description = "Imported from gitter"
stream_id = 0
stream = build_stream(timestamp, realm_id, stream_name, stream_description,
stream_id)
defaultstream = build_defaultstream(realm_id=realm_id, stream_id=stream_id,
defaultstream_id=0)
logging.info('######### IMPORTING STREAMS FINISHED #########\n')
return [stream], [defaultstream]
def process_channels(channels: List[Dict[str, Any]], invite_only: bool = False) -> None:
nonlocal stream_id_count
nonlocal recipient_id_count
nonlocal defaultstream_id
nonlocal subscription_id_count
for channel in channels:
# map Slack's topic and purpose content into Zulip's stream description.
# WARN This mapping is lossy since the topic.creator, topic.last_set,
# purpose.creator, purpose.last_set fields are not preserved.
description = channel["purpose"]["value"]
stream_id = stream_id_count
recipient_id = recipient_id_count
stream = build_stream(
float(channel["created"]),
realm_id,
channel["name"],
description,
stream_id,
channel["is_archived"],
invite_only,
)
realm["zerver_stream"].append(stream)
slack_default_channels = ["general", "random"]
if channel["name"] in slack_default_channels and not stream["deactivated"]:
defaultstream = build_defaultstream(realm_id, stream_id, defaultstream_id)
realm["zerver_defaultstream"].append(defaultstream)
defaultstream_id += 1
added_channels[stream["name"]] = (channel["id"], stream_id)
recipient = build_recipient(stream_id, recipient_id, Recipient.STREAM)
realm["zerver_recipient"].append(recipient)
slack_recipient_name_to_zulip_recipient_id[stream["name"]] = recipient_id
subscription_id_count = get_subscription(
channel["members"],
realm["zerver_subscription"],
recipient_id,
slack_user_id_to_zulip_user_id,
subscription_id_count,
)
stream_id_count += 1
recipient_id_count += 1
logging.info("%s -> created", channel["name"])
def convert_channel_data(
room_id_to_room_map: Dict[str, Dict[str, Any]],
team_id_to_team_map: Dict[str, Dict[str, Any]],
stream_id_mapper: IdMapper,
realm_id: int,
) -> List[ZerverFieldsT]:
streams = []
for rc_room_id in room_id_to_room_map:
channel_dict = room_id_to_room_map[rc_room_id]
date_created = float(channel_dict["ts"].timestamp())
stream_id = stream_id_mapper.get(rc_room_id)
invite_only = channel_dict["t"] == "p"
stream_name = get_stream_name(channel_dict)
stream_desc = channel_dict.get("description", "")
if channel_dict.get("teamId") and not channel_dict.get("teamMain"):
stream_desc = "[Team {} channel]. {}".format(
team_id_to_team_map[channel_dict["teamId"]]["name"],
stream_desc)
# If the channel is read-only, then only admins and moderators
# should be allowed to post in the converted Zulip stream.
# For more details: https://zulip.com/help/stream-sending-policy
#
# See `Stream` model in `zerver/models.py` to know about what each
# number represent.
stream_post_policy = 4 if channel_dict.get("ro", False) else 1
stream = build_stream(
date_created=date_created,
realm_id=realm_id,
name=stream_name,
description=stream_desc,
stream_id=stream_id,
deactivated=False,
invite_only=invite_only,
stream_post_policy=stream_post_policy,
)
streams.append(stream)
return streams
def process(in_dict: ZerverFieldsT) -> ZerverFieldsT:
now = int(timezone_now().timestamp())
out_dict = build_stream(
date_created=now,
realm_id=realm_id,
name=in_dict['name'],
description=in_dict['topic'],
stream_id=in_dict['id'],
deactivated=in_dict['is_archived'],
invite_only=invite_only(in_dict['privacy']),
)
# unmapped fields:
# guest_access_url: no Zulip equivalent
# created: we just use "now"
# members: no good sample data
# owners: no good sample data
# participants: no good sample data
return out_dict
def process(in_dict: ZerverFieldsT) -> ZerverFieldsT:
now = int(timezone_now().timestamp())
stream_id = stream_id_mapper.get(in_dict['id'])
out_dict = build_stream(
date_created=now,
realm_id=realm_id,
name=in_dict['name'],
description=in_dict['topic'],
stream_id=stream_id,
deactivated=in_dict['is_archived'],
invite_only=invite_only(in_dict['privacy']),
)
if not user_id_mapper.has(in_dict['owner']):
raise Exception('bad owner')
owner = user_id_mapper.get(in_dict['owner'])
members = {
user_id_mapper.get(key)
for key in in_dict['members'] if user_id_mapper.has(key)
}
subscriber_handler.set_info(
stream_id=stream_id,
owner=owner,
members=members,
)
# unmapped fields:
# guest_access_url: no Zulip equivalent
# created: we just use "now"
# members: no good sample data
# owners: no good sample data
# participants: no good sample data
return out_dict
def channels_to_zerver_stream(slack_data_dir: str, realm_id: int, added_users: AddedUsersT,
zerver_userprofile: List[ZerverFieldsT]) -> Tuple[List[ZerverFieldsT],
List[ZerverFieldsT],
AddedChannelsT,
List[ZerverFieldsT],
List[ZerverFieldsT],
AddedRecipientsT]:
"""
Returns:
1. zerver_defaultstream, which is a list of the default streams
2. zerver_stream, while is a list of all streams
3. added_channels, which is a dictionary to map from channel name to channel id, zulip stream_id
4. zerver_subscription, which is a list of the subscriptions
5. zerver_recipient, which is a list of the recipients
6. added_recipient, which is a dictionary to map from channel name to zulip recipient_id
"""
logging.info('######### IMPORTING CHANNELS STARTED #########\n')
channels = get_data_file(slack_data_dir + '/channels.json')
added_channels = {}
added_recipient = {}
zerver_stream = []
zerver_subscription = [] # type: List[ZerverFieldsT]
zerver_recipient = []
zerver_defaultstream = []
stream_id_count = subscription_id_count = recipient_id_count = defaultstream_id = 0
for channel in channels:
# slack_channel_id = channel['id']
# map Slack's topic and purpose content into Zulip's stream description.
# WARN This mapping is lossy since the topic.creator, topic.last_set,
# purpose.creator, purpose.last_set fields are not preserved.
description = channel["purpose"]["value"]
stream_id = stream_id_count
recipient_id = recipient_id_count
# construct the stream object and append it to zerver_stream
stream = build_stream(float(channel["created"]), realm_id, channel["name"],
description, stream_id, channel["is_archived"])
zerver_stream.append(stream)
# construct defaultstream object
# slack has the default channel 'general' and 'random'
# where every user is subscribed
default_channels = ['general', 'random'] # Slack specific
if channel['name'] in default_channels:
defaultstream = build_defaultstream(realm_id, stream_id,
defaultstream_id)
zerver_defaultstream.append(defaultstream)
defaultstream_id += 1
added_channels[stream['name']] = (channel['id'], stream_id)
recipient = build_recipient(stream_id, recipient_id, Recipient.STREAM)
zerver_recipient.append(recipient)
added_recipient[stream['name']] = recipient_id
# TODO add recipients for private message and huddles
# construct the subscription object and append it to zerver_subscription
subscription_id_count = get_subscription(channel['members'], zerver_subscription,
recipient_id, added_users,
subscription_id_count)
# TODO add zerver_subscription which correspond to
# huddles type recipient
# For huddles:
# sub['recipient']=recipient['id'] where recipient['type_id']=added_users[member]
stream_id_count += 1
recipient_id_count += 1
logging.info(u"{} -> created".format(channel['name']))
# TODO map Slack's pins to Zulip's stars
# There is the security model that Slack's pins are known to the team owner
# as evident from where it is stored at (channels)
# "pins": [
# {
# "id": "1444755381.000003",
# "type": "C",
# "user": "******",
# "owner": "U061A5N1G",
# "created": "1444755463"
# }
# ],
for user in zerver_userprofile:
# this maps the recipients and subscriptions
# related to private messages
recipient = build_recipient(user['id'], recipient_id_count, Recipient.PERSONAL)
sub = build_subscription(recipient_id_count, user['id'], subscription_id_count)
zerver_recipient.append(recipient)
zerver_subscription.append(sub)
subscription_id_count += 1
recipient_id_count += 1
logging.info('######### IMPORTING STREAMS FINISHED #########\n')
return zerver_defaultstream, zerver_stream, added_channels, zerver_subscription, \
zerver_recipient, added_recipient
def channels_to_zerver_stream(slack_data_dir: str, realm_id: int, added_users: AddedUsersT,
zerver_userprofile: List[ZerverFieldsT]) -> Tuple[List[ZerverFieldsT],
List[ZerverFieldsT],
AddedChannelsT,
List[ZerverFieldsT],
List[ZerverFieldsT],
AddedRecipientsT]:
"""
Returns:
1. zerver_defaultstream, which is a list of the default streams
2. zerver_stream, while is a list of all streams
3. added_channels, which is a dictionary to map from channel name to channel id, zulip stream_id
4. zerver_subscription, which is a list of the subscriptions
5. zerver_recipient, which is a list of the recipients
6. added_recipient, which is a dictionary to map from channel name to zulip recipient_id
"""
logging.info('######### IMPORTING CHANNELS STARTED #########\n')
channels = get_data_file(slack_data_dir + '/channels.json')
added_channels = {}
added_recipient = {}
zerver_stream = []
zerver_subscription = [] # type: List[ZerverFieldsT]
zerver_recipient = []
zerver_defaultstream = []
stream_id_count = subscription_id_count = recipient_id_count = defaultstream_id = 0
for channel in channels:
# slack_channel_id = channel['id']
# map Slack's topic and purpose content into Zulip's stream description.
# WARN This mapping is lossy since the topic.creator, topic.last_set,
# purpose.creator, purpose.last_set fields are not preserved.
description = channel["purpose"]["value"]
stream_id = stream_id_count
recipient_id = recipient_id_count
# construct the stream object and append it to zerver_stream
stream = build_stream(float(channel["created"]), realm_id, channel["name"],
description, stream_id, channel["is_archived"])
zerver_stream.append(stream)
# construct defaultstream object
# slack has the default channel 'general' and 'random'
# where every user is subscribed
default_channels = ['general', 'random'] # Slack specific
if channel['name'] in default_channels:
defaultstream = build_defaultstream(realm_id, stream_id,
defaultstream_id)
zerver_defaultstream.append(defaultstream)
defaultstream_id += 1
added_channels[stream['name']] = (channel['id'], stream_id)
recipient = build_recipient(stream_id, recipient_id, Recipient.STREAM)
zerver_recipient.append(recipient)
added_recipient[stream['name']] = recipient_id
# TODO add recipients for private message and huddles
# construct the subscription object and append it to zerver_subscription
subscription_id_count = get_subscription(channel['members'], zerver_subscription,
recipient_id, added_users,
subscription_id_count)
# TODO add zerver_subscription which correspond to
# huddles type recipient
# For huddles:
# sub['recipient']=recipient['id'] where recipient['type_id']=added_users[member]
stream_id_count += 1
recipient_id_count += 1
logging.info(u"{} -> created".format(channel['name']))
# TODO map Slack's pins to Zulip's stars
# There is the security model that Slack's pins are known to the team owner
# as evident from where it is stored at (channels)
# "pins": [
# {
# "id": "1444755381.000003",
# "type": "C",
# "user": "******",
# "owner": "U061A5N1G",
# "created": "1444755463"
# }
# ],
for user in zerver_userprofile:
# this maps the recipients and subscriptions
# related to private messages
recipient = build_recipient(user['id'], recipient_id_count, Recipient.PERSONAL)
sub = build_subscription(recipient_id_count, user['id'], subscription_id_count)
zerver_recipient.append(recipient)
zerver_subscription.append(sub)
subscription_id_count += 1
recipient_id_count += 1
logging.info('######### IMPORTING STREAMS FINISHED #########\n')
return zerver_defaultstream, zerver_stream, added_channels, zerver_subscription, \
zerver_recipient, added_recipient
def convert_room_data(raw_data: List[ZerverFieldsT],
subscriber_handler: SubscriberHandler,
stream_id_mapper: IdMapper,
user_id_mapper: IdMapper,
realm_id: int,
api_token: Optional[str] = None) -> List[ZerverFieldsT]:
flat_data = [d['Room'] for d in raw_data]
def get_invite_only(v: str) -> bool:
if v == 'public':
return False
elif v == 'private':
return True
else:
raise Exception('unexpected value')
streams = []
for in_dict in flat_data:
now = int(timezone_now().timestamp())
stream_id = stream_id_mapper.get(in_dict['id'])
invite_only = get_invite_only(in_dict['privacy'])
stream = build_stream(
date_created=now,
realm_id=realm_id,
name=in_dict['name'],
description=in_dict['topic'],
stream_id=stream_id,
deactivated=in_dict['is_archived'],
invite_only=invite_only,
)
if invite_only:
users: Set[int] = {
user_id_mapper.get(key)
for key in in_dict['members'] if user_id_mapper.has(key)
}
if user_id_mapper.has(in_dict['owner']):
owner = user_id_mapper.get(in_dict['owner'])
users.add(owner)
else:
users = set()
if api_token is not None:
hc = hypchat.HypChat(api_token)
room_data = hc.fromurl('{}/v2/room/{}/member'.format(
hc.endpoint, in_dict['id']))
for item in room_data['items']:
hipchat_user_id = item['id']
zulip_user_id = user_id_mapper.get(hipchat_user_id)
users.add(zulip_user_id)
if users:
subscriber_handler.set_info(
stream_id=stream_id,
users=users,
)
# unmapped fields:
# guest_access_url: no Zulip equivalent
# created: we just use "now"
# participants: no good sample data
streams.append(stream)
return streams
def convert_channel_data(
channel_data: List[ZerverFieldsT],
user_data_map: Dict[str, Dict[str, Any]],
subscriber_handler: SubscriberHandler,
stream_id_mapper: IdMapper,
user_id_mapper: IdMapper,
realm_id: int,
team_name: str,
) -> List[ZerverFieldsT]:
channel_data_list = [d for d in channel_data if d["team"] == team_name]
channel_members_map: Dict[str, List[str]] = {}
channel_admins_map: Dict[str, List[str]] = {}
def initialize_stream_membership_dicts() -> None:
for channel in channel_data:
channel_name = channel["name"]
channel_members_map[channel_name] = []
channel_admins_map[channel_name] = []
for username in user_data_map:
user_dict = user_data_map[username]
teams = user_dict["teams"]
if user_dict["teams"] is None:
continue
for team in teams:
if team["name"] != team_name:
continue
for channel in team["channels"]:
channel_roles = channel["roles"]
channel_name = channel["name"]
if "channel_admin" in channel_roles:
channel_admins_map[channel_name].append(username)
elif "channel_user" in channel_roles:
channel_members_map[channel_name].append(username)
def get_invite_only_value_from_channel_type(channel_type: str) -> bool:
# Channel can have two types in Mattermost
# "O" for a public channel.
# "P" for a private channel.
if channel_type == "O":
return False
elif channel_type == "P":
return True
else: # nocoverage
raise Exception("unexpected value")
streams = []
initialize_stream_membership_dicts()
for channel_dict in channel_data_list:
now = int(timezone_now().timestamp())
stream_id = stream_id_mapper.get(channel_dict["name"])
stream_name = channel_dict["name"]
invite_only = get_invite_only_value_from_channel_type(channel_dict["type"])
stream = build_stream(
date_created=now,
realm_id=realm_id,
name=channel_dict["display_name"],
# Purpose describes how the channel should be used. It is similar to
# stream description and is shown in channel list to help others decide
# whether to join.
# Header text always appears right next to channel name in channel header.
# Can be used for advertising the purpose of stream, making announcements as
# well as including frequently used links. So probably not a bad idea to use
# this as description if the channel purpose is empty.
description=channel_dict["purpose"] or channel_dict["header"],
stream_id=stream_id,
# Mattermost export don't include data of archived(~ deactivated) channels.
deactivated=False,
invite_only=invite_only,
)
channel_users = set()
for username in channel_admins_map[stream_name]:
channel_users.add(user_id_mapper.get(username))
for username in channel_members_map[stream_name]:
channel_users.add(user_id_mapper.get(username))
subscriber_handler.set_info(
users=channel_users,
stream_id=stream_id,
)
streams.append(stream)
return streams
