def process(in_dict: ZerverFieldsT) -> ZerverFieldsT: now = int(timezone_now().timestamp()) out_dict = build_stream( date_created=now, realm_id=realm_id, name=in_dict['name'], description=in_dict['topic'], stream_id=in_dict['id'], deactivated=in_dict['is_archived'], invite_only=invite_only(in_dict['privacy']), ) subscriber_handler.set_info( stream_id=in_dict['id'], owner=in_dict['owner'], members=set(in_dict['members']), ) # unmapped fields: # guest_access_url: no Zulip equivalent # created: we just use "now" # members: no good sample data # owners: no good sample data # participants: no good sample data return out_dict
def convert_room_data(raw_data: List[ZerverFieldsT], subscriber_handler: SubscriberHandler, stream_id_mapper: IdMapper, user_id_mapper: IdMapper, realm_id: int) -> List[ZerverFieldsT]: flat_data = [d['Room'] for d in raw_data] def get_invite_only(v: str) -> bool: if v == 'public': return False elif v == 'private': return True else: raise Exception('unexpected value') streams = [] for in_dict in flat_data: now = int(timezone_now().timestamp()) stream_id = stream_id_mapper.get(in_dict['id']) invite_only = get_invite_only(in_dict['privacy']) stream = build_stream( date_created=now, realm_id=realm_id, name=in_dict['name'], description=in_dict['topic'], stream_id=stream_id, deactivated=in_dict['is_archived'], invite_only=invite_only, ) if invite_only: users = { user_id_mapper.get(key) for key in in_dict['members'] if user_id_mapper.has(key) } # type: Set[int] if user_id_mapper.has(in_dict['owner']): owner = user_id_mapper.get(in_dict['owner']) users.add(owner) if not users: continue subscriber_handler.set_info( stream_id=stream_id, users=users, ) # unmapped fields: # guest_access_url: no Zulip equivalent # created: we just use "now" # participants: no good sample data streams.append(stream) return streams
def process_channels(channels: List[Dict[str, Any]], invite_only: bool = False) -> None: nonlocal stream_id_count nonlocal recipient_id_count nonlocal defaultstream_id nonlocal subscription_id_count for channel in channels: # slack_channel_id = channel['id'] # map Slack's topic and purpose content into Zulip's stream description. # WARN This mapping is lossy since the topic.creator, topic.last_set, # purpose.creator, purpose.last_set fields are not preserved. description = channel["purpose"]["value"] stream_id = stream_id_count recipient_id = recipient_id_count # construct the stream object and append it to zerver_stream stream = build_stream(float(channel["created"]), realm_id, channel["name"], description, stream_id, channel["is_archived"], invite_only) zerver_stream.append(stream) # construct defaultstream object # slack has the default channel 'general' and 'random' # where every user is subscribed default_channels = ['general', 'random'] # Slack specific if channel['name'] in default_channels: defaultstream = build_defaultstream(realm_id, stream_id, defaultstream_id) zerver_defaultstream.append(defaultstream) defaultstream_id += 1 added_channels[stream['name']] = (channel['id'], stream_id) recipient = build_recipient(stream_id, recipient_id, Recipient.STREAM) zerver_recipient.append(recipient) added_recipient[stream['name']] = recipient_id # TODO add recipients for private message and huddles # construct the subscription object and append it to zerver_subscription subscription_id_count = get_subscription(channel['members'], zerver_subscription, recipient_id, added_users, subscription_id_count) # TODO add zerver_subscription which correspond to # huddles type recipient # For huddles: # sub['recipient']=recipient['id'] where recipient['type_id']=added_users[member] stream_id_count += 1 recipient_id_count += 1 logging.info(u"{} -> created".format(channel['name']))
def convert_channel_data( room_id_to_room_map: Dict[str, Dict[str, Any]], team_id_to_team_map: Dict[str, Dict[str, Any]], stream_id_mapper: IdMapper, realm_id: int, ) -> List[ZerverFieldsT]: streams = [] for rc_room_id in room_id_to_room_map: channel_dict = room_id_to_room_map[rc_room_id] date_created = float(channel_dict["ts"].timestamp()) stream_id = stream_id_mapper.get(rc_room_id) invite_only = channel_dict["t"] == "p" stream_name = channel_dict["name"] stream_desc = channel_dict.get("description", "") if channel_dict.get("teamId"): if channel_dict.get("teamMain") is True: # In case you change this, please also change the stream name # used while adding the Rocket.Chat channel mention data to # message_dict in `message_to_dict` inner-function of # `process_messages` function below. stream_name = "[TEAM] " + stream_name else: stream_desc = "[Team {} channel]. {}".format( team_id_to_team_map[channel_dict["teamId"]]["name"], stream_desc) # If the channel is read-only, then only admins and moderators # should be allowed to post in the converted Zulip stream. # For more details: https://zulip.com/help/stream-sending-policy # # See `Stream` model in `zerver/models.py` to know about what each # number represent. stream_post_policy = 4 if channel_dict.get("ro", False) else 1 stream = build_stream( date_created=date_created, realm_id=realm_id, name=stream_name, description=stream_desc, stream_id=stream_id, deactivated=False, invite_only=invite_only, stream_post_policy=stream_post_policy, ) streams.append(stream) return streams
def build_stream_and_defaultstream(timestamp: Any) -> Tuple[List[ZerverFieldsT], List[ZerverFieldsT]]: logging.info('######### IMPORTING STREAM STARTED #########\n') # We have only one stream for gitter export stream_name = 'from gitter' stream_description = "Imported from gitter" stream_id = 0 stream = build_stream(timestamp, realm_id, stream_name, stream_description, stream_id) defaultstream = build_defaultstream(realm_id=realm_id, stream_id=stream_id, defaultstream_id=0) logging.info('######### IMPORTING STREAMS FINISHED #########\n') return [stream], [defaultstream]
def process_channels(channels: List[Dict[str, Any]], invite_only: bool = False) -> None: nonlocal stream_id_count nonlocal recipient_id_count nonlocal defaultstream_id nonlocal subscription_id_count for channel in channels: # map Slack's topic and purpose content into Zulip's stream description. # WARN This mapping is lossy since the topic.creator, topic.last_set, # purpose.creator, purpose.last_set fields are not preserved. description = channel["purpose"]["value"] stream_id = stream_id_count recipient_id = recipient_id_count stream = build_stream( float(channel["created"]), realm_id, channel["name"], description, stream_id, channel["is_archived"], invite_only, ) realm["zerver_stream"].append(stream) slack_default_channels = ["general", "random"] if channel["name"] in slack_default_channels and not stream["deactivated"]: defaultstream = build_defaultstream(realm_id, stream_id, defaultstream_id) realm["zerver_defaultstream"].append(defaultstream) defaultstream_id += 1 added_channels[stream["name"]] = (channel["id"], stream_id) recipient = build_recipient(stream_id, recipient_id, Recipient.STREAM) realm["zerver_recipient"].append(recipient) slack_recipient_name_to_zulip_recipient_id[stream["name"]] = recipient_id subscription_id_count = get_subscription( channel["members"], realm["zerver_subscription"], recipient_id, slack_user_id_to_zulip_user_id, subscription_id_count, ) stream_id_count += 1 recipient_id_count += 1 logging.info("%s -> created", channel["name"])
def convert_channel_data( room_id_to_room_map: Dict[str, Dict[str, Any]], team_id_to_team_map: Dict[str, Dict[str, Any]], stream_id_mapper: IdMapper, realm_id: int, ) -> List[ZerverFieldsT]: streams = [] for rc_room_id in room_id_to_room_map: channel_dict = room_id_to_room_map[rc_room_id] date_created = float(channel_dict["ts"].timestamp()) stream_id = stream_id_mapper.get(rc_room_id) invite_only = channel_dict["t"] == "p" stream_name = get_stream_name(channel_dict) stream_desc = channel_dict.get("description", "") if channel_dict.get("teamId") and not channel_dict.get("teamMain"): stream_desc = "[Team {} channel]. {}".format( team_id_to_team_map[channel_dict["teamId"]]["name"], stream_desc) # If the channel is read-only, then only admins and moderators # should be allowed to post in the converted Zulip stream. # For more details: https://zulip.com/help/stream-sending-policy # # See `Stream` model in `zerver/models.py` to know about what each # number represent. stream_post_policy = 4 if channel_dict.get("ro", False) else 1 stream = build_stream( date_created=date_created, realm_id=realm_id, name=stream_name, description=stream_desc, stream_id=stream_id, deactivated=False, invite_only=invite_only, stream_post_policy=stream_post_policy, ) streams.append(stream) return streams
def process(in_dict: ZerverFieldsT) -> ZerverFieldsT: now = int(timezone_now().timestamp()) out_dict = build_stream( date_created=now, realm_id=realm_id, name=in_dict['name'], description=in_dict['topic'], stream_id=in_dict['id'], deactivated=in_dict['is_archived'], invite_only=invite_only(in_dict['privacy']), ) # unmapped fields: # guest_access_url: no Zulip equivalent # created: we just use "now" # members: no good sample data # owners: no good sample data # participants: no good sample data return out_dict
def process(in_dict: ZerverFieldsT) -> ZerverFieldsT: now = int(timezone_now().timestamp()) stream_id = stream_id_mapper.get(in_dict['id']) out_dict = build_stream( date_created=now, realm_id=realm_id, name=in_dict['name'], description=in_dict['topic'], stream_id=stream_id, deactivated=in_dict['is_archived'], invite_only=invite_only(in_dict['privacy']), ) if not user_id_mapper.has(in_dict['owner']): raise Exception('bad owner') owner = user_id_mapper.get(in_dict['owner']) members = { user_id_mapper.get(key) for key in in_dict['members'] if user_id_mapper.has(key) } subscriber_handler.set_info( stream_id=stream_id, owner=owner, members=members, ) # unmapped fields: # guest_access_url: no Zulip equivalent # created: we just use "now" # members: no good sample data # owners: no good sample data # participants: no good sample data return out_dict
def channels_to_zerver_stream(slack_data_dir: str, realm_id: int, added_users: AddedUsersT, zerver_userprofile: List[ZerverFieldsT]) -> Tuple[List[ZerverFieldsT], List[ZerverFieldsT], AddedChannelsT, List[ZerverFieldsT], List[ZerverFieldsT], AddedRecipientsT]: """ Returns: 1. zerver_defaultstream, which is a list of the default streams 2. zerver_stream, while is a list of all streams 3. added_channels, which is a dictionary to map from channel name to channel id, zulip stream_id 4. zerver_subscription, which is a list of the subscriptions 5. zerver_recipient, which is a list of the recipients 6. added_recipient, which is a dictionary to map from channel name to zulip recipient_id """ logging.info('######### IMPORTING CHANNELS STARTED #########\n') channels = get_data_file(slack_data_dir + '/channels.json') added_channels = {} added_recipient = {} zerver_stream = [] zerver_subscription = [] # type: List[ZerverFieldsT] zerver_recipient = [] zerver_defaultstream = [] stream_id_count = subscription_id_count = recipient_id_count = defaultstream_id = 0 for channel in channels: # slack_channel_id = channel['id'] # map Slack's topic and purpose content into Zulip's stream description. # WARN This mapping is lossy since the topic.creator, topic.last_set, # purpose.creator, purpose.last_set fields are not preserved. description = channel["purpose"]["value"] stream_id = stream_id_count recipient_id = recipient_id_count # construct the stream object and append it to zerver_stream stream = build_stream(float(channel["created"]), realm_id, channel["name"], description, stream_id, channel["is_archived"]) zerver_stream.append(stream) # construct defaultstream object # slack has the default channel 'general' and 'random' # where every user is subscribed default_channels = ['general', 'random'] # Slack specific if channel['name'] in default_channels: defaultstream = build_defaultstream(realm_id, stream_id, defaultstream_id) zerver_defaultstream.append(defaultstream) defaultstream_id += 1 added_channels[stream['name']] = (channel['id'], stream_id) recipient = build_recipient(stream_id, recipient_id, Recipient.STREAM) zerver_recipient.append(recipient) added_recipient[stream['name']] = recipient_id # TODO add recipients for private message and huddles # construct the subscription object and append it to zerver_subscription subscription_id_count = get_subscription(channel['members'], zerver_subscription, recipient_id, added_users, subscription_id_count) # TODO add zerver_subscription which correspond to # huddles type recipient # For huddles: # sub['recipient']=recipient['id'] where recipient['type_id']=added_users[member] stream_id_count += 1 recipient_id_count += 1 logging.info(u"{} -> created".format(channel['name'])) # TODO map Slack's pins to Zulip's stars # There is the security model that Slack's pins are known to the team owner # as evident from where it is stored at (channels) # "pins": [ # { # "id": "1444755381.000003", # "type": "C", # "user": "******", # "owner": "U061A5N1G", # "created": "1444755463" # } # ], for user in zerver_userprofile: # this maps the recipients and subscriptions # related to private messages recipient = build_recipient(user['id'], recipient_id_count, Recipient.PERSONAL) sub = build_subscription(recipient_id_count, user['id'], subscription_id_count) zerver_recipient.append(recipient) zerver_subscription.append(sub) subscription_id_count += 1 recipient_id_count += 1 logging.info('######### IMPORTING STREAMS FINISHED #########\n') return zerver_defaultstream, zerver_stream, added_channels, zerver_subscription, \ zerver_recipient, added_recipient
def convert_room_data(raw_data: List[ZerverFieldsT], subscriber_handler: SubscriberHandler, stream_id_mapper: IdMapper, user_id_mapper: IdMapper, realm_id: int, api_token: Optional[str] = None) -> List[ZerverFieldsT]: flat_data = [d['Room'] for d in raw_data] def get_invite_only(v: str) -> bool: if v == 'public': return False elif v == 'private': return True else: raise Exception('unexpected value') streams = [] for in_dict in flat_data: now = int(timezone_now().timestamp()) stream_id = stream_id_mapper.get(in_dict['id']) invite_only = get_invite_only(in_dict['privacy']) stream = build_stream( date_created=now, realm_id=realm_id, name=in_dict['name'], description=in_dict['topic'], stream_id=stream_id, deactivated=in_dict['is_archived'], invite_only=invite_only, ) if invite_only: users: Set[int] = { user_id_mapper.get(key) for key in in_dict['members'] if user_id_mapper.has(key) } if user_id_mapper.has(in_dict['owner']): owner = user_id_mapper.get(in_dict['owner']) users.add(owner) else: users = set() if api_token is not None: hc = hypchat.HypChat(api_token) room_data = hc.fromurl('{}/v2/room/{}/member'.format( hc.endpoint, in_dict['id'])) for item in room_data['items']: hipchat_user_id = item['id'] zulip_user_id = user_id_mapper.get(hipchat_user_id) users.add(zulip_user_id) if users: subscriber_handler.set_info( stream_id=stream_id, users=users, ) # unmapped fields: # guest_access_url: no Zulip equivalent # created: we just use "now" # participants: no good sample data streams.append(stream) return streams
def convert_channel_data( channel_data: List[ZerverFieldsT], user_data_map: Dict[str, Dict[str, Any]], subscriber_handler: SubscriberHandler, stream_id_mapper: IdMapper, user_id_mapper: IdMapper, realm_id: int, team_name: str, ) -> List[ZerverFieldsT]: channel_data_list = [d for d in channel_data if d["team"] == team_name] channel_members_map: Dict[str, List[str]] = {} channel_admins_map: Dict[str, List[str]] = {} def initialize_stream_membership_dicts() -> None: for channel in channel_data: channel_name = channel["name"] channel_members_map[channel_name] = [] channel_admins_map[channel_name] = [] for username in user_data_map: user_dict = user_data_map[username] teams = user_dict["teams"] if user_dict["teams"] is None: continue for team in teams: if team["name"] != team_name: continue for channel in team["channels"]: channel_roles = channel["roles"] channel_name = channel["name"] if "channel_admin" in channel_roles: channel_admins_map[channel_name].append(username) elif "channel_user" in channel_roles: channel_members_map[channel_name].append(username) def get_invite_only_value_from_channel_type(channel_type: str) -> bool: # Channel can have two types in Mattermost # "O" for a public channel. # "P" for a private channel. if channel_type == "O": return False elif channel_type == "P": return True else: # nocoverage raise Exception("unexpected value") streams = [] initialize_stream_membership_dicts() for channel_dict in channel_data_list: now = int(timezone_now().timestamp()) stream_id = stream_id_mapper.get(channel_dict["name"]) stream_name = channel_dict["name"] invite_only = get_invite_only_value_from_channel_type(channel_dict["type"]) stream = build_stream( date_created=now, realm_id=realm_id, name=channel_dict["display_name"], # Purpose describes how the channel should be used. It is similar to # stream description and is shown in channel list to help others decide # whether to join. # Header text always appears right next to channel name in channel header. # Can be used for advertising the purpose of stream, making announcements as # well as including frequently used links. So probably not a bad idea to use # this as description if the channel purpose is empty. description=channel_dict["purpose"] or channel_dict["header"], stream_id=stream_id, # Mattermost export don't include data of archived(~ deactivated) channels. deactivated=False, invite_only=invite_only, ) channel_users = set() for username in channel_admins_map[stream_name]: channel_users.add(user_id_mapper.get(username)) for username in channel_members_map[stream_name]: channel_users.add(user_id_mapper.get(username)) subscriber_handler.set_info( users=channel_users, stream_id=stream_id, ) streams.append(stream) return streams