def convert_user_data(user_handler: UserHandler, user_id_mapper: IdMapper, raw_data: List[ZerverFieldsT], realm_id: int) -> None: flat_data = [d['User'] for d in raw_data] def process(in_dict: ZerverFieldsT) -> ZerverFieldsT: delivery_email = in_dict['email'] email = in_dict['email'] full_name = in_dict['name'] id = user_id_mapper.get(in_dict['id']) is_realm_admin = in_dict['account_type'] == 'admin' is_guest = in_dict['account_type'] == 'guest' is_mirror_dummy = False short_name = in_dict['mention_name'] timezone = in_dict['timezone'] date_joined = int(timezone_now().timestamp()) is_active = not in_dict['is_deleted'] if not email: if is_guest: # Hipchat guest users don't have emails, so # we just fake them. email = 'guest-{id}@example.com'.format(id=id) delivery_email = email else: # Hipchat sometimes doesn't export an email for deactivated users. assert not is_active email = delivery_email = "deactivated-{id}@example.com".format( id=id) # unmapped fields: # title - Developer, Project Manager, etc. # rooms - no good sample data # created - we just use "now" # roles - we just use account_type if in_dict.get('avatar'): avatar_source = 'U' else: avatar_source = 'G' return build_user_profile( avatar_source=avatar_source, date_joined=date_joined, delivery_email=delivery_email, email=email, full_name=full_name, id=id, is_active=is_active, is_realm_admin=is_realm_admin, is_guest=is_guest, is_mirror_dummy=is_mirror_dummy, realm_id=realm_id, short_name=short_name, timezone=timezone, ) for raw_item in flat_data: user = process(raw_item) user_handler.add_user(user)
def get_hipchat_sender_id(realm_id: int, message_dict: Dict[str, Any], user_handler: UserHandler) -> int: ''' The HipChat export is inconsistent in how it renders senders, and sometimes we don't even get an id. ''' if isinstance(message_dict['sender'], str): # Some Hipchat instances just give us a person's # name in the sender field for NotificationMessage. # We turn them into a mirror user. mirror_user = user_handler.get_mirror_user( realm_id=realm_id, name=message_dict['sender'], ) sender_id = mirror_user['id'] return sender_id sender_id = message_dict['sender']['id'] if sender_id == 0: mirror_user = user_handler.get_mirror_user( realm_id=realm_id, name=message_dict['sender']['name'], ) sender_id = mirror_user['id'] return sender_id # HAPPY PATH: Hipchat just gave us an ordinary # sender_id. return sender_id
def test_sender_ids(self) -> None: realm_id = 5 user_handler = UserHandler() user_id_mapper = IdMapper() user_id_mapper.has = lambda key: True # type: ignore[assignment] # it's just a stub # Simulate a "normal" user first. user_with_id = dict(id=1, # other fields don't matter here ) user_handler.add_user(user=user_with_id) normal_message: Dict[str, Any] = dict(sender=dict(id=1, ), ) sender_id = get_hipchat_sender_id( realm_id=realm_id, slim_mode=False, message_dict=normal_message, user_id_mapper=user_id_mapper, user_handler=user_handler, ) self.assertEqual(sender_id, 1) bot_message = dict(sender='fred_bot', ) # Every message from fred_bot should # return the same sender_id. fred_bot_sender_id = 2 for i in range(3): sender_id = get_hipchat_sender_id( realm_id=realm_id, slim_mode=False, message_dict=bot_message, user_id_mapper=user_id_mapper, user_handler=user_handler, ) self.assertEqual(sender_id, fred_bot_sender_id) id_zero_message = dict(sender=dict( id=0, name='hal_bot', ), ) hal_bot_sender_id = 3 for i in range(3): sender_id = get_hipchat_sender_id( realm_id=realm_id, slim_mode=False, message_dict=id_zero_message, user_id_mapper=user_id_mapper, user_handler=user_handler, ) self.assertEqual(sender_id, hal_bot_sender_id)
def test_sender_ids(self) -> None: realm_id = 5 user_handler = UserHandler() # Simulate a "normal" user first. user_with_id = dict( id=1, # other fields don't matter here ) user_handler.add_user(user=user_with_id) normal_message = dict( sender=dict( id=1, ) ) # type: Dict[str, Any] sender_id = get_hipchat_sender_id( realm_id=realm_id, message_dict=normal_message, user_handler=user_handler, ) self.assertEqual(sender_id, 1) bot_message = dict( sender='fred_bot', ) # Every message from fred_bot should # return the same sender_id. fred_bot_sender_id = 2 for i in range(3): sender_id = get_hipchat_sender_id( realm_id=realm_id, message_dict=bot_message, user_handler=user_handler, ) self.assertEqual(sender_id, fred_bot_sender_id) id_zero_message = dict( sender=dict( id=0, name='hal_bot', ), ) hal_bot_sender_id = 3 for i in range(3): sender_id = get_hipchat_sender_id( realm_id=realm_id, message_dict=id_zero_message, user_handler=user_handler, ) self.assertEqual(sender_id, hal_bot_sender_id)
def get_hipchat_sender_id(realm_id: int, slim_mode: bool, message_dict: Dict[str, Any], user_id_mapper: IdMapper, user_handler: UserHandler) -> Optional[int]: ''' The HipChat export is inconsistent in how it renders senders, and sometimes we don't even get an id. ''' if isinstance(message_dict['sender'], str): if slim_mode: return None # Some Hipchat instances just give us a person's # name in the sender field for NotificationMessage. # We turn them into a mirror user. mirror_user = user_handler.get_mirror_user( realm_id=realm_id, name=message_dict['sender'], ) sender_id = mirror_user['id'] return sender_id raw_sender_id = message_dict['sender']['id'] if raw_sender_id == 0: if slim_mode: return None mirror_user = user_handler.get_mirror_user( realm_id=realm_id, name=message_dict['sender']['name'] ) sender_id = mirror_user['id'] return sender_id if not user_id_mapper.has(raw_sender_id): if slim_mode: return None mirror_user = user_handler.get_mirror_user( realm_id=realm_id, name=message_dict['sender']['id'] ) sender_id = mirror_user['id'] return sender_id # HAPPY PATH: Hipchat just gave us an ordinary # sender_id. sender_id = user_id_mapper.get(raw_sender_id) return sender_id
def do_convert_data(input_tar_file: str, output_dir: str, masking_content: bool, api_token: Optional[str] = None, slim_mode: bool = False) -> None: input_data_dir = untar_input_file(input_tar_file) attachment_handler = AttachmentHandler() user_handler = UserHandler() subscriber_handler = SubscriberHandler() user_id_mapper = IdMapper() stream_id_mapper = IdMapper() realm_id = 0 realm = make_realm(realm_id=realm_id) # users.json -> UserProfile raw_user_data = read_user_data(data_dir=input_data_dir) convert_user_data( user_handler=user_handler, slim_mode=slim_mode, user_id_mapper=user_id_mapper, raw_data=raw_user_data, realm_id=realm_id, ) normal_users = user_handler.get_normal_users() # Don't write zerver_userprofile here, because we # may add more users later. # streams.json -> Stream raw_stream_data = read_room_data(data_dir=input_data_dir) zerver_stream = convert_room_data( raw_data=raw_stream_data, subscriber_handler=subscriber_handler, stream_id_mapper=stream_id_mapper, user_id_mapper=user_id_mapper, realm_id=realm_id, api_token=api_token, ) realm['zerver_stream'] = zerver_stream zerver_recipient = build_recipients( zerver_userprofile=normal_users, zerver_stream=zerver_stream, ) realm['zerver_recipient'] = zerver_recipient if api_token is None: if slim_mode: public_stream_subscriptions: List[ZerverFieldsT] = [] else: public_stream_subscriptions = build_public_stream_subscriptions( zerver_userprofile=normal_users, zerver_recipient=zerver_recipient, zerver_stream=zerver_stream, ) private_stream_subscriptions = build_stream_subscriptions( get_users=subscriber_handler.get_users, zerver_recipient=zerver_recipient, zerver_stream=[ stream_dict for stream_dict in zerver_stream if stream_dict['invite_only'] ], ) stream_subscriptions = public_stream_subscriptions + private_stream_subscriptions else: stream_subscriptions = build_stream_subscriptions( get_users=subscriber_handler.get_users, zerver_recipient=zerver_recipient, zerver_stream=zerver_stream, ) personal_subscriptions = build_personal_subscriptions( zerver_recipient=zerver_recipient, ) zerver_subscription = personal_subscriptions + stream_subscriptions realm['zerver_subscription'] = zerver_subscription zerver_realmemoji = write_emoticon_data( realm_id=realm_id, data_dir=input_data_dir, output_dir=output_dir, ) realm['zerver_realmemoji'] = zerver_realmemoji subscriber_map = make_subscriber_map( zerver_subscription=zerver_subscription, ) logging.info('Start importing message data') for message_key in [ 'UserMessage', 'NotificationMessage', 'PrivateUserMessage' ]: write_message_data( realm_id=realm_id, slim_mode=slim_mode, message_key=message_key, zerver_recipient=zerver_recipient, subscriber_map=subscriber_map, data_dir=input_data_dir, output_dir=output_dir, masking_content=masking_content, stream_id_mapper=stream_id_mapper, user_id_mapper=user_id_mapper, user_handler=user_handler, attachment_handler=attachment_handler, ) # Order is important here...don't write users until # we process everything else, since we may introduce # mirror users when processing messages. realm['zerver_userprofile'] = user_handler.get_all_users() realm['sort_by_date'] = True create_converted_data_files(realm, output_dir, '/realm.json') logging.info('Start importing avatar data') write_avatar_data( raw_user_data=raw_user_data, output_dir=output_dir, user_id_mapper=user_id_mapper, realm_id=realm_id, ) attachment_handler.write_info( output_dir=output_dir, realm_id=realm_id, ) logging.info('Start making tarball') subprocess.check_call( ["tar", "-czf", output_dir + '.tar.gz', output_dir, '-P']) logging.info('Done making tarball')
def test_sender_ids(self) -> None: realm_id = 5 user_handler = UserHandler() user_id_mapper = IdMapper() user_id_mapper.has = lambda key: True # type: ignore # it's just a stub # Simulate a "normal" user first. user_with_id = dict( id=1, # other fields don't matter here ) user_handler.add_user(user=user_with_id) normal_message = dict( sender=dict( id=1, ) ) # type: Dict[str, Any] sender_id = get_hipchat_sender_id( realm_id=realm_id, slim_mode=False, message_dict=normal_message, user_id_mapper=user_id_mapper, user_handler=user_handler, ) self.assertEqual(sender_id, 1) bot_message = dict( sender='fred_bot', ) # Every message from fred_bot should # return the same sender_id. fred_bot_sender_id = 2 for i in range(3): sender_id = get_hipchat_sender_id( realm_id=realm_id, slim_mode=False, message_dict=bot_message, user_id_mapper=user_id_mapper, user_handler=user_handler, ) self.assertEqual(sender_id, fred_bot_sender_id) id_zero_message = dict( sender=dict( id=0, name='hal_bot', ), ) hal_bot_sender_id = 3 for i in range(3): sender_id = get_hipchat_sender_id( realm_id=realm_id, slim_mode=False, message_dict=id_zero_message, user_id_mapper=user_id_mapper, user_handler=user_handler, ) self.assertEqual(sender_id, hal_bot_sender_id)
def do_convert_data(input_tar_file: str, output_dir: str) -> None: input_data_dir = untar_input_file(input_tar_file) attachment_handler = AttachmentHandler() user_handler = UserHandler() realm_id = 0 realm = make_realm(realm_id=realm_id) # users.json -> UserProfile raw_user_data = read_user_data(data_dir=input_data_dir) convert_user_data( user_handler=user_handler, raw_data=raw_user_data, realm_id=realm_id, ) normal_users = user_handler.get_normal_users() # Don't write zerver_userprofile here, because we # may add more users later. # streams.json -> Stream raw_stream_data = read_room_data(data_dir=input_data_dir) zerver_stream = convert_room_data( raw_data=raw_stream_data, realm_id=realm_id, ) realm['zerver_stream'] = zerver_stream zerver_recipient = build_recipients( zerver_userprofile=normal_users, zerver_stream=zerver_stream, ) realm['zerver_recipient'] = zerver_recipient zerver_subscription = build_subscriptions( zerver_userprofile=normal_users, zerver_recipient=zerver_recipient, zerver_stream=zerver_stream, ) realm['zerver_subscription'] = zerver_subscription zerver_realmemoji = write_emoticon_data( realm_id=realm_id, data_dir=input_data_dir, output_dir=output_dir, ) realm['zerver_realmemoji'] = zerver_realmemoji logging.info('Start importing message data') for message_key in ['UserMessage', 'PrivateUserMessage']: write_message_data( realm_id=realm_id, message_key=message_key, zerver_recipient=zerver_recipient, zerver_subscription=zerver_subscription, data_dir=input_data_dir, output_dir=output_dir, user_handler=user_handler, attachment_handler=attachment_handler, ) # Order is important here...don't write users until # we process everything else, since we may introduce # mirror users when processing messages. realm['zerver_userprofile'] = user_handler.get_all_users() create_converted_data_files(realm, output_dir, '/realm.json') logging.info('Start importing avatar data') write_avatar_data( raw_user_data=raw_user_data, output_dir=output_dir, realm_id=realm_id, ) attachment_handler.write_info( output_dir=output_dir, realm_id=realm_id, ) logging.info('Start making tarball') subprocess.check_call(["tar", "-czf", output_dir + '.tar.gz', output_dir, '-P']) logging.info('Done making tarball')