def convert_user_data(user_handler: UserHandler, user_id_mapper: IdMapper,
raw_data: List[ZerverFieldsT], realm_id: int) -> None:
flat_data = [d['User'] for d in raw_data]
def process(in_dict: ZerverFieldsT) -> ZerverFieldsT:
delivery_email = in_dict['email']
email = in_dict['email']
full_name = in_dict['name']
id = user_id_mapper.get(in_dict['id'])
is_realm_admin = in_dict['account_type'] == 'admin'
is_guest = in_dict['account_type'] == 'guest'
is_mirror_dummy = False
short_name = in_dict['mention_name']
timezone = in_dict['timezone']
date_joined = int(timezone_now().timestamp())
is_active = not in_dict['is_deleted']
if not email:
if is_guest:
# Hipchat guest users don't have emails, so
# we just fake them.
email = 'guest-{id}@example.com'.format(id=id)
delivery_email = email
else:
# Hipchat sometimes doesn't export an email for deactivated users.
assert not is_active
email = delivery_email = "deactivated-{id}@example.com".format(
id=id)
# unmapped fields:
# title - Developer, Project Manager, etc.
# rooms - no good sample data
# created - we just use "now"
# roles - we just use account_type
if in_dict.get('avatar'):
avatar_source = 'U'
else:
avatar_source = 'G'
return build_user_profile(
avatar_source=avatar_source,
date_joined=date_joined,
delivery_email=delivery_email,
email=email,
full_name=full_name,
id=id,
is_active=is_active,
is_realm_admin=is_realm_admin,
is_guest=is_guest,
is_mirror_dummy=is_mirror_dummy,
realm_id=realm_id,
short_name=short_name,
timezone=timezone,
)
for raw_item in flat_data:
user = process(raw_item)
user_handler.add_user(user)
def get_hipchat_sender_id(realm_id: int, message_dict: Dict[str, Any],
user_handler: UserHandler) -> int:
'''
The HipChat export is inconsistent in how it renders
senders, and sometimes we don't even get an id.
'''
if isinstance(message_dict['sender'], str):
# Some Hipchat instances just give us a person's
# name in the sender field for NotificationMessage.
# We turn them into a mirror user.
mirror_user = user_handler.get_mirror_user(
realm_id=realm_id,
name=message_dict['sender'],
)
sender_id = mirror_user['id']
return sender_id
sender_id = message_dict['sender']['id']
if sender_id == 0:
mirror_user = user_handler.get_mirror_user(
realm_id=realm_id,
name=message_dict['sender']['name'],
)
sender_id = mirror_user['id']
return sender_id
# HAPPY PATH: Hipchat just gave us an ordinary
# sender_id.
return sender_id
def test_sender_ids(self) -> None:
realm_id = 5
user_handler = UserHandler()
user_id_mapper = IdMapper()
user_id_mapper.has = lambda key: True # type: ignore[assignment] # it's just a stub
# Simulate a "normal" user first.
user_with_id = dict(id=1,
# other fields don't matter here
)
user_handler.add_user(user=user_with_id)
normal_message: Dict[str, Any] = dict(sender=dict(id=1, ), )
sender_id = get_hipchat_sender_id(
realm_id=realm_id,
slim_mode=False,
message_dict=normal_message,
user_id_mapper=user_id_mapper,
user_handler=user_handler,
)
self.assertEqual(sender_id, 1)
bot_message = dict(sender='fred_bot', )
# Every message from fred_bot should
# return the same sender_id.
fred_bot_sender_id = 2
for i in range(3):
sender_id = get_hipchat_sender_id(
realm_id=realm_id,
slim_mode=False,
message_dict=bot_message,
user_id_mapper=user_id_mapper,
user_handler=user_handler,
)
self.assertEqual(sender_id, fred_bot_sender_id)
id_zero_message = dict(sender=dict(
id=0,
name='hal_bot',
), )
hal_bot_sender_id = 3
for i in range(3):
sender_id = get_hipchat_sender_id(
realm_id=realm_id,
slim_mode=False,
message_dict=id_zero_message,
user_id_mapper=user_id_mapper,
user_handler=user_handler,
)
self.assertEqual(sender_id, hal_bot_sender_id)
def test_sender_ids(self) -> None:
realm_id = 5
user_handler = UserHandler()
# Simulate a "normal" user first.
user_with_id = dict(
id=1,
# other fields don't matter here
)
user_handler.add_user(user=user_with_id)
normal_message = dict(
sender=dict(
id=1,
)
) # type: Dict[str, Any]
sender_id = get_hipchat_sender_id(
realm_id=realm_id,
message_dict=normal_message,
user_handler=user_handler,
)
self.assertEqual(sender_id, 1)
bot_message = dict(
sender='fred_bot',
)
# Every message from fred_bot should
# return the same sender_id.
fred_bot_sender_id = 2
for i in range(3):
sender_id = get_hipchat_sender_id(
realm_id=realm_id,
message_dict=bot_message,
user_handler=user_handler,
)
self.assertEqual(sender_id, fred_bot_sender_id)
id_zero_message = dict(
sender=dict(
id=0,
name='hal_bot',
),
)
hal_bot_sender_id = 3
for i in range(3):
sender_id = get_hipchat_sender_id(
realm_id=realm_id,
message_dict=id_zero_message,
user_handler=user_handler,
)
self.assertEqual(sender_id, hal_bot_sender_id)
def get_hipchat_sender_id(realm_id: int,
slim_mode: bool,
message_dict: Dict[str, Any],
user_id_mapper: IdMapper,
user_handler: UserHandler) -> Optional[int]:
'''
The HipChat export is inconsistent in how it renders
senders, and sometimes we don't even get an id.
'''
if isinstance(message_dict['sender'], str):
if slim_mode:
return None
# Some Hipchat instances just give us a person's
# name in the sender field for NotificationMessage.
# We turn them into a mirror user.
mirror_user = user_handler.get_mirror_user(
realm_id=realm_id,
name=message_dict['sender'],
)
sender_id = mirror_user['id']
return sender_id
raw_sender_id = message_dict['sender']['id']
if raw_sender_id == 0:
if slim_mode:
return None
mirror_user = user_handler.get_mirror_user(
realm_id=realm_id,
name=message_dict['sender']['name']
)
sender_id = mirror_user['id']
return sender_id
if not user_id_mapper.has(raw_sender_id):
if slim_mode:
return None
mirror_user = user_handler.get_mirror_user(
realm_id=realm_id,
name=message_dict['sender']['id']
)
sender_id = mirror_user['id']
return sender_id
# HAPPY PATH: Hipchat just gave us an ordinary
# sender_id.
sender_id = user_id_mapper.get(raw_sender_id)
return sender_id
def do_convert_data(input_tar_file: str,
output_dir: str,
masking_content: bool,
api_token: Optional[str] = None,
slim_mode: bool = False) -> None:
input_data_dir = untar_input_file(input_tar_file)
attachment_handler = AttachmentHandler()
user_handler = UserHandler()
subscriber_handler = SubscriberHandler()
user_id_mapper = IdMapper()
stream_id_mapper = IdMapper()
realm_id = 0
realm = make_realm(realm_id=realm_id)
# users.json -> UserProfile
raw_user_data = read_user_data(data_dir=input_data_dir)
convert_user_data(
user_handler=user_handler,
slim_mode=slim_mode,
user_id_mapper=user_id_mapper,
raw_data=raw_user_data,
realm_id=realm_id,
)
normal_users = user_handler.get_normal_users()
# Don't write zerver_userprofile here, because we
# may add more users later.
# streams.json -> Stream
raw_stream_data = read_room_data(data_dir=input_data_dir)
zerver_stream = convert_room_data(
raw_data=raw_stream_data,
subscriber_handler=subscriber_handler,
stream_id_mapper=stream_id_mapper,
user_id_mapper=user_id_mapper,
realm_id=realm_id,
api_token=api_token,
)
realm['zerver_stream'] = zerver_stream
zerver_recipient = build_recipients(
zerver_userprofile=normal_users,
zerver_stream=zerver_stream,
)
realm['zerver_recipient'] = zerver_recipient
if api_token is None:
if slim_mode:
public_stream_subscriptions: List[ZerverFieldsT] = []
else:
public_stream_subscriptions = build_public_stream_subscriptions(
zerver_userprofile=normal_users,
zerver_recipient=zerver_recipient,
zerver_stream=zerver_stream,
)
private_stream_subscriptions = build_stream_subscriptions(
get_users=subscriber_handler.get_users,
zerver_recipient=zerver_recipient,
zerver_stream=[
stream_dict for stream_dict in zerver_stream
if stream_dict['invite_only']
],
)
stream_subscriptions = public_stream_subscriptions + private_stream_subscriptions
else:
stream_subscriptions = build_stream_subscriptions(
get_users=subscriber_handler.get_users,
zerver_recipient=zerver_recipient,
zerver_stream=zerver_stream,
)
personal_subscriptions = build_personal_subscriptions(
zerver_recipient=zerver_recipient, )
zerver_subscription = personal_subscriptions + stream_subscriptions
realm['zerver_subscription'] = zerver_subscription
zerver_realmemoji = write_emoticon_data(
realm_id=realm_id,
data_dir=input_data_dir,
output_dir=output_dir,
)
realm['zerver_realmemoji'] = zerver_realmemoji
subscriber_map = make_subscriber_map(
zerver_subscription=zerver_subscription, )
logging.info('Start importing message data')
for message_key in [
'UserMessage', 'NotificationMessage', 'PrivateUserMessage'
]:
write_message_data(
realm_id=realm_id,
slim_mode=slim_mode,
message_key=message_key,
zerver_recipient=zerver_recipient,
subscriber_map=subscriber_map,
data_dir=input_data_dir,
output_dir=output_dir,
masking_content=masking_content,
stream_id_mapper=stream_id_mapper,
user_id_mapper=user_id_mapper,
user_handler=user_handler,
attachment_handler=attachment_handler,
)
# Order is important here...don't write users until
# we process everything else, since we may introduce
# mirror users when processing messages.
realm['zerver_userprofile'] = user_handler.get_all_users()
realm['sort_by_date'] = True
create_converted_data_files(realm, output_dir, '/realm.json')
logging.info('Start importing avatar data')
write_avatar_data(
raw_user_data=raw_user_data,
output_dir=output_dir,
user_id_mapper=user_id_mapper,
realm_id=realm_id,
)
attachment_handler.write_info(
output_dir=output_dir,
realm_id=realm_id,
)
logging.info('Start making tarball')
subprocess.check_call(
["tar", "-czf", output_dir + '.tar.gz', output_dir, '-P'])
logging.info('Done making tarball')
def test_sender_ids(self) -> None:
realm_id = 5
user_handler = UserHandler()
user_id_mapper = IdMapper()
user_id_mapper.has = lambda key: True # type: ignore # it's just a stub
# Simulate a "normal" user first.
user_with_id = dict(
id=1,
# other fields don't matter here
)
user_handler.add_user(user=user_with_id)
normal_message = dict(
sender=dict(
id=1,
)
) # type: Dict[str, Any]
sender_id = get_hipchat_sender_id(
realm_id=realm_id,
slim_mode=False,
message_dict=normal_message,
user_id_mapper=user_id_mapper,
user_handler=user_handler,
)
self.assertEqual(sender_id, 1)
bot_message = dict(
sender='fred_bot',
)
# Every message from fred_bot should
# return the same sender_id.
fred_bot_sender_id = 2
for i in range(3):
sender_id = get_hipchat_sender_id(
realm_id=realm_id,
slim_mode=False,
message_dict=bot_message,
user_id_mapper=user_id_mapper,
user_handler=user_handler,
)
self.assertEqual(sender_id, fred_bot_sender_id)
id_zero_message = dict(
sender=dict(
id=0,
name='hal_bot',
),
)
hal_bot_sender_id = 3
for i in range(3):
sender_id = get_hipchat_sender_id(
realm_id=realm_id,
slim_mode=False,
message_dict=id_zero_message,
user_id_mapper=user_id_mapper,
user_handler=user_handler,
)
self.assertEqual(sender_id, hal_bot_sender_id)
def do_convert_data(input_tar_file: str, output_dir: str) -> None:
input_data_dir = untar_input_file(input_tar_file)
attachment_handler = AttachmentHandler()
user_handler = UserHandler()
realm_id = 0
realm = make_realm(realm_id=realm_id)
# users.json -> UserProfile
raw_user_data = read_user_data(data_dir=input_data_dir)
convert_user_data(
user_handler=user_handler,
raw_data=raw_user_data,
realm_id=realm_id,
)
normal_users = user_handler.get_normal_users()
# Don't write zerver_userprofile here, because we
# may add more users later.
# streams.json -> Stream
raw_stream_data = read_room_data(data_dir=input_data_dir)
zerver_stream = convert_room_data(
raw_data=raw_stream_data,
realm_id=realm_id,
)
realm['zerver_stream'] = zerver_stream
zerver_recipient = build_recipients(
zerver_userprofile=normal_users,
zerver_stream=zerver_stream,
)
realm['zerver_recipient'] = zerver_recipient
zerver_subscription = build_subscriptions(
zerver_userprofile=normal_users,
zerver_recipient=zerver_recipient,
zerver_stream=zerver_stream,
)
realm['zerver_subscription'] = zerver_subscription
zerver_realmemoji = write_emoticon_data(
realm_id=realm_id,
data_dir=input_data_dir,
output_dir=output_dir,
)
realm['zerver_realmemoji'] = zerver_realmemoji
logging.info('Start importing message data')
for message_key in ['UserMessage',
'PrivateUserMessage']:
write_message_data(
realm_id=realm_id,
message_key=message_key,
zerver_recipient=zerver_recipient,
zerver_subscription=zerver_subscription,
data_dir=input_data_dir,
output_dir=output_dir,
user_handler=user_handler,
attachment_handler=attachment_handler,
)
# Order is important here...don't write users until
# we process everything else, since we may introduce
# mirror users when processing messages.
realm['zerver_userprofile'] = user_handler.get_all_users()
create_converted_data_files(realm, output_dir, '/realm.json')
logging.info('Start importing avatar data')
write_avatar_data(
raw_user_data=raw_user_data,
output_dir=output_dir,
realm_id=realm_id,
)
attachment_handler.write_info(
output_dir=output_dir,
realm_id=realm_id,
)
logging.info('Start making tarball')
subprocess.check_call(["tar", "-czf", output_dir + '.tar.gz', output_dir, '-P'])
logging.info('Done making tarball')
