def test_gitter_import_to_existing_database(self, mock_process_avatars: mock.Mock) -> None:
output_dir = self._make_output_dir()
gitter_file = os.path.join(os.path.dirname(__file__), 'fixtures/gitter_data.json')
do_convert_data(gitter_file, output_dir)
do_import_realm(output_dir, 'test-gitter-import')
realm = get_realm('test-gitter-import')
# test rendered_messages
realm_users = UserProfile.objects.filter(realm=realm)
messages = Message.objects.filter(sender__in=realm_users)
for message in messages:
self.assertIsNotNone(message.rendered_content, None)
def test_gitter_import_to_existing_database(
self, mock_process_avatars: mock.Mock) -> None:
output_dir = self._make_output_dir()
gitter_file = os.path.join(os.path.dirname(__file__),
'fixtures/gitter_data.json')
do_convert_data(gitter_file, output_dir)
do_import_realm(output_dir, 'test-gitter-import')
realm = get_realm('test-gitter-import')
# test rendered_messages
realm_users = UserProfile.objects.filter(realm=realm)
messages = Message.objects.filter(sender__in=realm_users)
for message in messages:
self.assertIsNotNone(message.rendered_content, None)
def handle(self, *args: Any, **options: Any) -> None:
output_dir = options["output_dir"]
if output_dir is None:
output_dir = tempfile.mkdtemp(prefix="converted-gitter-data-")
else:
output_dir = os.path.realpath(output_dir)
num_threads = int(options['threads'])
if num_threads < 1:
raise CommandError('You must have at least one thread.')
for path in options['gitter_data']:
if not os.path.exists(path):
raise CommandError(f"Gitter data file not found: '{path}'")
# TODO add json check
print("Converting data ...")
do_convert_data(path, output_dir, num_threads)
def handle(self, *args: Any, **options: Any) -> None:
output_dir = options["output_dir"]
if output_dir is None:
output_dir = tempfile.mkdtemp(prefix="/tmp/converted-gitter-data-")
else:
output_dir = os.path.realpath(output_dir)
num_threads = int(options['threads'])
if num_threads < 1:
raise CommandError('You must have at least one thread.')
for path in options['gitter_data']:
if not os.path.exists(path):
print("Gitter data file not found: '%s'" % (path,))
exit(1)
# TODO add json check
print("Converting Data ...")
do_convert_data(path, output_dir, num_threads)
def test_gitter_import_to_existing_database(
self, mock_process_avatars: mock.Mock) -> None:
output_dir = self.make_import_output_dir("gitter")
gitter_file = os.path.join(os.path.dirname(__file__),
"fixtures/gitter_data.json")
with self.assertLogs(level="INFO"):
do_convert_data(gitter_file, output_dir)
with self.assertLogs(level="INFO"):
do_import_realm(output_dir, "test-gitter-import")
realm = get_realm("test-gitter-import")
# test rendered_messages
realm_users = UserProfile.objects.filter(realm=realm)
messages = Message.objects.filter(sender__in=realm_users)
for message in messages:
self.assertIsNotNone(message.rendered_content, None)
def test_gitter_import_data_conversion(self, mock_process_avatars: mock.Mock) -> None:
output_dir = self._make_output_dir()
gitter_file = os.path.join(os.path.dirname(__file__), 'fixtures/gitter_data.json')
do_convert_data(gitter_file, output_dir)
def read_file(output_file: str) -> Any:
full_path = os.path.join(output_dir, output_file)
with open(full_path) as f:
return ujson.load(f)
def get_set(data: List[Dict[str, Any]], field: str) -> Set[str]:
values = set(r[field] for r in data)
return values
self.assertEqual(os.path.exists(os.path.join(output_dir, 'avatars')), True)
self.assertEqual(os.path.exists(os.path.join(output_dir, 'emoji')), True)
self.assertEqual(os.path.exists(os.path.join(output_dir, 'attachment.json')), True)
realm = read_file('realm.json')
# test realm
self.assertEqual('Organization imported from Gitter!',
realm['zerver_realm'][0]['description'])
# test users
exported_user_ids = get_set(realm['zerver_userprofile'], 'id')
exported_user_full_name = get_set(realm['zerver_userprofile'], 'full_name')
self.assertIn('User Full Name', exported_user_full_name)
exported_user_email = get_set(realm['zerver_userprofile'], 'email')
self.assertIn('*****@*****.**', exported_user_email)
# test stream
self.assertEqual(len(realm['zerver_stream']), 1)
self.assertEqual(realm['zerver_stream'][0]['name'], 'from gitter')
self.assertEqual(realm['zerver_stream'][0]['deactivated'], False)
self.assertEqual(realm['zerver_stream'][0]['realm'], realm['zerver_realm'][0]['id'])
self.assertEqual(realm['zerver_defaultstream'][0]['stream'], realm['zerver_stream'][0]['id'])
# test recipient
exported_recipient_id = get_set(realm['zerver_recipient'], 'id')
exported_recipient_type = get_set(realm['zerver_recipient'], 'type')
self.assertEqual(set([1, 2]), exported_recipient_type)
# test subscription
exported_subscription_userprofile = get_set(realm['zerver_subscription'], 'user_profile')
self.assertEqual(set([0, 1]), exported_subscription_userprofile)
exported_subscription_recipient = get_set(realm['zerver_subscription'], 'recipient')
self.assertEqual(len(exported_subscription_recipient), 3)
self.assertIn(realm['zerver_subscription'][1]['recipient'], exported_recipient_id)
messages = read_file('messages-000001.json')
# test messages
exported_messages_id = get_set(messages['zerver_message'], 'id')
self.assertIn(messages['zerver_message'][0]['sender'], exported_user_ids)
self.assertIn(messages['zerver_message'][1]['recipient'], exported_recipient_id)
self.assertIn(messages['zerver_message'][0]['content'], 'test message')
# test usermessages
exported_usermessage_userprofile = get_set(messages['zerver_usermessage'], 'user_profile')
self.assertEqual(exported_user_ids, exported_usermessage_userprofile)
exported_usermessage_message = get_set(messages['zerver_usermessage'], 'message')
self.assertEqual(exported_usermessage_message, exported_messages_id)
def test_gitter_import_data_conversion(
self, mock_process_avatars: mock.Mock) -> None:
output_dir = self.make_import_output_dir("gitter")
gitter_file = os.path.join(os.path.dirname(__file__),
"fixtures/gitter_data.json")
with self.assertLogs(level="INFO"):
do_convert_data(gitter_file, output_dir)
def read_file(output_file: str) -> Any:
full_path = os.path.join(output_dir, output_file)
with open(full_path, "rb") as f:
return orjson.loads(f.read())
self.assertEqual(os.path.exists(os.path.join(output_dir, "avatars")),
True)
self.assertEqual(os.path.exists(os.path.join(output_dir, "emoji")),
True)
self.assertEqual(
os.path.exists(os.path.join(output_dir, "attachment.json")), True)
realm = read_file("realm.json")
# test realm
self.assertEqual("Organization imported from Gitter!",
realm["zerver_realm"][0]["description"])
# test users
exported_user_ids = self.get_set(realm["zerver_userprofile"], "id")
exported_user_full_name = self.get_set(realm["zerver_userprofile"],
"full_name")
self.assertIn("User Full Name", exported_user_full_name)
exported_user_email = self.get_set(realm["zerver_userprofile"],
"email")
self.assertIn("*****@*****.**",
exported_user_email)
# test stream
self.assert_length(realm["zerver_stream"], 1)
self.assertEqual(realm["zerver_stream"][0]["name"], "from gitter")
self.assertEqual(realm["zerver_stream"][0]["deactivated"], False)
self.assertEqual(realm["zerver_stream"][0]["realm"],
realm["zerver_realm"][0]["id"])
self.assertEqual(realm["zerver_defaultstream"][0]["stream"],
realm["zerver_stream"][0]["id"])
# test recipient
exported_recipient_id = self.get_set(realm["zerver_recipient"], "id")
exported_recipient_type = self.get_set(realm["zerver_recipient"],
"type")
self.assertEqual({1, 2}, exported_recipient_type)
# test subscription
exported_subscription_userprofile = self.get_set(
realm["zerver_subscription"], "user_profile")
self.assertEqual({0, 1}, exported_subscription_userprofile)
exported_subscription_recipient = self.get_set(
realm["zerver_subscription"], "recipient")
self.assert_length(exported_subscription_recipient, 3)
self.assertIn(realm["zerver_subscription"][1]["recipient"],
exported_recipient_id)
messages = read_file("messages-000001.json")
# test messages
exported_messages_id = self.get_set(messages["zerver_message"], "id")
self.assertIn(messages["zerver_message"][0]["sender"],
exported_user_ids)
self.assertIn(messages["zerver_message"][1]["recipient"],
exported_recipient_id)
self.assertIn(messages["zerver_message"][0]["content"], "test message")
# test usermessages
exported_usermessage_userprofile = self.get_set(
messages["zerver_usermessage"], "user_profile")
self.assertEqual(exported_user_ids, exported_usermessage_userprofile)
exported_usermessage_message = self.get_set(
messages["zerver_usermessage"], "message")
self.assertEqual(exported_usermessage_message, exported_messages_id)
def test_gitter_import_data_conversion(
self, mock_process_avatars: mock.Mock) -> None:
output_dir = self._make_output_dir()
gitter_file = os.path.join(os.path.dirname(__file__),
'fixtures/gitter_data.json')
do_convert_data(gitter_file, output_dir)
def read_file(output_file: str) -> Any:
full_path = os.path.join(output_dir, output_file)
with open(full_path) as f:
return ujson.load(f)
def get_set(data: List[Dict[str, Any]], field: str) -> Set[str]:
values = set(r[field] for r in data)
return values
self.assertEqual(os.path.exists(os.path.join(output_dir, 'avatars')),
True)
self.assertEqual(os.path.exists(os.path.join(output_dir, 'emoji')),
True)
self.assertEqual(
os.path.exists(os.path.join(output_dir, 'attachment.json')), True)
realm = read_file('realm.json')
# test realm
self.assertEqual('Organization imported from Gitter!',
realm['zerver_realm'][0]['description'])
# test users
exported_user_ids = get_set(realm['zerver_userprofile'], 'id')
exported_user_full_name = get_set(realm['zerver_userprofile'],
'full_name')
self.assertIn('User Full Name', exported_user_full_name)
exported_user_email = get_set(realm['zerver_userprofile'], 'email')
self.assertIn('*****@*****.**',
exported_user_email)
# test stream
self.assertEqual(len(realm['zerver_stream']), 1)
self.assertEqual(realm['zerver_stream'][0]['name'], 'from gitter')
self.assertEqual(realm['zerver_stream'][0]['deactivated'], False)
self.assertEqual(realm['zerver_stream'][0]['realm'],
realm['zerver_realm'][0]['id'])
self.assertEqual(realm['zerver_defaultstream'][0]['stream'],
realm['zerver_stream'][0]['id'])
# test recipient
exported_recipient_id = get_set(realm['zerver_recipient'], 'id')
exported_recipient_type = get_set(realm['zerver_recipient'], 'type')
self.assertEqual(set([1, 2]), exported_recipient_type)
# test subscription
exported_subscription_userprofile = get_set(
realm['zerver_subscription'], 'user_profile')
self.assertEqual(set([0, 1]), exported_subscription_userprofile)
exported_subscription_recipient = get_set(realm['zerver_subscription'],
'recipient')
self.assertEqual(len(exported_subscription_recipient), 3)
self.assertIn(realm['zerver_subscription'][1]['recipient'],
exported_recipient_id)
messages = read_file('messages-000001.json')
# test messages
exported_messages_id = get_set(messages['zerver_message'], 'id')
self.assertIn(messages['zerver_message'][0]['sender'],
exported_user_ids)
self.assertIn(messages['zerver_message'][1]['recipient'],
exported_recipient_id)
self.assertIn(messages['zerver_message'][0]['content'], 'test message')
# test usermessages
exported_usermessage_userprofile = get_set(
messages['zerver_usermessage'], 'user_profile')
self.assertEqual(exported_user_ids, exported_usermessage_userprofile)
exported_usermessage_message = get_set(messages['zerver_usermessage'],
'message')
self.assertEqual(exported_usermessage_message, exported_messages_id)
def test_gitter_import_data_conversion(
self, mock_process_avatars: mock.Mock) -> None:
output_dir = self.make_import_output_dir("gitter")
gitter_file = os.path.join(os.path.dirname(__file__),
'fixtures/gitter_data.json')
with self.assertLogs(level='INFO'):
do_convert_data(gitter_file, output_dir)
def read_file(output_file: str) -> Any:
full_path = os.path.join(output_dir, output_file)
with open(full_path, "rb") as f:
return orjson.loads(f.read())
self.assertEqual(os.path.exists(os.path.join(output_dir, 'avatars')),
True)
self.assertEqual(os.path.exists(os.path.join(output_dir, 'emoji')),
True)
self.assertEqual(
os.path.exists(os.path.join(output_dir, 'attachment.json')), True)
realm = read_file('realm.json')
# test realm
self.assertEqual('Organization imported from Gitter!',
realm['zerver_realm'][0]['description'])
# test users
exported_user_ids = self.get_set(realm['zerver_userprofile'], 'id')
exported_user_full_name = self.get_set(realm['zerver_userprofile'],
'full_name')
self.assertIn('User Full Name', exported_user_full_name)
exported_user_email = self.get_set(realm['zerver_userprofile'],
'email')
self.assertIn('*****@*****.**',
exported_user_email)
# test stream
self.assertEqual(len(realm['zerver_stream']), 1)
self.assertEqual(realm['zerver_stream'][0]['name'], 'from gitter')
self.assertEqual(realm['zerver_stream'][0]['deactivated'], False)
self.assertEqual(realm['zerver_stream'][0]['realm'],
realm['zerver_realm'][0]['id'])
self.assertEqual(realm['zerver_defaultstream'][0]['stream'],
realm['zerver_stream'][0]['id'])
# test recipient
exported_recipient_id = self.get_set(realm['zerver_recipient'], 'id')
exported_recipient_type = self.get_set(realm['zerver_recipient'],
'type')
self.assertEqual({1, 2}, exported_recipient_type)
# test subscription
exported_subscription_userprofile = self.get_set(
realm['zerver_subscription'], 'user_profile')
self.assertEqual({0, 1}, exported_subscription_userprofile)
exported_subscription_recipient = self.get_set(
realm['zerver_subscription'], 'recipient')
self.assertEqual(len(exported_subscription_recipient), 3)
self.assertIn(realm['zerver_subscription'][1]['recipient'],
exported_recipient_id)
messages = read_file('messages-000001.json')
# test messages
exported_messages_id = self.get_set(messages['zerver_message'], 'id')
self.assertIn(messages['zerver_message'][0]['sender'],
exported_user_ids)
self.assertIn(messages['zerver_message'][1]['recipient'],
exported_recipient_id)
self.assertIn(messages['zerver_message'][0]['content'], 'test message')
# test usermessages
exported_usermessage_userprofile = self.get_set(
messages['zerver_usermessage'], 'user_profile')
self.assertEqual(exported_user_ids, exported_usermessage_userprofile)
exported_usermessage_message = self.get_set(
messages['zerver_usermessage'], 'message')
self.assertEqual(exported_usermessage_message, exported_messages_id)
