def test_gitter_import_to_existing_database(self, mock_process_avatars: mock.Mock) -> None: output_dir = self._make_output_dir() gitter_file = os.path.join(os.path.dirname(__file__), 'fixtures/gitter_data.json') do_convert_data(gitter_file, output_dir) do_import_realm(output_dir, 'test-gitter-import') realm = get_realm('test-gitter-import') # test rendered_messages realm_users = UserProfile.objects.filter(realm=realm) messages = Message.objects.filter(sender__in=realm_users) for message in messages: self.assertIsNotNone(message.rendered_content, None)
def test_gitter_import_to_existing_database( self, mock_process_avatars: mock.Mock) -> None: output_dir = self._make_output_dir() gitter_file = os.path.join(os.path.dirname(__file__), 'fixtures/gitter_data.json') do_convert_data(gitter_file, output_dir) do_import_realm(output_dir, 'test-gitter-import') realm = get_realm('test-gitter-import') # test rendered_messages realm_users = UserProfile.objects.filter(realm=realm) messages = Message.objects.filter(sender__in=realm_users) for message in messages: self.assertIsNotNone(message.rendered_content, None)
def handle(self, *args: Any, **options: Any) -> None: output_dir = options["output_dir"] if output_dir is None: output_dir = tempfile.mkdtemp(prefix="converted-gitter-data-") else: output_dir = os.path.realpath(output_dir) num_threads = int(options['threads']) if num_threads < 1: raise CommandError('You must have at least one thread.') for path in options['gitter_data']: if not os.path.exists(path): raise CommandError(f"Gitter data file not found: '{path}'") # TODO add json check print("Converting data ...") do_convert_data(path, output_dir, num_threads)
def handle(self, *args: Any, **options: Any) -> None: output_dir = options["output_dir"] if output_dir is None: output_dir = tempfile.mkdtemp(prefix="/tmp/converted-gitter-data-") else: output_dir = os.path.realpath(output_dir) num_threads = int(options['threads']) if num_threads < 1: raise CommandError('You must have at least one thread.') for path in options['gitter_data']: if not os.path.exists(path): print("Gitter data file not found: '%s'" % (path,)) exit(1) # TODO add json check print("Converting Data ...") do_convert_data(path, output_dir, num_threads)
def test_gitter_import_to_existing_database( self, mock_process_avatars: mock.Mock) -> None: output_dir = self.make_import_output_dir("gitter") gitter_file = os.path.join(os.path.dirname(__file__), "fixtures/gitter_data.json") with self.assertLogs(level="INFO"): do_convert_data(gitter_file, output_dir) with self.assertLogs(level="INFO"): do_import_realm(output_dir, "test-gitter-import") realm = get_realm("test-gitter-import") # test rendered_messages realm_users = UserProfile.objects.filter(realm=realm) messages = Message.objects.filter(sender__in=realm_users) for message in messages: self.assertIsNotNone(message.rendered_content, None)
def test_gitter_import_data_conversion(self, mock_process_avatars: mock.Mock) -> None: output_dir = self._make_output_dir() gitter_file = os.path.join(os.path.dirname(__file__), 'fixtures/gitter_data.json') do_convert_data(gitter_file, output_dir) def read_file(output_file: str) -> Any: full_path = os.path.join(output_dir, output_file) with open(full_path) as f: return ujson.load(f) def get_set(data: List[Dict[str, Any]], field: str) -> Set[str]: values = set(r[field] for r in data) return values self.assertEqual(os.path.exists(os.path.join(output_dir, 'avatars')), True) self.assertEqual(os.path.exists(os.path.join(output_dir, 'emoji')), True) self.assertEqual(os.path.exists(os.path.join(output_dir, 'attachment.json')), True) realm = read_file('realm.json') # test realm self.assertEqual('Organization imported from Gitter!', realm['zerver_realm'][0]['description']) # test users exported_user_ids = get_set(realm['zerver_userprofile'], 'id') exported_user_full_name = get_set(realm['zerver_userprofile'], 'full_name') self.assertIn('User Full Name', exported_user_full_name) exported_user_email = get_set(realm['zerver_userprofile'], 'email') self.assertIn('*****@*****.**', exported_user_email) # test stream self.assertEqual(len(realm['zerver_stream']), 1) self.assertEqual(realm['zerver_stream'][0]['name'], 'from gitter') self.assertEqual(realm['zerver_stream'][0]['deactivated'], False) self.assertEqual(realm['zerver_stream'][0]['realm'], realm['zerver_realm'][0]['id']) self.assertEqual(realm['zerver_defaultstream'][0]['stream'], realm['zerver_stream'][0]['id']) # test recipient exported_recipient_id = get_set(realm['zerver_recipient'], 'id') exported_recipient_type = get_set(realm['zerver_recipient'], 'type') self.assertEqual(set([1, 2]), exported_recipient_type) # test subscription exported_subscription_userprofile = get_set(realm['zerver_subscription'], 'user_profile') self.assertEqual(set([0, 1]), exported_subscription_userprofile) exported_subscription_recipient = get_set(realm['zerver_subscription'], 'recipient') self.assertEqual(len(exported_subscription_recipient), 3) self.assertIn(realm['zerver_subscription'][1]['recipient'], exported_recipient_id) messages = read_file('messages-000001.json') # test messages exported_messages_id = get_set(messages['zerver_message'], 'id') self.assertIn(messages['zerver_message'][0]['sender'], exported_user_ids) self.assertIn(messages['zerver_message'][1]['recipient'], exported_recipient_id) self.assertIn(messages['zerver_message'][0]['content'], 'test message') # test usermessages exported_usermessage_userprofile = get_set(messages['zerver_usermessage'], 'user_profile') self.assertEqual(exported_user_ids, exported_usermessage_userprofile) exported_usermessage_message = get_set(messages['zerver_usermessage'], 'message') self.assertEqual(exported_usermessage_message, exported_messages_id)
def test_gitter_import_data_conversion( self, mock_process_avatars: mock.Mock) -> None: output_dir = self.make_import_output_dir("gitter") gitter_file = os.path.join(os.path.dirname(__file__), "fixtures/gitter_data.json") with self.assertLogs(level="INFO"): do_convert_data(gitter_file, output_dir) def read_file(output_file: str) -> Any: full_path = os.path.join(output_dir, output_file) with open(full_path, "rb") as f: return orjson.loads(f.read()) self.assertEqual(os.path.exists(os.path.join(output_dir, "avatars")), True) self.assertEqual(os.path.exists(os.path.join(output_dir, "emoji")), True) self.assertEqual( os.path.exists(os.path.join(output_dir, "attachment.json")), True) realm = read_file("realm.json") # test realm self.assertEqual("Organization imported from Gitter!", realm["zerver_realm"][0]["description"]) # test users exported_user_ids = self.get_set(realm["zerver_userprofile"], "id") exported_user_full_name = self.get_set(realm["zerver_userprofile"], "full_name") self.assertIn("User Full Name", exported_user_full_name) exported_user_email = self.get_set(realm["zerver_userprofile"], "email") self.assertIn("*****@*****.**", exported_user_email) # test stream self.assert_length(realm["zerver_stream"], 1) self.assertEqual(realm["zerver_stream"][0]["name"], "from gitter") self.assertEqual(realm["zerver_stream"][0]["deactivated"], False) self.assertEqual(realm["zerver_stream"][0]["realm"], realm["zerver_realm"][0]["id"]) self.assertEqual(realm["zerver_defaultstream"][0]["stream"], realm["zerver_stream"][0]["id"]) # test recipient exported_recipient_id = self.get_set(realm["zerver_recipient"], "id") exported_recipient_type = self.get_set(realm["zerver_recipient"], "type") self.assertEqual({1, 2}, exported_recipient_type) # test subscription exported_subscription_userprofile = self.get_set( realm["zerver_subscription"], "user_profile") self.assertEqual({0, 1}, exported_subscription_userprofile) exported_subscription_recipient = self.get_set( realm["zerver_subscription"], "recipient") self.assert_length(exported_subscription_recipient, 3) self.assertIn(realm["zerver_subscription"][1]["recipient"], exported_recipient_id) messages = read_file("messages-000001.json") # test messages exported_messages_id = self.get_set(messages["zerver_message"], "id") self.assertIn(messages["zerver_message"][0]["sender"], exported_user_ids) self.assertIn(messages["zerver_message"][1]["recipient"], exported_recipient_id) self.assertIn(messages["zerver_message"][0]["content"], "test message") # test usermessages exported_usermessage_userprofile = self.get_set( messages["zerver_usermessage"], "user_profile") self.assertEqual(exported_user_ids, exported_usermessage_userprofile) exported_usermessage_message = self.get_set( messages["zerver_usermessage"], "message") self.assertEqual(exported_usermessage_message, exported_messages_id)
def test_gitter_import_data_conversion( self, mock_process_avatars: mock.Mock) -> None: output_dir = self._make_output_dir() gitter_file = os.path.join(os.path.dirname(__file__), 'fixtures/gitter_data.json') do_convert_data(gitter_file, output_dir) def read_file(output_file: str) -> Any: full_path = os.path.join(output_dir, output_file) with open(full_path) as f: return ujson.load(f) def get_set(data: List[Dict[str, Any]], field: str) -> Set[str]: values = set(r[field] for r in data) return values self.assertEqual(os.path.exists(os.path.join(output_dir, 'avatars')), True) self.assertEqual(os.path.exists(os.path.join(output_dir, 'emoji')), True) self.assertEqual( os.path.exists(os.path.join(output_dir, 'attachment.json')), True) realm = read_file('realm.json') # test realm self.assertEqual('Organization imported from Gitter!', realm['zerver_realm'][0]['description']) # test users exported_user_ids = get_set(realm['zerver_userprofile'], 'id') exported_user_full_name = get_set(realm['zerver_userprofile'], 'full_name') self.assertIn('User Full Name', exported_user_full_name) exported_user_email = get_set(realm['zerver_userprofile'], 'email') self.assertIn('*****@*****.**', exported_user_email) # test stream self.assertEqual(len(realm['zerver_stream']), 1) self.assertEqual(realm['zerver_stream'][0]['name'], 'from gitter') self.assertEqual(realm['zerver_stream'][0]['deactivated'], False) self.assertEqual(realm['zerver_stream'][0]['realm'], realm['zerver_realm'][0]['id']) self.assertEqual(realm['zerver_defaultstream'][0]['stream'], realm['zerver_stream'][0]['id']) # test recipient exported_recipient_id = get_set(realm['zerver_recipient'], 'id') exported_recipient_type = get_set(realm['zerver_recipient'], 'type') self.assertEqual(set([1, 2]), exported_recipient_type) # test subscription exported_subscription_userprofile = get_set( realm['zerver_subscription'], 'user_profile') self.assertEqual(set([0, 1]), exported_subscription_userprofile) exported_subscription_recipient = get_set(realm['zerver_subscription'], 'recipient') self.assertEqual(len(exported_subscription_recipient), 3) self.assertIn(realm['zerver_subscription'][1]['recipient'], exported_recipient_id) messages = read_file('messages-000001.json') # test messages exported_messages_id = get_set(messages['zerver_message'], 'id') self.assertIn(messages['zerver_message'][0]['sender'], exported_user_ids) self.assertIn(messages['zerver_message'][1]['recipient'], exported_recipient_id) self.assertIn(messages['zerver_message'][0]['content'], 'test message') # test usermessages exported_usermessage_userprofile = get_set( messages['zerver_usermessage'], 'user_profile') self.assertEqual(exported_user_ids, exported_usermessage_userprofile) exported_usermessage_message = get_set(messages['zerver_usermessage'], 'message') self.assertEqual(exported_usermessage_message, exported_messages_id)
def test_gitter_import_data_conversion( self, mock_process_avatars: mock.Mock) -> None: output_dir = self.make_import_output_dir("gitter") gitter_file = os.path.join(os.path.dirname(__file__), 'fixtures/gitter_data.json') with self.assertLogs(level='INFO'): do_convert_data(gitter_file, output_dir) def read_file(output_file: str) -> Any: full_path = os.path.join(output_dir, output_file) with open(full_path, "rb") as f: return orjson.loads(f.read()) self.assertEqual(os.path.exists(os.path.join(output_dir, 'avatars')), True) self.assertEqual(os.path.exists(os.path.join(output_dir, 'emoji')), True) self.assertEqual( os.path.exists(os.path.join(output_dir, 'attachment.json')), True) realm = read_file('realm.json') # test realm self.assertEqual('Organization imported from Gitter!', realm['zerver_realm'][0]['description']) # test users exported_user_ids = self.get_set(realm['zerver_userprofile'], 'id') exported_user_full_name = self.get_set(realm['zerver_userprofile'], 'full_name') self.assertIn('User Full Name', exported_user_full_name) exported_user_email = self.get_set(realm['zerver_userprofile'], 'email') self.assertIn('*****@*****.**', exported_user_email) # test stream self.assertEqual(len(realm['zerver_stream']), 1) self.assertEqual(realm['zerver_stream'][0]['name'], 'from gitter') self.assertEqual(realm['zerver_stream'][0]['deactivated'], False) self.assertEqual(realm['zerver_stream'][0]['realm'], realm['zerver_realm'][0]['id']) self.assertEqual(realm['zerver_defaultstream'][0]['stream'], realm['zerver_stream'][0]['id']) # test recipient exported_recipient_id = self.get_set(realm['zerver_recipient'], 'id') exported_recipient_type = self.get_set(realm['zerver_recipient'], 'type') self.assertEqual({1, 2}, exported_recipient_type) # test subscription exported_subscription_userprofile = self.get_set( realm['zerver_subscription'], 'user_profile') self.assertEqual({0, 1}, exported_subscription_userprofile) exported_subscription_recipient = self.get_set( realm['zerver_subscription'], 'recipient') self.assertEqual(len(exported_subscription_recipient), 3) self.assertIn(realm['zerver_subscription'][1]['recipient'], exported_recipient_id) messages = read_file('messages-000001.json') # test messages exported_messages_id = self.get_set(messages['zerver_message'], 'id') self.assertIn(messages['zerver_message'][0]['sender'], exported_user_ids) self.assertIn(messages['zerver_message'][1]['recipient'], exported_recipient_id) self.assertIn(messages['zerver_message'][0]['content'], 'test message') # test usermessages exported_usermessage_userprofile = self.get_set( messages['zerver_usermessage'], 'user_profile') self.assertEqual(exported_user_ids, exported_usermessage_userprofile) exported_usermessage_message = self.get_set( messages['zerver_usermessage'], 'message') self.assertEqual(exported_usermessage_message, exported_messages_id)