def _export_realm( self, realm: Realm, exportable_user_ids: Optional[Set[int]] = None) -> Dict[str, Any]: output_dir = self._make_output_dir() with patch('logging.info'), patch( 'zerver.lib.export.create_soft_link'): do_export_realm( realm=realm, output_dir=output_dir, threads=0, exportable_user_ids=exportable_user_ids, ) # TODO: Process the second partial file, which can be created # for certain edge cases. export_usermessages_batch( input_path=os.path.join(output_dir, 'messages-000001.json.partial'), output_path=os.path.join(output_dir, 'message.json')) def read_file(fn: str) -> Any: full_fn = os.path.join(output_dir, fn) with open(full_fn) as f: return ujson.load(f) result = {} result['realm'] = read_file('realm.json') result['attachment'] = read_file('attachment.json') result['message'] = read_file('message.json') result['uploads_dir'] = os.path.join(output_dir, 'uploads') return result
def _export_realm(self, realm, exportable_user_ids=None): # type: (Realm, Set[int]) -> Dict[str, Any] output_dir = self._make_output_dir() with patch('logging.info'), patch('zerver.lib.export.create_soft_link'): do_export_realm( realm=realm, output_dir=output_dir, threads=0, exportable_user_ids=exportable_user_ids, ) # TODO: Process the second partial file, which can be created # for certain edge cases. export_usermessages_batch( input_path=os.path.join(output_dir, 'messages-000001.json.partial'), output_path=os.path.join(output_dir, 'message.json') ) def read_file(fn): # type: (str) -> Any full_fn = os.path.join(output_dir, fn) with open(full_fn) as f: return ujson.load(f) result = {} result['realm'] = read_file('realm.json') result['attachment'] = read_file('attachment.json') result['message'] = read_file('message.json') result['uploads_dir'] = os.path.join(output_dir, 'uploads') return result
def _export_realm( self, realm: Realm, exportable_user_ids: Optional[Set[int]] = None, consent_message_id: Optional[int] = None) -> Dict[str, Any]: output_dir = self._make_output_dir() with patch('logging.info'), patch( 'zerver.lib.export.create_soft_link'): do_export_realm( realm=realm, output_dir=output_dir, threads=0, exportable_user_ids=exportable_user_ids, consent_message_id=consent_message_id, ) export_usermessages_batch( input_path=os.path.join(output_dir, 'messages-000001.json.partial'), output_path=os.path.join(output_dir, 'messages-000001.json'), consent_message_id=consent_message_id, ) try: export_usermessages_batch( input_path=os.path.join(output_dir, 'messages-000002.json.partial'), output_path=os.path.join(output_dir, 'messages-000002.json'), consent_message_id=consent_message_id, ) except FileNotFoundError: pass def read_file(fn: str) -> Any: full_fn = os.path.join(output_dir, fn) with open(full_fn) as f: return ujson.load(f) result = {} result['realm'] = read_file('realm.json') result['attachment'] = read_file('attachment.json') result['message'] = read_file('messages-000001.json') try: message = read_file('messages-000002.json') result["message"]["zerver_usermessage"].extend( message["zerver_usermessage"]) result["message"]["zerver_message"].extend( message["zerver_message"]) except FileNotFoundError: pass result['uploads_dir'] = os.path.join(output_dir, 'uploads') result['uploads_dir_records'] = read_file( os.path.join('uploads', 'records.json')) result['emoji_dir'] = os.path.join(output_dir, 'emoji') result['emoji_dir_records'] = read_file( os.path.join('emoji', 'records.json')) result['avatar_dir'] = os.path.join(output_dir, 'avatars') result['avatar_dir_records'] = read_file( os.path.join('avatars', 'records.json')) return result
def handle(self, *args, **options): # type: (*Any, **Any) -> None realm = self.get_realm(options) output_dir = options["output_dir"] if output_dir is None: output_dir = tempfile.mkdtemp(prefix="/tmp/zulip-export-") if os.path.exists(output_dir): shutil.rmtree(output_dir) os.makedirs(output_dir) print("Exporting realm %s" % (realm.string_id, )) num_threads = int(options['threads']) if num_threads < 1: raise CommandError('You must have at least one thread.') do_export_realm(realm, output_dir, threads=num_threads) print("Finished exporting to %s; tarring" % (output_dir, )) do_write_stats_file_for_realm_export(output_dir) tarball_path = output_dir.rstrip('/') + '.tar.gz' os.chdir(os.path.dirname(output_dir)) subprocess.check_call( ["tar", "-czf", tarball_path, os.path.basename(output_dir)]) print("Tarball written to %s" % (tarball_path, ))
def handle(self, *args: Any, **options: Any) -> None: realm = self.get_realm(options) assert realm is not None # Should be ensured by parser output_dir = options["output_dir"] if output_dir is None: output_dir = tempfile.mkdtemp(prefix="/tmp/zulip-export-") else: output_dir = os.path.realpath(output_dir) if os.path.exists(output_dir): shutil.rmtree(output_dir) os.makedirs(output_dir) print("Exporting realm %s" % (realm.string_id,)) num_threads = int(options['threads']) if num_threads < 1: raise CommandError('You must have at least one thread.') do_export_realm(realm, output_dir, threads=num_threads) print("Finished exporting to %s; tarring" % (output_dir,)) do_write_stats_file_for_realm_export(output_dir) tarball_path = output_dir.rstrip('/') + '.tar.gz' os.chdir(os.path.dirname(output_dir)) subprocess.check_call(["tar", "-czf", tarball_path, os.path.basename(output_dir)]) print("Tarball written to %s" % (tarball_path,))
def handle(self, *args, **options): # type: (*Any, **Any) -> None try: realm = get_realm(options["realm"]) except ValidationError: raise CommandError("No such realm.") output_dir = options["output_dir"] if output_dir is None: output_dir = tempfile.mkdtemp(prefix="/tmp/zulip-export-") if os.path.exists(output_dir): shutil.rmtree(output_dir) os.makedirs(output_dir) print("Exporting realm %s" % (realm.string_id,)) num_threads = int(options['threads']) if num_threads < 1: raise CommandError('You must have at least one thread.') do_export_realm(realm, output_dir, threads=num_threads) print("Finished exporting to %s; tarring" % (output_dir,)) do_write_stats_file_for_realm_export(output_dir) tarball_path = output_dir.rstrip('/') + '.tar.gz' os.chdir(os.path.dirname(output_dir)) subprocess.check_call(["tar", "-czf", tarball_path, os.path.basename(output_dir)]) print("Tarball written to %s" % (tarball_path,))
def handle(self, *args: Any, **options: Any) -> None: realm = self.get_realm(options) assert realm is not None # Should be ensured by parser output_dir = options["output_dir"] if output_dir is None: output_dir = tempfile.mkdtemp(prefix="zulip-export-") else: output_dir = os.path.realpath(os.path.expanduser(output_dir)) if os.path.exists(output_dir): shutil.rmtree(output_dir) os.makedirs(output_dir) print("Exporting realm %s" % (realm.string_id, )) num_threads = int(options['threads']) if num_threads < 1: raise CommandError('You must have at least one thread.') do_export_realm(realm, output_dir, threads=num_threads, public_only=options["public_only"]) print("Finished exporting to %s; tarring" % (output_dir, )) do_write_stats_file_for_realm_export(output_dir) tarball_path = output_dir.rstrip('/') + '.tar.gz' os.chdir(os.path.dirname(output_dir)) subprocess.check_call( ["tar", "-czf", tarball_path, os.path.basename(output_dir)]) print("Tarball written to %s" % (tarball_path, )) if not options["upload_to_s3"]: return def percent_callback(complete: Any, total: Any) -> None: sys.stdout.write('.') sys.stdout.flush() if settings.LOCAL_UPLOADS_DIR is not None: raise CommandError("S3 backend must be configured to upload to S3") print("Uploading export tarball to S3") from zerver.lib.upload import S3Connection, get_bucket, Key conn = S3Connection(settings.S3_KEY, settings.S3_SECRET_KEY) # We use the avatar bucket, because it's world-readable. bucket = get_bucket(conn, settings.S3_AVATAR_BUCKET) key = Key(bucket) key.key = os.path.join("exports", generate_random_token(32), os.path.basename(tarball_path)) key.set_contents_from_filename(tarball_path, cb=percent_callback, num_cb=40) public_url = 'https://{bucket}.{host}/{key}'.format( host=conn.server_name(), bucket=bucket.name, key=key.key) print("Uploaded to %s" % (public_url, ))
def handle(self, *args: Any, **options: Any) -> None: realm = self.get_realm(options) assert realm is not None # Should be ensured by parser output_dir = options["output_dir"] if output_dir is None: output_dir = tempfile.mkdtemp(prefix="zulip-export-") else: output_dir = os.path.realpath(os.path.expanduser(output_dir)) if os.path.exists(output_dir): shutil.rmtree(output_dir) os.makedirs(output_dir) print("Exporting realm %s" % (realm.string_id,)) num_threads = int(options['threads']) if num_threads < 1: raise CommandError('You must have at least one thread.') do_export_realm(realm, output_dir, threads=num_threads, public_only=options["public_only"]) print("Finished exporting to %s; tarring" % (output_dir,)) do_write_stats_file_for_realm_export(output_dir) tarball_path = output_dir.rstrip('/') + '.tar.gz' os.chdir(os.path.dirname(output_dir)) subprocess.check_call(["tar", "-czf", tarball_path, os.path.basename(output_dir)]) print("Tarball written to %s" % (tarball_path,)) if not options["upload_to_s3"]: return def percent_callback(complete: Any, total: Any) -> None: sys.stdout.write('.') sys.stdout.flush() if settings.LOCAL_UPLOADS_DIR is not None: raise CommandError("S3 backend must be configured to upload to S3") print("Uploading export tarball to S3") from zerver.lib.upload import S3Connection, get_bucket, Key conn = S3Connection(settings.S3_KEY, settings.S3_SECRET_KEY) # We use the avatar bucket, because it's world-readable. bucket = get_bucket(conn, settings.S3_AVATAR_BUCKET) key = Key(bucket) key.key = os.path.join("exports", generate_random_token(32), os.path.basename(tarball_path)) key.set_contents_from_filename(tarball_path, cb=percent_callback, num_cb=40) public_url = 'https://{bucket}.{host}/{key}'.format( host=conn.server_name(), bucket=bucket.name, key=key.key) print("Uploaded to %s" % (public_url,))
def handle(self, *args, **options): try: realm = get_realm(options["realm"]) except ValidationError: raise CommandError("No such realm.") output_dir = options["output_dir"] if output_dir is None: output_dir = tempfile.mkdtemp(prefix="/tmp/zulip-export-") if os.path.exists(output_dir): shutil.rmtree(output_dir) os.makedirs(output_dir) print("Exporting realm %s" % (realm.domain,)) do_export_realm(realm, output_dir, threads=int(options['threads'])) print("Finished exporting to %s; tarring" % (output_dir,)) tarball_path = output_dir.rstrip('/') + '.tar.gz' os.chdir(os.path.dirname(output_dir)) subprocess.check_call(["tar", "-czf", tarball_path, os.path.basename(output_dir)]) print("Tarball written to %s" % (tarball_path,))