def _export_realm(
self,
realm: Realm,
exportable_user_ids: Optional[Set[int]] = None) -> Dict[str, Any]:
output_dir = self._make_output_dir()
with patch('logging.info'), patch(
'zerver.lib.export.create_soft_link'):
do_export_realm(
realm=realm,
output_dir=output_dir,
threads=0,
exportable_user_ids=exportable_user_ids,
)
# TODO: Process the second partial file, which can be created
# for certain edge cases.
export_usermessages_batch(
input_path=os.path.join(output_dir,
'messages-000001.json.partial'),
output_path=os.path.join(output_dir, 'message.json'))
def read_file(fn: str) -> Any:
full_fn = os.path.join(output_dir, fn)
with open(full_fn) as f:
return ujson.load(f)
result = {}
result['realm'] = read_file('realm.json')
result['attachment'] = read_file('attachment.json')
result['message'] = read_file('message.json')
result['uploads_dir'] = os.path.join(output_dir, 'uploads')
return result
def _export_realm(self, realm, exportable_user_ids=None):
# type: (Realm, Set[int]) -> Dict[str, Any]
output_dir = self._make_output_dir()
with patch('logging.info'), patch('zerver.lib.export.create_soft_link'):
do_export_realm(
realm=realm,
output_dir=output_dir,
threads=0,
exportable_user_ids=exportable_user_ids,
)
# TODO: Process the second partial file, which can be created
# for certain edge cases.
export_usermessages_batch(
input_path=os.path.join(output_dir, 'messages-000001.json.partial'),
output_path=os.path.join(output_dir, 'message.json')
)
def read_file(fn):
# type: (str) -> Any
full_fn = os.path.join(output_dir, fn)
with open(full_fn) as f:
return ujson.load(f)
result = {}
result['realm'] = read_file('realm.json')
result['attachment'] = read_file('attachment.json')
result['message'] = read_file('message.json')
result['uploads_dir'] = os.path.join(output_dir, 'uploads')
return result
def _export_realm(
self,
realm: Realm,
exportable_user_ids: Optional[Set[int]] = None,
consent_message_id: Optional[int] = None) -> Dict[str, Any]:
output_dir = self._make_output_dir()
with patch('logging.info'), patch(
'zerver.lib.export.create_soft_link'):
do_export_realm(
realm=realm,
output_dir=output_dir,
threads=0,
exportable_user_ids=exportable_user_ids,
consent_message_id=consent_message_id,
)
export_usermessages_batch(
input_path=os.path.join(output_dir,
'messages-000001.json.partial'),
output_path=os.path.join(output_dir, 'messages-000001.json'),
consent_message_id=consent_message_id,
)
try:
export_usermessages_batch(
input_path=os.path.join(output_dir,
'messages-000002.json.partial'),
output_path=os.path.join(output_dir,
'messages-000002.json'),
consent_message_id=consent_message_id,
)
except FileNotFoundError:
pass
def read_file(fn: str) -> Any:
full_fn = os.path.join(output_dir, fn)
with open(full_fn) as f:
return ujson.load(f)
result = {}
result['realm'] = read_file('realm.json')
result['attachment'] = read_file('attachment.json')
result['message'] = read_file('messages-000001.json')
try:
message = read_file('messages-000002.json')
result["message"]["zerver_usermessage"].extend(
message["zerver_usermessage"])
result["message"]["zerver_message"].extend(
message["zerver_message"])
except FileNotFoundError:
pass
result['uploads_dir'] = os.path.join(output_dir, 'uploads')
result['uploads_dir_records'] = read_file(
os.path.join('uploads', 'records.json'))
result['emoji_dir'] = os.path.join(output_dir, 'emoji')
result['emoji_dir_records'] = read_file(
os.path.join('emoji', 'records.json'))
result['avatar_dir'] = os.path.join(output_dir, 'avatars')
result['avatar_dir_records'] = read_file(
os.path.join('avatars', 'records.json'))
return result
def handle(self, *args, **options):
# type: (*Any, **Any) -> None
realm = self.get_realm(options)
output_dir = options["output_dir"]
if output_dir is None:
output_dir = tempfile.mkdtemp(prefix="/tmp/zulip-export-")
if os.path.exists(output_dir):
shutil.rmtree(output_dir)
os.makedirs(output_dir)
print("Exporting realm %s" % (realm.string_id, ))
num_threads = int(options['threads'])
if num_threads < 1:
raise CommandError('You must have at least one thread.')
do_export_realm(realm, output_dir, threads=num_threads)
print("Finished exporting to %s; tarring" % (output_dir, ))
do_write_stats_file_for_realm_export(output_dir)
tarball_path = output_dir.rstrip('/') + '.tar.gz'
os.chdir(os.path.dirname(output_dir))
subprocess.check_call(
["tar", "-czf", tarball_path,
os.path.basename(output_dir)])
print("Tarball written to %s" % (tarball_path, ))
def handle(self, *args: Any, **options: Any) -> None:
realm = self.get_realm(options)
assert realm is not None # Should be ensured by parser
output_dir = options["output_dir"]
if output_dir is None:
output_dir = tempfile.mkdtemp(prefix="/tmp/zulip-export-")
else:
output_dir = os.path.realpath(output_dir)
if os.path.exists(output_dir):
shutil.rmtree(output_dir)
os.makedirs(output_dir)
print("Exporting realm %s" % (realm.string_id,))
num_threads = int(options['threads'])
if num_threads < 1:
raise CommandError('You must have at least one thread.')
do_export_realm(realm, output_dir, threads=num_threads)
print("Finished exporting to %s; tarring" % (output_dir,))
do_write_stats_file_for_realm_export(output_dir)
tarball_path = output_dir.rstrip('/') + '.tar.gz'
os.chdir(os.path.dirname(output_dir))
subprocess.check_call(["tar", "-czf", tarball_path, os.path.basename(output_dir)])
print("Tarball written to %s" % (tarball_path,))
def handle(self, *args, **options):
# type: (*Any, **Any) -> None
try:
realm = get_realm(options["realm"])
except ValidationError:
raise CommandError("No such realm.")
output_dir = options["output_dir"]
if output_dir is None:
output_dir = tempfile.mkdtemp(prefix="/tmp/zulip-export-")
if os.path.exists(output_dir):
shutil.rmtree(output_dir)
os.makedirs(output_dir)
print("Exporting realm %s" % (realm.string_id,))
num_threads = int(options['threads'])
if num_threads < 1:
raise CommandError('You must have at least one thread.')
do_export_realm(realm, output_dir, threads=num_threads)
print("Finished exporting to %s; tarring" % (output_dir,))
do_write_stats_file_for_realm_export(output_dir)
tarball_path = output_dir.rstrip('/') + '.tar.gz'
os.chdir(os.path.dirname(output_dir))
subprocess.check_call(["tar", "-czf", tarball_path, os.path.basename(output_dir)])
print("Tarball written to %s" % (tarball_path,))
def handle(self, *args: Any, **options: Any) -> None:
realm = self.get_realm(options)
assert realm is not None # Should be ensured by parser
output_dir = options["output_dir"]
if output_dir is None:
output_dir = tempfile.mkdtemp(prefix="zulip-export-")
else:
output_dir = os.path.realpath(os.path.expanduser(output_dir))
if os.path.exists(output_dir):
shutil.rmtree(output_dir)
os.makedirs(output_dir)
print("Exporting realm %s" % (realm.string_id, ))
num_threads = int(options['threads'])
if num_threads < 1:
raise CommandError('You must have at least one thread.')
do_export_realm(realm,
output_dir,
threads=num_threads,
public_only=options["public_only"])
print("Finished exporting to %s; tarring" % (output_dir, ))
do_write_stats_file_for_realm_export(output_dir)
tarball_path = output_dir.rstrip('/') + '.tar.gz'
os.chdir(os.path.dirname(output_dir))
subprocess.check_call(
["tar", "-czf", tarball_path,
os.path.basename(output_dir)])
print("Tarball written to %s" % (tarball_path, ))
if not options["upload_to_s3"]:
return
def percent_callback(complete: Any, total: Any) -> None:
sys.stdout.write('.')
sys.stdout.flush()
if settings.LOCAL_UPLOADS_DIR is not None:
raise CommandError("S3 backend must be configured to upload to S3")
print("Uploading export tarball to S3")
from zerver.lib.upload import S3Connection, get_bucket, Key
conn = S3Connection(settings.S3_KEY, settings.S3_SECRET_KEY)
# We use the avatar bucket, because it's world-readable.
bucket = get_bucket(conn, settings.S3_AVATAR_BUCKET)
key = Key(bucket)
key.key = os.path.join("exports", generate_random_token(32),
os.path.basename(tarball_path))
key.set_contents_from_filename(tarball_path,
cb=percent_callback,
num_cb=40)
public_url = 'https://{bucket}.{host}/{key}'.format(
host=conn.server_name(), bucket=bucket.name, key=key.key)
print("Uploaded to %s" % (public_url, ))
def handle(self, *args: Any, **options: Any) -> None:
realm = self.get_realm(options)
assert realm is not None # Should be ensured by parser
output_dir = options["output_dir"]
if output_dir is None:
output_dir = tempfile.mkdtemp(prefix="zulip-export-")
else:
output_dir = os.path.realpath(os.path.expanduser(output_dir))
if os.path.exists(output_dir):
shutil.rmtree(output_dir)
os.makedirs(output_dir)
print("Exporting realm %s" % (realm.string_id,))
num_threads = int(options['threads'])
if num_threads < 1:
raise CommandError('You must have at least one thread.')
do_export_realm(realm, output_dir, threads=num_threads, public_only=options["public_only"])
print("Finished exporting to %s; tarring" % (output_dir,))
do_write_stats_file_for_realm_export(output_dir)
tarball_path = output_dir.rstrip('/') + '.tar.gz'
os.chdir(os.path.dirname(output_dir))
subprocess.check_call(["tar", "-czf", tarball_path, os.path.basename(output_dir)])
print("Tarball written to %s" % (tarball_path,))
if not options["upload_to_s3"]:
return
def percent_callback(complete: Any, total: Any) -> None:
sys.stdout.write('.')
sys.stdout.flush()
if settings.LOCAL_UPLOADS_DIR is not None:
raise CommandError("S3 backend must be configured to upload to S3")
print("Uploading export tarball to S3")
from zerver.lib.upload import S3Connection, get_bucket, Key
conn = S3Connection(settings.S3_KEY, settings.S3_SECRET_KEY)
# We use the avatar bucket, because it's world-readable.
bucket = get_bucket(conn, settings.S3_AVATAR_BUCKET)
key = Key(bucket)
key.key = os.path.join("exports", generate_random_token(32), os.path.basename(tarball_path))
key.set_contents_from_filename(tarball_path, cb=percent_callback, num_cb=40)
public_url = 'https://{bucket}.{host}/{key}'.format(
host=conn.server_name(),
bucket=bucket.name,
key=key.key)
print("Uploaded to %s" % (public_url,))
def handle(self, *args, **options):
try:
realm = get_realm(options["realm"])
except ValidationError:
raise CommandError("No such realm.")
output_dir = options["output_dir"]
if output_dir is None:
output_dir = tempfile.mkdtemp(prefix="/tmp/zulip-export-")
if os.path.exists(output_dir):
shutil.rmtree(output_dir)
os.makedirs(output_dir)
print("Exporting realm %s" % (realm.domain,))
do_export_realm(realm, output_dir, threads=int(options['threads']))
print("Finished exporting to %s; tarring" % (output_dir,))
tarball_path = output_dir.rstrip('/') + '.tar.gz'
os.chdir(os.path.dirname(output_dir))
subprocess.check_call(["tar", "-czf", tarball_path, os.path.basename(output_dir)])
print("Tarball written to %s" % (tarball_path,))
