def _export_realm(self, realm, exportable_user_ids=None):
# type: (Realm, Set[int]) -> Dict[str, Any]
output_dir = self._make_output_dir()
with patch('logging.info'), patch('zerver.lib.export.create_soft_link'):
do_export_realm(
realm=realm,
output_dir=output_dir,
threads=0,
exportable_user_ids=exportable_user_ids,
)
# TODO: Process the second partial file, which can be created
# for certain edge cases.
export_usermessages_batch(
input_path=os.path.join(output_dir, 'messages-000001.json.partial'),
output_path=os.path.join(output_dir, 'message.json')
)
def read_file(fn):
# type: (str) -> Any
full_fn = os.path.join(output_dir, fn)
with open(full_fn) as f:
return ujson.load(f)
result = {}
result['realm'] = read_file('realm.json')
result['attachment'] = read_file('attachment.json')
result['message'] = read_file('message.json')
result['uploads_dir'] = os.path.join(output_dir, 'uploads')
return result
def _export_realm(
self,
realm: Realm,
exportable_user_ids: Optional[Set[int]] = None,
consent_message_id: Optional[int] = None) -> Dict[str, Any]:
output_dir = self._make_output_dir()
with patch('logging.info'), patch(
'zerver.lib.export.create_soft_link'):
do_export_realm(
realm=realm,
output_dir=output_dir,
threads=0,
exportable_user_ids=exportable_user_ids,
consent_message_id=consent_message_id,
)
export_usermessages_batch(
input_path=os.path.join(output_dir,
'messages-000001.json.partial'),
output_path=os.path.join(output_dir, 'messages-000001.json'),
consent_message_id=consent_message_id,
)
try:
export_usermessages_batch(
input_path=os.path.join(output_dir,
'messages-000002.json.partial'),
output_path=os.path.join(output_dir,
'messages-000002.json'),
consent_message_id=consent_message_id,
)
except FileNotFoundError:
pass
def read_file(fn: str) -> Any:
full_fn = os.path.join(output_dir, fn)
with open(full_fn) as f:
return ujson.load(f)
result = {}
result['realm'] = read_file('realm.json')
result['attachment'] = read_file('attachment.json')
result['message'] = read_file('messages-000001.json')
try:
message = read_file('messages-000002.json')
result["message"]["zerver_usermessage"].extend(
message["zerver_usermessage"])
result["message"]["zerver_message"].extend(
message["zerver_message"])
except FileNotFoundError:
pass
result['uploads_dir'] = os.path.join(output_dir, 'uploads')
result['uploads_dir_records'] = read_file(
os.path.join('uploads', 'records.json'))
result['emoji_dir'] = os.path.join(output_dir, 'emoji')
result['emoji_dir_records'] = read_file(
os.path.join('emoji', 'records.json'))
result['avatar_dir'] = os.path.join(output_dir, 'avatars')
result['avatar_dir_records'] = read_file(
os.path.join('avatars', 'records.json'))
return result
def _export_realm(
self,
realm: Realm,
exportable_user_ids: Optional[Set[int]] = None) -> Dict[str, Any]:
output_dir = self._make_output_dir()
with patch('logging.info'), patch(
'zerver.lib.export.create_soft_link'):
do_export_realm(
realm=realm,
output_dir=output_dir,
threads=0,
exportable_user_ids=exportable_user_ids,
)
# TODO: Process the second partial file, which can be created
# for certain edge cases.
export_usermessages_batch(
input_path=os.path.join(output_dir,
'messages-000001.json.partial'),
output_path=os.path.join(output_dir, 'message.json'))
def read_file(fn: str) -> Any:
full_fn = os.path.join(output_dir, fn)
with open(full_fn) as f:
return ujson.load(f)
result = {}
result['realm'] = read_file('realm.json')
result['attachment'] = read_file('attachment.json')
result['message'] = read_file('message.json')
result['uploads_dir'] = os.path.join(output_dir, 'uploads')
return result
def handle(self, *args: Any, **options: Any) -> None:
logging.info("Starting UserMessage batch thread %s", options['thread'])
files = set(glob.glob(os.path.join(options['path'], 'messages-*.json.partial')))
for partial_path in files:
locked_path = partial_path.replace(".json.partial", ".json.locked")
output_path = partial_path.replace(".json.partial", ".json")
try:
shutil.move(partial_path, locked_path)
except Exception:
# Already claimed by another process
continue
logging.info("Thread %s processing %s", options['thread'], output_path)
try:
export_usermessages_batch(locked_path, output_path, options["consent_message_id"])
except Exception:
# Put the item back in the free pool when we fail
shutil.move(locked_path, partial_path)
raise
def handle(self, *args: Any, **options: Any) -> None:
logging.info("Starting UserMessage batch thread %s" % (options['thread'],))
files = set(glob.glob(os.path.join(options['path'], 'messages-*.json.partial')))
for partial_path in files:
locked_path = partial_path.replace(".json.partial", ".json.locked")
output_path = partial_path.replace(".json.partial", ".json")
try:
shutil.move(partial_path, locked_path)
except Exception:
# Already claimed by another process
continue
logging.info("Thread %s processing %s" % (options['thread'], output_path))
try:
export_usermessages_batch(locked_path, output_path)
except Exception:
# Put the item back in the free pool when we fail
shutil.move(locked_path, partial_path)
raise
