def _export_realm(self, realm, exportable_user_ids=None): # type: (Realm, Set[int]) -> Dict[str, Any] output_dir = self._make_output_dir() with patch('logging.info'), patch('zerver.lib.export.create_soft_link'): do_export_realm( realm=realm, output_dir=output_dir, threads=0, exportable_user_ids=exportable_user_ids, ) # TODO: Process the second partial file, which can be created # for certain edge cases. export_usermessages_batch( input_path=os.path.join(output_dir, 'messages-000001.json.partial'), output_path=os.path.join(output_dir, 'message.json') ) def read_file(fn): # type: (str) -> Any full_fn = os.path.join(output_dir, fn) with open(full_fn) as f: return ujson.load(f) result = {} result['realm'] = read_file('realm.json') result['attachment'] = read_file('attachment.json') result['message'] = read_file('message.json') result['uploads_dir'] = os.path.join(output_dir, 'uploads') return result
def _export_realm( self, realm: Realm, exportable_user_ids: Optional[Set[int]] = None, consent_message_id: Optional[int] = None) -> Dict[str, Any]: output_dir = self._make_output_dir() with patch('logging.info'), patch( 'zerver.lib.export.create_soft_link'): do_export_realm( realm=realm, output_dir=output_dir, threads=0, exportable_user_ids=exportable_user_ids, consent_message_id=consent_message_id, ) export_usermessages_batch( input_path=os.path.join(output_dir, 'messages-000001.json.partial'), output_path=os.path.join(output_dir, 'messages-000001.json'), consent_message_id=consent_message_id, ) try: export_usermessages_batch( input_path=os.path.join(output_dir, 'messages-000002.json.partial'), output_path=os.path.join(output_dir, 'messages-000002.json'), consent_message_id=consent_message_id, ) except FileNotFoundError: pass def read_file(fn: str) -> Any: full_fn = os.path.join(output_dir, fn) with open(full_fn) as f: return ujson.load(f) result = {} result['realm'] = read_file('realm.json') result['attachment'] = read_file('attachment.json') result['message'] = read_file('messages-000001.json') try: message = read_file('messages-000002.json') result["message"]["zerver_usermessage"].extend( message["zerver_usermessage"]) result["message"]["zerver_message"].extend( message["zerver_message"]) except FileNotFoundError: pass result['uploads_dir'] = os.path.join(output_dir, 'uploads') result['uploads_dir_records'] = read_file( os.path.join('uploads', 'records.json')) result['emoji_dir'] = os.path.join(output_dir, 'emoji') result['emoji_dir_records'] = read_file( os.path.join('emoji', 'records.json')) result['avatar_dir'] = os.path.join(output_dir, 'avatars') result['avatar_dir_records'] = read_file( os.path.join('avatars', 'records.json')) return result
def _export_realm( self, realm: Realm, exportable_user_ids: Optional[Set[int]] = None) -> Dict[str, Any]: output_dir = self._make_output_dir() with patch('logging.info'), patch( 'zerver.lib.export.create_soft_link'): do_export_realm( realm=realm, output_dir=output_dir, threads=0, exportable_user_ids=exportable_user_ids, ) # TODO: Process the second partial file, which can be created # for certain edge cases. export_usermessages_batch( input_path=os.path.join(output_dir, 'messages-000001.json.partial'), output_path=os.path.join(output_dir, 'message.json')) def read_file(fn: str) -> Any: full_fn = os.path.join(output_dir, fn) with open(full_fn) as f: return ujson.load(f) result = {} result['realm'] = read_file('realm.json') result['attachment'] = read_file('attachment.json') result['message'] = read_file('message.json') result['uploads_dir'] = os.path.join(output_dir, 'uploads') return result
def handle(self, *args: Any, **options: Any) -> None: logging.info("Starting UserMessage batch thread %s", options['thread']) files = set(glob.glob(os.path.join(options['path'], 'messages-*.json.partial'))) for partial_path in files: locked_path = partial_path.replace(".json.partial", ".json.locked") output_path = partial_path.replace(".json.partial", ".json") try: shutil.move(partial_path, locked_path) except Exception: # Already claimed by another process continue logging.info("Thread %s processing %s", options['thread'], output_path) try: export_usermessages_batch(locked_path, output_path, options["consent_message_id"]) except Exception: # Put the item back in the free pool when we fail shutil.move(locked_path, partial_path) raise
def handle(self, *args: Any, **options: Any) -> None: logging.info("Starting UserMessage batch thread %s" % (options['thread'],)) files = set(glob.glob(os.path.join(options['path'], 'messages-*.json.partial'))) for partial_path in files: locked_path = partial_path.replace(".json.partial", ".json.locked") output_path = partial_path.replace(".json.partial", ".json") try: shutil.move(partial_path, locked_path) except Exception: # Already claimed by another process continue logging.info("Thread %s processing %s" % (options['thread'], output_path)) try: export_usermessages_batch(locked_path, output_path) except Exception: # Put the item back in the free pool when we fail shutil.move(locked_path, partial_path) raise