def DoDeferredPhysicalDeletes(self): num_files_deleted = 0 num_thumbnails_deleted = 0 pauser = HydrusData.BigJobPauser() (file_hash, thumbnail_hash) = self.Read('deferred_physical_delete') while (file_hash is not None or thumbnail_hash is not None) and not HG.view_shutdown: if file_hash is not None: path = ServerFiles.GetExpectedFilePath(file_hash) if os.path.exists(path): HydrusPaths.RecyclePath(path) num_files_deleted += 1 if thumbnail_hash is not None: path = ServerFiles.GetExpectedThumbnailPath(thumbnail_hash) if os.path.exists(path): HydrusPaths.RecyclePath(path) num_thumbnails_deleted += 1 self.WriteSynchronous('clear_deferred_physical_delete', file_hash=file_hash, thumbnail_hash=thumbnail_hash) (file_hash, thumbnail_hash) = self.Read('deferred_physical_delete') pauser.Pause() if num_files_deleted > 0 or num_thumbnails_deleted > 0: HydrusData.Print( 'Physically deleted {} files and {} thumbnails from file storage.' .format(HydrusData.ToHumanInt(num_files_deleted), HydrusData.ToHumanInt(num_files_deleted)))
def CopyAndMergeTree(source, dest): pauser = HydrusData.BigJobPauser() MakeSureDirectoryExists(dest) num_errors = 0 for (root, dirnames, filenames) in os.walk(source): dest_root = root.replace(source, dest) for dirname in dirnames: pauser.Pause() source_path = os.path.join(root, dirname) dest_path = os.path.join(dest_root, dirname) MakeSureDirectoryExists(dest_path) shutil.copystat(source_path, dest_path) for filename in filenames: if num_errors > 5: raise Exception('Too many errors, directory copy abandoned.') pauser.Pause() source_path = os.path.join(root, filename) dest_path = os.path.join(dest_root, filename) ok = MirrorFile(source_path, dest_path) if not ok: num_errors += 1
def do_it(directory, neighbouring_txt_tag_service_keys, delete_afterwards, export_symlinks, quit_afterwards): job_key = ClientThreading.JobKey(cancellable=True) job_key.SetStatusTitle('file export') HG.client_controller.pub('message', job_key) pauser = HydrusData.BigJobPauser() for (index, (ordering_index, media, path)) in enumerate(to_do): if job_key.IsCancelled(): break try: x_of_y = HydrusData.ConvertValueRangeToPrettyString( index + 1, num_to_do) job_key.SetVariable('popup_text_1', 'Done {}'.format(x_of_y)) job_key.SetVariable('popup_gauge_1', (index + 1, num_to_do)) QP.CallAfter(qt_update_label, x_of_y) hash = media.GetHash() mime = media.GetMime() path = os.path.normpath(path) if not path.startswith(directory): raise Exception( 'It seems a destination path was above the main export directory! The file was "{}" and its destination path was "{}".' .format(hash.hex(), path)) path_dir = os.path.dirname(path) HydrusPaths.MakeSureDirectoryExists(path_dir) if export_tag_txts: tags_manager = media.GetTagsManager() tags = set() for service_key in neighbouring_txt_tag_service_keys: current_tags = tags_manager.GetCurrent( service_key, ClientTags.TAG_DISPLAY_ACTUAL) tags.update(current_tags) tags = sorted(tags) txt_path = path + '.txt' with open(txt_path, 'w', encoding='utf-8') as f: f.write(os.linesep.join(tags)) source_path = client_files_manager.GetFilePath( hash, mime, check_file_exists=False) if export_symlinks: os.symlink(source_path, path) else: HydrusPaths.MirrorFile(source_path, path) HydrusPaths.MakeFileWriteable(path) except: QP.CallAfter( QW.QMessageBox.information, self, 'Information', 'Encountered a problem while attempting to export file with index ' + str(ordering_index + 1) + ':' + os.linesep * 2 + traceback.format_exc()) break pauser.Pause() if not job_key.IsCancelled() and delete_afterwards: QP.CallAfter(qt_update_label, 'deleting') delete_lock_for_archived_files = HG.client_controller.new_options.GetBoolean( 'delete_lock_for_archived_files') if delete_lock_for_archived_files: deletee_hashes = { media.GetHash() for (ordering_index, media, path) in to_do if not media.HasArchive() } else: deletee_hashes = { media.GetHash() for (ordering_index, media, path) in to_do } chunks_of_hashes = HydrusData.SplitListIntoChunks( deletee_hashes, 64) reason = 'Deleted after manual export to "{}".'.format( directory) content_updates = [ HydrusData.ContentUpdate(HC.CONTENT_TYPE_FILES, HC.CONTENT_UPDATE_DELETE, chunk_of_hashes, reason=reason) for chunk_of_hashes in chunks_of_hashes ] for content_update in content_updates: HG.client_controller.WriteSynchronous( 'content_updates', {CC.LOCAL_FILE_SERVICE_KEY: [content_update]}) job_key.DeleteVariable('popup_gauge_1') job_key.SetVariable('popup_text_1', 'Done!') job_key.Finish() job_key.Delete(5) QP.CallAfter(qt_update_label, 'done!') time.sleep(1) QP.CallAfter(qt_update_label, 'export') QP.CallAfter(qt_done, quit_afterwards)
def MirrorTree(source, dest, text_update_hook=None, is_cancelled_hook=None): pauser = HydrusData.BigJobPauser() MakeSureDirectoryExists(dest) num_errors = 0 for (root, dirnames, filenames) in os.walk(source): if is_cancelled_hook is not None and is_cancelled_hook(): return if text_update_hook is not None: text_update_hook('Copying ' + root + '.') dest_root = root.replace(source, dest) surplus_dest_paths = { os.path.join(dest_root, dest_filename) for dest_filename in os.listdir(dest_root) } for dirname in dirnames: pauser.Pause() source_path = os.path.join(root, dirname) dest_path = os.path.join(dest_root, dirname) surplus_dest_paths.discard(dest_path) MakeSureDirectoryExists(dest_path) shutil.copystat(source_path, dest_path) for filename in filenames: if num_errors > 5: raise Exception('Too many errors, directory copy abandoned.') pauser.Pause() source_path = os.path.join(root, filename) dest_path = os.path.join(dest_root, filename) surplus_dest_paths.discard(dest_path) ok = MirrorFile(source_path, dest_path) if not ok: num_errors += 1 for dest_path in surplus_dest_paths: pauser.Pause() DeletePath(dest_path)
def MergeTree(source, dest, text_update_hook=None): pauser = HydrusData.BigJobPauser() if not os.path.exists(dest): try: shutil.move(source, dest) except OSError: # if there were read only files in source and this was partition to partition, the copy2 goes ok but the subsequent source unlink fails # so, if it seems this has happened, let's just try a walking mergetree, which should be able to deal with these readonlies on a file-by-file basis if os.path.exists(dest): MergeTree(source, dest, text_update_hook=text_update_hook) else: if len(os.listdir(dest)) == 0: for filename in os.listdir(source): source_path = os.path.join(source, filename) dest_path = os.path.join(dest, filename) if not os.path.isdir(source_path): MakeFileWritable(source_path) shutil.move(source_path, dest_path) else: num_errors = 0 for (root, dirnames, filenames) in os.walk(source): if text_update_hook is not None: text_update_hook('Copying ' + root + '.') dest_root = root.replace(source, dest) for dirname in dirnames: pauser.Pause() source_path = os.path.join(root, dirname) dest_path = os.path.join(dest_root, dirname) MakeSureDirectoryExists(dest_path) shutil.copystat(source_path, dest_path) for filename in filenames: if num_errors > 5: raise Exception( 'Too many errors, directory move abandoned.') pauser.Pause() source_path = os.path.join(root, filename) dest_path = os.path.join(dest_root, filename) ok = MergeFile(source_path, dest_path) if not ok: num_errors += 1 if num_errors == 0: DeletePath(source)
def work_callable(): job_key = ClientThreading.JobKey( cancellable = True ) title = 'moving files' if action == HC.CONTENT_UPDATE_MOVE else 'adding files' job_key.SetStatusTitle( title ) BLOCK_SIZE = 64 if len( applicable_media ) > BLOCK_SIZE: HG.client_controller.pub( 'message', job_key ) pauser = HydrusData.BigJobPauser() num_to_do = len( applicable_media ) now = HydrusData.GetNow() for ( i, block_of_media ) in enumerate( HydrusData.SplitListIntoChunks( applicable_media, BLOCK_SIZE ) ): if job_key.IsCancelled(): break job_key.SetVariable( 'popup_text_1', HydrusData.ConvertValueRangeToPrettyString( i * BLOCK_SIZE, num_to_do ) ) job_key.SetVariable( 'popup_gauge_1', ( i * BLOCK_SIZE, num_to_do ) ) content_updates = [] undelete_hashes = set() for m in block_of_media: if dest_service_key in m.GetLocationsManager().GetDeleted(): undelete_hashes.add( m.GetHash() ) else: content_updates.append( HydrusData.ContentUpdate( HC.CONTENT_TYPE_FILES, HC.CONTENT_UPDATE_ADD, ( m.GetMediaResult().GetFileInfoManager(), now ) ) ) if len( undelete_hashes ) > 0: content_updates.append( HydrusData.ContentUpdate( HC.CONTENT_TYPE_FILES, HC.CONTENT_UPDATE_UNDELETE, undelete_hashes ) ) HG.client_controller.WriteSynchronous( 'content_updates', { dest_service_key : content_updates } ) if action == HC.CONTENT_UPDATE_MOVE: block_of_hashes = [ m.GetHash() for m in block_of_media ] content_updates = [ HydrusData.ContentUpdate( HC.CONTENT_TYPE_FILES, HC.CONTENT_UPDATE_DELETE, block_of_hashes, reason = 'Moved to {}'.format( dest_service_name ) ) ] HG.client_controller.WriteSynchronous( 'content_updates', { source_service_key : content_updates } ) pauser.Pause() job_key.Delete()
def do_it(directory, neighbouring_txt_tag_service_keys, delete_afterwards, export_symlinks, quit_afterwards): pauser = HydrusData.BigJobPauser() for (index, (ordering_index, media)) in enumerate(to_do): try: QP.CallAfter( qt_update_label, HydrusData.ConvertValueRangeToPrettyString( index + 1, num_to_do)) hash = media.GetHash() mime = media.GetMime() path = self._GetPath(media) path = os.path.normpath(path) if not path.startswith(directory): raise Exception( 'It seems a destination path was above the main export directory! The file was "{}" and its destination path was "{}".' .format(hash.hex(), path)) path_dir = os.path.dirname(path) HydrusPaths.MakeSureDirectoryExists(path_dir) if export_tag_txts: tags_manager = media.GetTagsManager() tags = set() for service_key in neighbouring_txt_tag_service_keys: current_tags = tags_manager.GetCurrent( service_key, ClientTags.TAG_DISPLAY_SIBLINGS_AND_PARENTS) tags.update(current_tags) tags = sorted(tags) txt_path = path + '.txt' with open(txt_path, 'w', encoding='utf-8') as f: f.write(os.linesep.join(tags)) source_path = client_files_manager.GetFilePath( hash, mime, check_file_exists=False) if export_symlinks: os.symlink(source_path, path) else: HydrusPaths.MirrorFile(source_path, path) HydrusPaths.MakeFileWritable(path) except: QP.CallAfter( QW.QMessageBox.information, self, 'Information', 'Encountered a problem while attempting to export file with index ' + str(ordering_index + 1) + ':' + os.linesep * 2 + traceback.format_exc()) break pauser.Pause() if delete_afterwards: QP.CallAfter(qt_update_label, 'deleting') deletee_hashes = { media.GetHash() for (ordering_index, media) in to_do } chunks_of_hashes = HydrusData.SplitListIntoChunks( deletee_hashes, 64) reason = 'Deleted after manual export to "{}".'.format( directory) content_updates = [ HydrusData.ContentUpdate(HC.CONTENT_TYPE_FILES, HC.CONTENT_UPDATE_DELETE, chunk_of_hashes, reason=reason) for chunk_of_hashes in chunks_of_hashes ] for content_update in content_updates: HG.client_controller.WriteSynchronous( 'content_updates', {CC.LOCAL_FILE_SERVICE_KEY: [content_update]}) QP.CallAfter(qt_update_label, 'done!') time.sleep(1) QP.CallAfter(qt_update_label, 'export') QP.CallAfter(qt_done, quit_afterwards)