def SetEnvTempDir(path): if os.path.exists(path) and not os.path.isdir(path): raise Exception( 'The given temp directory, "{}", does not seem to be a directory!'. format(path)) try: HydrusPaths.MakeSureDirectoryExists(path) except Exception as e: raise Exception('Could not create the temp dir: {}'.format(e)) if not HydrusPaths.DirectoryIsWriteable(path): raise Exception( 'The given temp directory, "{}", does not seem to be writeable-to!' .format(path)) for tmp_name in ('TMPDIR', 'TEMP', 'TMP'): if tmp_name in os.environ: os.environ[tmp_name] = path tempfile.tempdir = path
def GetExportPath(): portable_path = HG.client_controller.options[ 'export_path' ] if portable_path is None: desired_path = os.path.join( '~', 'hydrus_export' ) path = os.path.expanduser( desired_path ) if path == desired_path: # could not figure it out, probably crazy user setup atm return None HydrusPaths.MakeSureDirectoryExists( path ) else: path = HydrusPaths.ConvertPortablePathToAbsPath( portable_path ) return path
def test_import_folders_daemon( self ): test_dir = HydrusTemp.GetTempDir() try: HG.test_controller.SetRead( 'hash_status', ClientImportFiles.FileImportStatus.STATICGetUnknownStatus() ) HydrusPaths.MakeSureDirectoryExists( test_dir ) hydrus_png_path = os.path.join( HC.STATIC_DIR, 'hydrus.png' ) HydrusPaths.MirrorFile( hydrus_png_path, os.path.join( test_dir, '0' ) ) HydrusPaths.MirrorFile( hydrus_png_path, os.path.join( test_dir, '1' ) ) # previously imported HydrusPaths.MirrorFile( hydrus_png_path, os.path.join( test_dir, '2' ) ) with open( os.path.join( test_dir, '3' ), 'wb' ) as f: f.write( b'blarg' ) # broken with open( os.path.join( test_dir, '4' ), 'wb' ) as f: f.write( b'blarg' ) # previously failed # actions = {} actions[ CC.STATUS_SUCCESSFUL_AND_NEW ] = CC.IMPORT_FOLDER_DELETE actions[ CC.STATUS_SUCCESSFUL_BUT_REDUNDANT ] = CC.IMPORT_FOLDER_DELETE actions[ CC.STATUS_DELETED ] = CC.IMPORT_FOLDER_DELETE actions[ CC.STATUS_ERROR ] = CC.IMPORT_FOLDER_IGNORE import_folder = ClientImportLocal.ImportFolder( 'imp', path = test_dir, actions = actions ) HG.test_controller.SetRead( 'serialisable_names', [ 'imp' ] ) HG.test_controller.SetRead( 'serialisable_named', import_folder ) HG.test_controller.ClearWrites( 'import_file' ) HG.test_controller.ClearWrites( 'serialisable' ) ClientDaemons.DAEMONCheckImportFolders() import_file = HG.test_controller.GetWrite( 'import_file' ) self.assertEqual( len( import_file ), 3 ) # I need to expand tests here with the new file system [ ( ( updated_import_folder, ), empty_dict ) ] = HG.test_controller.GetWrite( 'serialisable' ) self.assertEqual( updated_import_folder, import_folder ) self.assertTrue( not os.path.exists( os.path.join( test_dir, '0' ) ) ) self.assertTrue( not os.path.exists( os.path.join( test_dir, '1' ) ) ) self.assertTrue( not os.path.exists( os.path.join( test_dir, '2' ) ) ) self.assertTrue( os.path.exists( os.path.join( test_dir, '3' ) ) ) self.assertTrue( os.path.exists( os.path.join( test_dir, '4' ) ) ) finally: shutil.rmtree( test_dir )
def GetExportPath(): portable_path = HG.client_controller.options['export_path'] if portable_path is None: path = os.path.join(os.path.expanduser('~'), 'hydrus_export') HydrusPaths.MakeSureDirectoryExists(path) else: path = HydrusPaths.ConvertPortablePathToAbsPath(portable_path) return path
def _test_repo(self, service): service_key = service.GetServiceKey() # num_petitions num_petitions = [[ HC.CONTENT_TYPE_MAPPINGS, HC.CONTENT_STATUS_PETITIONED, 23 ], [HC.CONTENT_TYPE_TAG_PARENTS, HC.CONTENT_STATUS_PENDING, 0]] HG.test_controller.SetRead('num_petitions', num_petitions) response = service.Request(HC.GET, 'num_petitions') self.assertEqual(response['num_petitions'], num_petitions) # petition action = HC.CONTENT_UPDATE_PETITION petitioner_account = HydrusNetwork.Account.GenerateUnknownAccount() reason = 'it sucks' contents = [ HydrusNetwork.Content( HC.CONTENT_TYPE_FILES, [HydrusData.GenerateKey() for i in range(10)]) ] petition = HydrusNetwork.Petition(action, petitioner_account, reason, contents) HG.test_controller.SetRead('petition', petition) response = service.Request( HC.GET, 'petition', { 'content_type': HC.CONTENT_TYPE_FILES, 'status': HC.CONTENT_UPDATE_PETITION }) self.assertEqual(response['petition'].GetSerialisableTuple(), petition.GetSerialisableTuple()) # definitions definitions_update = HydrusNetwork.DefinitionsUpdate() for i in range(100, 200): definitions_update.AddRow( (HC.DEFINITIONS_TYPE_TAGS, i, 'series:test ' + str(i))) definitions_update.AddRow((HC.DEFINITIONS_TYPE_HASHES, i + 500, HydrusData.GenerateKey())) definitions_update_network_bytes = definitions_update.DumpToNetworkBytes( ) definitions_update_hash = hashlib.sha256( definitions_update_network_bytes).digest() path = ServerFiles.GetExpectedFilePath(definitions_update_hash) HydrusPaths.MakeSureDirectoryExists(path) with open(path, 'wb') as f: f.write(definitions_update_network_bytes) response = service.Request(HC.GET, 'update', {'update_hash': definitions_update_hash}) try: os.remove(path) except: pass self.assertEqual(response, definitions_update_network_bytes) # content rows = [(random.randint(100, 1000), [random.randint(100, 1000) for i in range(50)]) for j in range(20)] content_update = HydrusNetwork.ContentUpdate() for row in rows: content_update.AddRow( (HC.CONTENT_TYPE_MAPPINGS, HC.CONTENT_UPDATE_ADD, row)) content_update_network_bytes = content_update.DumpToNetworkBytes() content_update_hash = hashlib.sha256( content_update_network_bytes).digest() path = ServerFiles.GetExpectedFilePath(content_update_hash) with open(path, 'wb') as f: f.write(content_update_network_bytes) response = service.Request(HC.GET, 'update', {'update_hash': content_update_hash}) try: os.remove(path) except: pass self.assertEqual(response, content_update_network_bytes) # metadata metadata = HydrusNetwork.Metadata() metadata.AppendUpdate([definitions_update_hash, content_update_hash], HydrusData.GetNow() - 101000, HydrusData.GetNow() - 1000, HydrusData.GetNow() + 100000) service._metadata = metadata response = service.Request(HC.GET, 'metadata_slice', {'since': 0}) self.assertEqual(response['metadata_slice'].GetSerialisableTuple(), metadata.GetSerialisableTuple()) # post content raise NotImplementedError() '''
def _test_file_repo(self, service): # file path = ServerFiles.GetExpectedFilePath(self._file_hash) HydrusPaths.MakeSureDirectoryExists(os.path.dirname(path)) with open(path, 'wb') as f: f.write(EXAMPLE_FILE) response = service.Request(HC.GET, 'file', {'hash': self._file_hash}) self.assertEqual(response, EXAMPLE_FILE) # try: os.remove(path) except: pass path = os.path.join(HC.STATIC_DIR, 'hydrus.png') with open(path, 'rb') as f: file_bytes = f.read() HG.test_controller.ClearWrites('file') service.Request(HC.POST, 'file', {'file': file_bytes}) written = HG.test_controller.GetWrite('file') [(args, kwargs)] = written (written_service_key, written_account, written_file_dict) = args hash = b'\xadm5\x99\xa6\xc4\x89\xa5u\xeb\x19\xc0&\xfa\xce\x97\xa9\xcdey\xe7G(\xb0\xce\x94\xa6\x01\xd22\xf3\xc3' self.assertEqual(written_file_dict['hash'], hash) self.assertEqual(written_file_dict['ip'], '127.0.0.1') self.assertEqual(written_file_dict['height'], 200) self.assertEqual(written_file_dict['width'], 200) self.assertEqual(written_file_dict['mime'], 2) self.assertEqual(written_file_dict['size'], 5270) # ip (ip, timestamp) = ('94.45.87.123', HydrusData.GetNow() - 100000) HG.test_controller.SetRead('ip', (ip, timestamp)) response = service.Request(HC.GET, 'ip', {'hash': self._file_hash}) self.assertEqual(response['ip'], ip) self.assertEqual(response['timestamp'], timestamp) # account from hash subject_content = HydrusNetwork.Content( content_type=HC.CONTENT_TYPE_FILES, content_data=hash) subject_account_identifier = HydrusNetwork.AccountIdentifier( content=subject_content) HG.test_controller.SetRead('account', self._account) response = service.Request( HC.GET, 'other_account', {'subject_identifier': subject_account_identifier}) self.assertEqual(repr(response['account']), repr(self._account)) # thumbnail path = ServerFiles.GetExpectedThumbnailPath(self._file_hash) HydrusPaths.MakeSureDirectoryExists(os.path.dirname(path)) with open(path, 'wb') as f: f.write(EXAMPLE_THUMBNAIL) response = service.Request(HC.GET, 'thumbnail', {'hash': self._file_hash}) self.assertEqual(response, EXAMPLE_THUMBNAIL) try: os.remove(path) except: pass
db_dir = HC.DEFAULT_DB_DIR if not HydrusPaths.DirectoryIsWritable( db_dir) or HC.RUNNING_FROM_MACOS_APP: db_dir = HC.USERPATH_DB_DIR else: db_dir = result.db_dir db_dir = HydrusPaths.ConvertPortablePathToAbsPath(db_dir, HC.BASE_DIR) try: HydrusPaths.MakeSureDirectoryExists(db_dir) except: raise Exception( 'Could not ensure db path "{}" exists! Check the location is correct and that you have permission to write to it!' .format(db_dir)) if not os.path.isdir(db_dir): raise Exception( 'The given db path "{}" is not a directory!'.format(db_dir)) if not HydrusPaths.DirectoryIsWritable(db_dir): raise Exception(
def _DoExport( self ): query_hash_ids = HG.client_controller.Read( 'file_query_ids', self._file_search_context ) media_results = [] i = 0 base = 256 while i < len( query_hash_ids ): if HG.client_controller.new_options.GetBoolean( 'pause_export_folders_sync' ) or HydrusThreading.IsThreadShuttingDown(): return if i == 0: ( last_i, i ) = ( 0, base ) else: ( last_i, i ) = ( i, i + base ) sub_query_hash_ids = query_hash_ids[ last_i : i ] more_media_results = HG.client_controller.Read( 'media_results_from_ids', sub_query_hash_ids ) media_results.extend( more_media_results ) media_results.sort( key = lambda mr: mr.GetHashId() ) # terms = ParseExportPhrase( self._phrase ) previous_paths = set() for ( root, dirnames, filenames ) in os.walk( self._path ): previous_paths.update( ( os.path.join( root, filename ) for filename in filenames ) ) sync_paths = set() client_files_manager = HG.client_controller.client_files_manager num_copied = 0 for media_result in media_results: if HG.client_controller.new_options.GetBoolean( 'pause_export_folders_sync' ) or HydrusThreading.IsThreadShuttingDown(): return hash = media_result.GetHash() mime = media_result.GetMime() size = media_result.GetSize() try: source_path = client_files_manager.GetFilePath( hash, mime ) except HydrusExceptions.FileMissingException: raise Exception( 'A file to be exported, hash "{}", was missing! You should run file maintenance (under database->maintenance->files) to check the files for the export folder\'s search, and possibly all your files.' ) filename = GenerateExportFilename( self._path, media_result, terms ) dest_path = os.path.normpath( os.path.join( self._path, filename ) ) if not dest_path.startswith( self._path ): raise Exception( 'It seems a destination path for export folder "{}" was above the main export directory! The file was "{}" and its destination path was "{}".'.format( self._path, hash.hex(), dest_path ) ) dest_path_dir = os.path.dirname( dest_path ) HydrusPaths.MakeSureDirectoryExists( dest_path_dir ) if dest_path not in sync_paths: copied = HydrusPaths.MirrorFile( source_path, dest_path ) if copied: num_copied += 1 HydrusPaths.TryToGiveFileNicePermissionBits( dest_path ) sync_paths.add( dest_path ) if num_copied > 0: HydrusData.Print( 'Export folder ' + self._name + ' exported ' + HydrusData.ToHumanInt( num_copied ) + ' files.' ) if self._export_type == HC.EXPORT_FOLDER_TYPE_SYNCHRONISE: deletee_paths = previous_paths.difference( sync_paths ) for deletee_path in deletee_paths: ClientPaths.DeletePath( deletee_path ) deletee_dirs = set() for ( root, dirnames, filenames ) in os.walk( self._path, topdown = False ): if root == self._path: continue no_files = len( filenames ) == 0 useful_dirnames = [ dirname for dirname in dirnames if os.path.join( root, dirname ) not in deletee_dirs ] no_useful_dirs = len( useful_dirnames ) == 0 if no_useful_dirs and no_files: deletee_dirs.add( root ) for deletee_dir in deletee_dirs: if os.path.exists( deletee_dir ): HydrusPaths.DeletePath( deletee_dir ) if len( deletee_paths ) > 0: HydrusData.Print( 'Export folder {} deleted {} files and {} folders.'.format( self._name, HydrusData.ToHumanInt( len( deletee_paths ) ), HydrusData.ToHumanInt( len( deletee_dirs ) ) ) ) if self._delete_from_client_after_export: local_file_service_keys = HG.client_controller.services_manager.GetServiceKeys( ( HC.LOCAL_FILE_DOMAIN, ) ) service_keys_to_deletee_hashes = collections.defaultdict( list ) delete_lock_for_archived_files = HG.client_controller.new_options.GetBoolean( 'delete_lock_for_archived_files' ) for media_result in media_results: if delete_lock_for_archived_files and not media_result.GetInbox(): continue hash = media_result.GetHash() deletee_service_keys = media_result.GetLocationsManager().GetCurrent().intersection( local_file_service_keys ) for deletee_service_key in deletee_service_keys: service_keys_to_deletee_hashes[ deletee_service_key ].append( hash ) reason = 'Deleted after export to Export Folder "{}".'.format( self._path ) for ( service_key, deletee_hashes ) in service_keys_to_deletee_hashes.items(): chunks_of_hashes = HydrusData.SplitListIntoChunks( deletee_hashes, 64 ) for chunk_of_hashes in chunks_of_hashes: content_update = HydrusData.ContentUpdate( HC.CONTENT_TYPE_FILES, HC.CONTENT_UPDATE_DELETE, chunk_of_hashes, reason = reason ) HG.client_controller.WriteSynchronous( 'content_updates', { service_key : [ content_update ] } )
def __init__(self, win, only_run): self.app = win self.win = win self.only_run = only_run self.db_dir = tempfile.mkdtemp() global DB_DIR DB_DIR = self.db_dir self._server_files_dir = os.path.join(self.db_dir, 'server_files') self._updates_dir = os.path.join(self.db_dir, 'test_updates') client_files_default = os.path.join(self.db_dir, 'client_files') HydrusPaths.MakeSureDirectoryExists(self._server_files_dir) HydrusPaths.MakeSureDirectoryExists(self._updates_dir) HydrusPaths.MakeSureDirectoryExists(client_files_default) HG.controller = self HG.client_controller = self HG.server_controller = self HG.test_controller = self self.db = self self.gui = self self.frame_splash_status = ClientGUISplash.FrameSplashStatus() self._call_to_threads = [] self._pubsub = HydrusPubSub.HydrusPubSub(self, lambda o: True) self.new_options = ClientOptions.ClientOptions() HC.options = ClientDefaults.GetClientDefaultOptions() self.options = HC.options def show_text(text): pass HydrusData.ShowText = show_text self._reads = {} self._reads['local_booru_share_keys'] = [] self._reads['messaging_sessions'] = [] self._reads['options'] = ClientDefaults.GetClientDefaultOptions() self._reads['file_system_predicates'] = [] self._reads['media_results'] = [] self._param_reads = {} self.example_tag_repo_service_key = HydrusData.GenerateKey() services = [] services.append( ClientServices.GenerateService(CC.LOCAL_BOORU_SERVICE_KEY, HC.LOCAL_BOORU, 'local booru')) services.append( ClientServices.GenerateService(CC.CLIENT_API_SERVICE_KEY, HC.CLIENT_API_SERVICE, 'client api')) services.append( ClientServices.GenerateService(CC.COMBINED_LOCAL_FILE_SERVICE_KEY, HC.COMBINED_LOCAL_FILE, 'all local files')) services.append( ClientServices.GenerateService(CC.LOCAL_FILE_SERVICE_KEY, HC.LOCAL_FILE_DOMAIN, 'my files')) services.append( ClientServices.GenerateService(CC.TRASH_SERVICE_KEY, HC.LOCAL_FILE_TRASH_DOMAIN, 'trash')) services.append( ClientServices.GenerateService(CC.DEFAULT_LOCAL_TAG_SERVICE_KEY, HC.LOCAL_TAG, 'my tags')) services.append( ClientServices.GenerateService(self.example_tag_repo_service_key, HC.TAG_REPOSITORY, 'example tag repo')) services.append( ClientServices.GenerateService(CC.COMBINED_TAG_SERVICE_KEY, HC.COMBINED_TAG, 'all known tags')) services.append( ClientServices.GenerateService( LOCAL_RATING_LIKE_SERVICE_KEY, HC.LOCAL_RATING_LIKE, 'example local rating like service')) services.append( ClientServices.GenerateService( LOCAL_RATING_NUMERICAL_SERVICE_KEY, HC.LOCAL_RATING_NUMERICAL, 'example local rating numerical service')) self._reads['services'] = services client_files_locations = {} for prefix in HydrusData.IterateHexPrefixes(): for c in ('f', 't'): client_files_locations[c + prefix] = client_files_default self._reads['client_files_locations'] = client_files_locations self._reads['sessions'] = [] self._reads['tag_parents'] = {} self._reads['tag_siblings_all_ideals'] = {} self._reads['in_inbox'] = False self._writes = collections.defaultdict(list) self._managers = {} self.column_list_manager = ClientGUIListManager.ColumnListManager() self.services_manager = ClientServices.ServicesManager(self) self.client_files_manager = ClientFiles.ClientFilesManager(self) self.parsing_cache = ClientCaches.ParsingCache() bandwidth_manager = ClientNetworkingBandwidth.NetworkBandwidthManager() session_manager = ClientNetworkingSessions.NetworkSessionManager() domain_manager = ClientNetworkingDomain.NetworkDomainManager() ClientDefaults.SetDefaultDomainManagerData(domain_manager) login_manager = ClientNetworkingLogin.NetworkLoginManager() self.network_engine = ClientNetworking.NetworkEngine( self, bandwidth_manager, session_manager, domain_manager, login_manager) self.CallToThreadLongRunning(self.network_engine.MainLoop) self.tag_display_manager = ClientTagsHandling.TagDisplayManager() self._managers['undo'] = ClientManagers.UndoManager(self) self.server_session_manager = HydrusSessions.HydrusSessionManagerServer( ) self.bitmap_manager = ClientManagers.BitmapManager(self) self.local_booru_manager = ClientCaches.LocalBooruCache(self) self.client_api_manager = ClientAPI.APIManager() self._cookies = {} self._job_scheduler = HydrusThreading.JobScheduler(self) self._job_scheduler.start()
def _DoExport( self ): query_hash_ids = HG.client_controller.Read( 'file_query_ids', self._file_search_context ) media_results = [] i = 0 base = 256 while i < len( query_hash_ids ): if HC.options[ 'pause_export_folders_sync' ] or HydrusThreading.IsThreadShuttingDown(): return if i == 0: ( last_i, i ) = ( 0, base ) else: ( last_i, i ) = ( i, i + base ) sub_query_hash_ids = query_hash_ids[ last_i : i ] more_media_results = HG.client_controller.Read( 'media_results_from_ids', sub_query_hash_ids ) media_results.extend( more_media_results ) media_results.sort( key = lambda mr: mr.GetHashId() ) # terms = ParseExportPhrase( self._phrase ) previous_paths = set() for ( root, dirnames, filenames ) in os.walk( self._path ): previous_paths.update( ( os.path.join( root, filename ) for filename in filenames ) ) sync_paths = set() client_files_manager = HG.client_controller.client_files_manager num_copied = 0 for media_result in media_results: if HC.options[ 'pause_export_folders_sync' ] or HydrusThreading.IsThreadShuttingDown(): return hash = media_result.GetHash() mime = media_result.GetMime() size = media_result.GetSize() source_path = client_files_manager.GetFilePath( hash, mime ) filename = GenerateExportFilename( self._path, media_result, terms ) dest_path = os.path.normpath( os.path.join( self._path, filename ) ) if not dest_path.startswith( self._path ): raise Exception( 'It seems a destination path for export folder "{}" was above the main export directory! The file was "{}" and its destination path was "{}".'.format( self._path, hash.hex(), dest_path ) ) dest_path_dir = os.path.dirname( dest_path ) HydrusPaths.MakeSureDirectoryExists( dest_path_dir ) if dest_path not in sync_paths: copied = HydrusPaths.MirrorFile( source_path, dest_path ) if copied: num_copied += 1 HydrusPaths.MakeFileWritable( dest_path ) sync_paths.add( dest_path ) if num_copied > 0: HydrusData.Print( 'Export folder ' + self._name + ' exported ' + HydrusData.ToHumanInt( num_copied ) + ' files.' ) if self._export_type == HC.EXPORT_FOLDER_TYPE_SYNCHRONISE: deletee_paths = previous_paths.difference( sync_paths ) for deletee_path in deletee_paths: ClientPaths.DeletePath( deletee_path ) deletee_dirs = set() for ( root, dirnames, filenames ) in os.walk( self._path, topdown = False ): if root == self._path: continue no_files = len( filenames ) == 0 useful_dirnames = [ dirname for dirname in dirnames if os.path.join( root, dirname ) not in deletee_dirs ] no_useful_dirs = len( useful_dirnames ) == 0 if no_useful_dirs and no_files: deletee_dirs.add( root ) for deletee_dir in deletee_dirs: if os.path.exists( deletee_dir ): HydrusPaths.DeletePath( deletee_dir ) if len( deletee_paths ) > 0: HydrusData.Print( 'Export folder {} deleted {} files and {} folders.'.format( self._name, HydrusData.ToHumanInt( len( deletee_paths ) ), HydrusData.ToHumanInt( len( deletee_dirs ) ) ) ) if self._delete_from_client_after_export: deletee_hashes = { media_result.GetHash() for media_result in media_results } chunks_of_hashes = HydrusData.SplitListIntoChunks( deletee_hashes, 64 ) reason = 'Deleted after export to Export Folder "{}".'.format( self._path ) content_updates = [ HydrusData.ContentUpdate( HC.CONTENT_TYPE_FILES, HC.CONTENT_UPDATE_DELETE, chunk_of_hashes, reason = reason ) for chunk_of_hashes in chunks_of_hashes ] for content_update in content_updates: HG.client_controller.WriteSynchronous( 'content_updates', { CC.LOCAL_FILE_SERVICE_KEY : [ content_update ] } )
def do_it(directory, neighbouring_txt_tag_service_keys, delete_afterwards, export_symlinks, quit_afterwards): job_key = ClientThreading.JobKey(cancellable=True) job_key.SetStatusTitle('file export') HG.client_controller.pub('message', job_key) pauser = HydrusData.BigJobPauser() for (index, (ordering_index, media, path)) in enumerate(to_do): if job_key.IsCancelled(): break try: x_of_y = HydrusData.ConvertValueRangeToPrettyString( index + 1, num_to_do) job_key.SetVariable('popup_text_1', 'Done {}'.format(x_of_y)) job_key.SetVariable('popup_gauge_1', (index + 1, num_to_do)) QP.CallAfter(qt_update_label, x_of_y) hash = media.GetHash() mime = media.GetMime() path = os.path.normpath(path) if not path.startswith(directory): raise Exception( 'It seems a destination path was above the main export directory! The file was "{}" and its destination path was "{}".' .format(hash.hex(), path)) path_dir = os.path.dirname(path) HydrusPaths.MakeSureDirectoryExists(path_dir) if export_tag_txts: tags_manager = media.GetTagsManager() tags = set() for service_key in neighbouring_txt_tag_service_keys: current_tags = tags_manager.GetCurrent( service_key, ClientTags.TAG_DISPLAY_ACTUAL) tags.update(current_tags) tags = sorted(tags) txt_path = path + '.txt' with open(txt_path, 'w', encoding='utf-8') as f: f.write(os.linesep.join(tags)) source_path = client_files_manager.GetFilePath( hash, mime, check_file_exists=False) if export_symlinks: os.symlink(source_path, path) else: HydrusPaths.MirrorFile(source_path, path) HydrusPaths.MakeFileWriteable(path) except: QP.CallAfter( QW.QMessageBox.information, self, 'Information', 'Encountered a problem while attempting to export file with index ' + str(ordering_index + 1) + ':' + os.linesep * 2 + traceback.format_exc()) break pauser.Pause() if not job_key.IsCancelled() and delete_afterwards: QP.CallAfter(qt_update_label, 'deleting') delete_lock_for_archived_files = HG.client_controller.new_options.GetBoolean( 'delete_lock_for_archived_files') if delete_lock_for_archived_files: deletee_hashes = { media.GetHash() for (ordering_index, media, path) in to_do if not media.HasArchive() } else: deletee_hashes = { media.GetHash() for (ordering_index, media, path) in to_do } chunks_of_hashes = HydrusData.SplitListIntoChunks( deletee_hashes, 64) reason = 'Deleted after manual export to "{}".'.format( directory) content_updates = [ HydrusData.ContentUpdate(HC.CONTENT_TYPE_FILES, HC.CONTENT_UPDATE_DELETE, chunk_of_hashes, reason=reason) for chunk_of_hashes in chunks_of_hashes ] for content_update in content_updates: HG.client_controller.WriteSynchronous( 'content_updates', {CC.LOCAL_FILE_SERVICE_KEY: [content_update]}) job_key.DeleteVariable('popup_gauge_1') job_key.SetVariable('popup_text_1', 'Done!') job_key.Finish() job_key.Delete(5) QP.CallAfter(qt_update_label, 'done!') time.sleep(1) QP.CallAfter(qt_update_label, 'export') QP.CallAfter(qt_done, quit_afterwards)
def _DoExport(self, quit_afterwards=False): delete_afterwards = self._delete_files_after_export.isChecked() export_symlinks = self._export_symlinks.isChecked( ) and not delete_afterwards if quit_afterwards: message = 'Export as shown?' if delete_afterwards: message += os.linesep * 2 message += 'THE FILES WILL BE DELETED FROM THE CLIENT AFTERWARDS' result = ClientGUIDialogsQuick.GetYesNo(self, message) if result != QW.QDialog.Accepted: self.parentWidget().close() return elif delete_afterwards: message = 'THE FILES WILL BE DELETED FROM THE CLIENT AFTERWARDS' result = ClientGUIDialogsQuick.GetYesNo(self, message) if result != QW.QDialog.Accepted: return self._RefreshPaths() export_tag_txts = self._export_tag_txts.isChecked() if self._export_tag_txts.isChecked(): neighbouring_txt_tag_service_keys = self._neighbouring_txt_tag_service_keys else: neighbouring_txt_tag_service_keys = [] directory = self._directory_picker.GetPath() HydrusPaths.MakeSureDirectoryExists(directory) pattern = self._pattern.text() HG.client_controller.new_options.SetString('export_phrase', pattern) try: terms = ClientExporting.ParseExportPhrase(pattern) except Exception as e: QW.QMessageBox.critical(self, 'Error', str(e)) return client_files_manager = HG.client_controller.client_files_manager self._export.setEnabled(False) to_do = self._paths.GetData() to_do = [(ordering_index, media, self._GetPath(media)) for (ordering_index, media) in to_do] num_to_do = len(to_do) def qt_update_label(text): if not QP.isValid(self) or not QP.isValid( self._export) or not self._export: return self._export.setText(text) def qt_done(quit_afterwards): if not QP.isValid(self) or not QP.isValid( self._export) or not self._export: return self._export.setEnabled(True) if quit_afterwards: QP.CallAfter(self.parentWidget().close) def do_it(directory, neighbouring_txt_tag_service_keys, delete_afterwards, export_symlinks, quit_afterwards): job_key = ClientThreading.JobKey(cancellable=True) job_key.SetStatusTitle('file export') HG.client_controller.pub('message', job_key) pauser = HydrusData.BigJobPauser() for (index, (ordering_index, media, path)) in enumerate(to_do): if job_key.IsCancelled(): break try: x_of_y = HydrusData.ConvertValueRangeToPrettyString( index + 1, num_to_do) job_key.SetVariable('popup_text_1', 'Done {}'.format(x_of_y)) job_key.SetVariable('popup_gauge_1', (index + 1, num_to_do)) QP.CallAfter(qt_update_label, x_of_y) hash = media.GetHash() mime = media.GetMime() path = os.path.normpath(path) if not path.startswith(directory): raise Exception( 'It seems a destination path was above the main export directory! The file was "{}" and its destination path was "{}".' .format(hash.hex(), path)) path_dir = os.path.dirname(path) HydrusPaths.MakeSureDirectoryExists(path_dir) if export_tag_txts: tags_manager = media.GetTagsManager() tags = set() for service_key in neighbouring_txt_tag_service_keys: current_tags = tags_manager.GetCurrent( service_key, ClientTags.TAG_DISPLAY_ACTUAL) tags.update(current_tags) tags = sorted(tags) txt_path = path + '.txt' with open(txt_path, 'w', encoding='utf-8') as f: f.write(os.linesep.join(tags)) source_path = client_files_manager.GetFilePath( hash, mime, check_file_exists=False) if export_symlinks: os.symlink(source_path, path) else: HydrusPaths.MirrorFile(source_path, path) HydrusPaths.MakeFileWriteable(path) except: QP.CallAfter( QW.QMessageBox.information, self, 'Information', 'Encountered a problem while attempting to export file with index ' + str(ordering_index + 1) + ':' + os.linesep * 2 + traceback.format_exc()) break pauser.Pause() if not job_key.IsCancelled() and delete_afterwards: QP.CallAfter(qt_update_label, 'deleting') delete_lock_for_archived_files = HG.client_controller.new_options.GetBoolean( 'delete_lock_for_archived_files') if delete_lock_for_archived_files: deletee_hashes = { media.GetHash() for (ordering_index, media, path) in to_do if not media.HasArchive() } else: deletee_hashes = { media.GetHash() for (ordering_index, media, path) in to_do } chunks_of_hashes = HydrusData.SplitListIntoChunks( deletee_hashes, 64) reason = 'Deleted after manual export to "{}".'.format( directory) content_updates = [ HydrusData.ContentUpdate(HC.CONTENT_TYPE_FILES, HC.CONTENT_UPDATE_DELETE, chunk_of_hashes, reason=reason) for chunk_of_hashes in chunks_of_hashes ] for content_update in content_updates: HG.client_controller.WriteSynchronous( 'content_updates', {CC.LOCAL_FILE_SERVICE_KEY: [content_update]}) job_key.DeleteVariable('popup_gauge_1') job_key.SetVariable('popup_text_1', 'Done!') job_key.Finish() job_key.Delete(5) QP.CallAfter(qt_update_label, 'done!') time.sleep(1) QP.CallAfter(qt_update_label, 'export') QP.CallAfter(qt_done, quit_afterwards) HG.client_controller.CallToThread(do_it, directory, neighbouring_txt_tag_service_keys, delete_afterwards, export_symlinks, quit_afterwards)
def do_it(directory, neighbouring_txt_tag_service_keys, delete_afterwards, export_symlinks, quit_afterwards): pauser = HydrusData.BigJobPauser() for (index, (ordering_index, media)) in enumerate(to_do): try: QP.CallAfter( qt_update_label, HydrusData.ConvertValueRangeToPrettyString( index + 1, num_to_do)) hash = media.GetHash() mime = media.GetMime() path = self._GetPath(media) path = os.path.normpath(path) if not path.startswith(directory): raise Exception( 'It seems a destination path was above the main export directory! The file was "{}" and its destination path was "{}".' .format(hash.hex(), path)) path_dir = os.path.dirname(path) HydrusPaths.MakeSureDirectoryExists(path_dir) if export_tag_txts: tags_manager = media.GetTagsManager() tags = set() for service_key in neighbouring_txt_tag_service_keys: current_tags = tags_manager.GetCurrent( service_key, ClientTags.TAG_DISPLAY_SIBLINGS_AND_PARENTS) tags.update(current_tags) tags = sorted(tags) txt_path = path + '.txt' with open(txt_path, 'w', encoding='utf-8') as f: f.write(os.linesep.join(tags)) source_path = client_files_manager.GetFilePath( hash, mime, check_file_exists=False) if export_symlinks: os.symlink(source_path, path) else: HydrusPaths.MirrorFile(source_path, path) HydrusPaths.MakeFileWritable(path) except: QP.CallAfter( QW.QMessageBox.information, self, 'Information', 'Encountered a problem while attempting to export file with index ' + str(ordering_index + 1) + ':' + os.linesep * 2 + traceback.format_exc()) break pauser.Pause() if delete_afterwards: QP.CallAfter(qt_update_label, 'deleting') deletee_hashes = { media.GetHash() for (ordering_index, media) in to_do } chunks_of_hashes = HydrusData.SplitListIntoChunks( deletee_hashes, 64) reason = 'Deleted after manual export to "{}".'.format( directory) content_updates = [ HydrusData.ContentUpdate(HC.CONTENT_TYPE_FILES, HC.CONTENT_UPDATE_DELETE, chunk_of_hashes, reason=reason) for chunk_of_hashes in chunks_of_hashes ] for content_update in content_updates: HG.client_controller.WriteSynchronous( 'content_updates', {CC.LOCAL_FILE_SERVICE_KEY: [content_update]}) QP.CallAfter(qt_update_label, 'done!') time.sleep(1) QP.CallAfter(qt_update_label, 'export') QP.CallAfter(qt_done, quit_afterwards)