def test_dict_to_content_updates( self ):
hash = HydrusData.GenerateKey()
hashes = { hash }
local_key = CC.LOCAL_TAG_SERVICE_KEY
remote_key = HydrusData.GenerateKey()
service_keys_to_tags = { local_key : { 'a' } }
content_updates = { local_key : [ HydrusData.ContentUpdate( HC.CONTENT_TYPE_MAPPINGS, HC.CONTENT_UPDATE_ADD, ( 'a', hashes ) ) ] }
self.assertEqual( ClientData.ConvertServiceKeysToTagsToServiceKeysToContentUpdates( { hash }, service_keys_to_tags ), content_updates )
service_keys_to_tags = { remote_key : { 'c' } }
content_updates = { remote_key : [ HydrusData.ContentUpdate( HC.CONTENT_TYPE_MAPPINGS, HC.CONTENT_UPDATE_PEND, ( 'c', hashes ) ) ] }
self.assertEqual( ClientData.ConvertServiceKeysToTagsToServiceKeysToContentUpdates( { hash }, service_keys_to_tags ), content_updates )
service_keys_to_tags = { local_key : [ 'a', 'character:b' ], remote_key : [ 'c', 'series:d' ] }
content_updates = {}
content_updates[ local_key ] = [ HydrusData.ContentUpdate( HC.CONTENT_TYPE_MAPPINGS, HC.CONTENT_UPDATE_ADD, ( 'a', hashes ) ), HydrusData.ContentUpdate( HC.CONTENT_TYPE_MAPPINGS, HC.CONTENT_UPDATE_ADD, ( 'character:b', hashes ) ) ]
content_updates[ remote_key ] = [ HydrusData.ContentUpdate( HC.CONTENT_TYPE_MAPPINGS, HC.CONTENT_UPDATE_PEND, ( 'c', hashes ) ), HydrusData.ContentUpdate( HC.CONTENT_TYPE_MAPPINGS, HC.CONTENT_UPDATE_PEND, ( 'series:d', hashes ) ) ]
self.assertEqual( HydrusData.ContentUpdate( HC.CONTENT_TYPE_MAPPINGS, HC.CONTENT_UPDATE_PEND, 'c' ), HydrusData.ContentUpdate( HC.CONTENT_TYPE_MAPPINGS, HC.CONTENT_UPDATE_PEND, 'c' ) )
self.assertEqual( ClientData.ConvertServiceKeysToTagsToServiceKeysToContentUpdates( { hash }, service_keys_to_tags ), content_updates )
def _WorkOnFiles( self, page_key ):
file_seed = self._file_seed_cache.GetNextFileSeed( CC.STATUS_UNKNOWN )
if file_seed is None:
return
did_substantial_work = False
path = file_seed.file_seed_data
with self._lock:
if path in self._paths_to_tags:
service_keys_to_tags = self._paths_to_tags[ path ]
else:
service_keys_to_tags = {}
with self._lock:
self._current_action = 'importing'
file_seed.ImportPath( self._file_seed_cache, self._file_import_options )
did_substantial_work = True
if file_seed.status in CC.SUCCESSFUL_IMPORT_STATES:
if file_seed.HasHash():
hash = file_seed.GetHash()
service_keys_to_content_updates = ClientData.ConvertServiceKeysToTagsToServiceKeysToContentUpdates( { hash }, service_keys_to_tags )
if len( service_keys_to_content_updates ) > 0:
HG.client_controller.WriteSynchronous( 'content_updates', service_keys_to_content_updates )
did_substantial_work = True
if file_seed.ShouldPresent( self._file_import_options ):
file_seed.PresentToPage( page_key )
did_substantial_work = True
if self._delete_after_success:
try:
ClientPaths.DeletePath( path )
except Exception as e:
HydrusData.ShowText( 'While attempting to delete ' + path + ', the following error occurred:' )
HydrusData.ShowException( e )
txt_path = path + '.txt'
if os.path.exists( txt_path ):
try:
ClientPaths.DeletePath( txt_path )
except Exception as e:
HydrusData.ShowText( 'While attempting to delete ' + txt_path + ', the following error occurred:' )
HydrusData.ShowException( e )
with self._lock:
self._current_action = ''
if did_substantial_work:
time.sleep( ClientImporting.DID_SUBSTANTIAL_FILE_WORK_MINIMUM_SLEEP_TIME )
def _ImportFiles( self, job_key ):
did_work = False
time_to_save = HydrusData.GetNow() + 600
num_files_imported = 0
presentation_hashes = []
presentation_hashes_fast = set()
i = 0
num_total = len( self._file_seed_cache )
num_total_unknown = self._file_seed_cache.GetFileSeedCount( CC.STATUS_UNKNOWN )
num_total_done = num_total - num_total_unknown
while True:
file_seed = self._file_seed_cache.GetNextFileSeed( CC.STATUS_UNKNOWN )
p1 = HC.options[ 'pause_import_folders_sync' ] or self._paused
p2 = HydrusThreading.IsThreadShuttingDown()
p3 = job_key.IsCancelled()
if file_seed is None or p1 or p2 or p3:
break
did_work = True
if HydrusData.TimeHasPassed( time_to_save ):
HG.client_controller.WriteSynchronous( 'serialisable', self )
time_to_save = HydrusData.GetNow() + 600
gauge_num_done = num_total_done + num_files_imported + 1
job_key.SetVariable( 'popup_text_1', 'importing file ' + HydrusData.ConvertValueRangeToPrettyString( gauge_num_done, num_total ) )
job_key.SetVariable( 'popup_gauge_1', ( gauge_num_done, num_total ) )
path = file_seed.file_seed_data
file_seed.ImportPath( self._file_seed_cache, self._file_import_options, limited_mimes = self._mimes )
if file_seed.status in CC.SUCCESSFUL_IMPORT_STATES:
if file_seed.HasHash():
hash = file_seed.GetHash()
if self._tag_import_options.HasAdditionalTags():
in_inbox = HG.client_controller.Read( 'in_inbox', hash )
downloaded_tags = []
service_keys_to_content_updates = self._tag_import_options.GetServiceKeysToContentUpdates( file_seed.status, in_inbox, hash, downloaded_tags ) # additional tags
if len( service_keys_to_content_updates ) > 0:
HG.client_controller.WriteSynchronous( 'content_updates', service_keys_to_content_updates )
service_keys_to_tags = {}
for ( tag_service_key, filename_tagging_options ) in self._tag_service_keys_to_filename_tagging_options.items():
if not HG.client_controller.services_manager.ServiceExists( tag_service_key ):
continue
try:
tags = filename_tagging_options.GetTags( tag_service_key, path )
if len( tags ) > 0:
service_keys_to_tags[ tag_service_key ] = tags
except Exception as e:
HydrusData.ShowText( 'Trying to parse filename tags in the import folder "' + self._name + '" threw an error!' )
HydrusData.ShowException( e )
if len( service_keys_to_tags ) > 0:
service_keys_to_content_updates = ClientData.ConvertServiceKeysToTagsToServiceKeysToContentUpdates( { hash }, service_keys_to_tags )
HG.client_controller.WriteSynchronous( 'content_updates', service_keys_to_content_updates )
num_files_imported += 1
if hash not in presentation_hashes_fast:
if file_seed.ShouldPresent( self._file_import_options ):
presentation_hashes.append( hash )
presentation_hashes_fast.add( hash )
elif file_seed.status == CC.STATUS_ERROR:
HydrusData.Print( 'A file failed to import from import folder ' + self._name + ':' + path )
i += 1
if i % 10 == 0:
self._ActionPaths()
if num_files_imported > 0:
HydrusData.Print( 'Import folder ' + self._name + ' imported ' + HydrusData.ToHumanInt( num_files_imported ) + ' files.' )
if len( presentation_hashes ) > 0:
ClientImporting.PublishPresentationHashes( self._name, presentation_hashes, self._publish_files_to_popup_button, self._publish_files_to_page )
self._ActionPaths()
return did_work