def GetPrettyJobSummary(self):
with self._waiting_lock:
num_jobs = len(self._waiting)
job_lines = [repr(job) for job in self._waiting]
lines = [HydrusData.ToHumanInt(num_jobs) + ' jobs:'] + job_lines
text = os.linesep.join(lines)
return text
def _ImportURLs( self, urls ):
gallery_seed_log = self._gallery_seed_log_get_callable()
filtered_urls = [ url for url in urls if not gallery_seed_log.HasGalleryURL( url ) ]
urls_to_add = urls
if len( filtered_urls ) < len( urls ):
num_urls = len( urls )
num_removed = num_urls - len( filtered_urls )
message = 'Of the ' + HydrusData.ToHumanInt( num_urls ) + ' URLs you mean to add, ' + HydrusData.ToHumanInt( num_removed ) + ' are already in the gallery log. Would you like to only add new URLs or add everything (which will force a re-check of the duplicates)?'
( result, was_cancelled ) = ClientGUIDialogsQuick.GetYesNo( self, message, yes_label = 'only add new urls', no_label = 'add all urls, even duplicates', check_for_cancelled = True )
if was_cancelled:
return
if result == QW.QDialog.Accepted:
urls_to_add = filtered_urls
elif result == QW.QDialog.Rejected:
return
can_generate_more_pages = False
if self._can_generate_more_pages:
message = 'Would you like these urls to only check for new files, or would you like them to also generate subsequent gallery pages, like a regular search would?'
( result, was_cancelled ) = ClientGUIDialogsQuick.GetYesNo( self, message, yes_label = 'just check what I am adding', no_label = 'start a potential new search for every url added', check_for_cancelled = True )
if was_cancelled:
return
can_generate_more_pages = result == QW.QDialog.Rejected
gallery_seeds = [ ClientImportGallerySeeds.GallerySeed( url, can_generate_more_pages = can_generate_more_pages ) for url in urls_to_add ]
gallery_seed_log.AddGallerySeeds( gallery_seeds )
def ImportFromDragDrop(self, paths):
from hydrus.client.gui import ClientGUIDialogsQuick
message = 'Try to import the ' + HydrusData.ToHumanInt(len(
paths)) + ' dropped files to this list? I am expecting png files.'
result = ClientGUIDialogsQuick.GetYesNo(self, message)
if result == QW.QDialog.Accepted:
self._ImportPngs(paths)
self._listctrl.Sort()
def GetSummary( self ):
if self._never_faster_than == self._never_slower_than:
timing_statement = 'Checking every ' + HydrusData.TimeDeltaToPrettyTimeDelta( self._never_faster_than ) + '.'
else:
timing_statement = 'Trying to get ' + HydrusData.ToHumanInt( self._intended_files_per_check ) + ' files per check, never faster than ' + HydrusData.TimeDeltaToPrettyTimeDelta( self._never_faster_than ) + ' and never slower than ' + HydrusData.TimeDeltaToPrettyTimeDelta( self._never_slower_than ) + '.'
( death_files_found, death_time_delta ) = self._death_file_velocity
if death_files_found == 0:
death_statement = 'Never stopping.'
else:
death_statement = 'Stopping if file velocity falls below ' + HydrusData.ToHumanInt( death_files_found ) + ' files per ' + HydrusData.TimeDeltaToPrettyTimeDelta( death_time_delta ) + '.'
return timing_statement + os.linesep * 2 + death_statement
def _DeleteSelected(self):
file_seeds_to_delete = self._list_ctrl.GetData(only_selected=True)
if len(file_seeds_to_delete) > 0:
message = 'Are you sure you want to delete the {} selected entries?'.format(
HydrusData.ToHumanInt(len(file_seeds_to_delete)))
result = ClientGUIDialogsQuick.GetYesNo(self, message)
if result == QW.QDialog.Accepted:
self._file_seed_cache.RemoveFileSeeds(file_seeds_to_delete)
def RegenerateTree(self):
job_key = ClientThreading.JobKey()
try:
job_key.SetStatusTitle('regenerating similar file search data')
HG.client_controller.pub('modal_message', job_key)
job_key.SetVariable('popup_text_1',
'purging search info of orphans')
(current_files_table_name, deleted_files_table_name,
pending_files_table_name, petitioned_files_table_name
) = ClientDBFilesStorage.GenerateFilesTableNames(
self.modules_services.combined_local_file_service_id)
self._Execute(
'DELETE FROM shape_perceptual_hash_map WHERE hash_id NOT IN ( SELECT hash_id FROM {} );'
.format(current_files_table_name))
job_key.SetVariable('popup_text_1', 'gathering all leaves')
self._Execute('DELETE FROM shape_vptree;')
all_nodes = self._Execute(
'SELECT phash_id, phash FROM shape_perceptual_hashes;'
).fetchall()
job_key.SetVariable(
'popup_text_1',
HydrusData.ToHumanInt(len(all_nodes)) +
' leaves found, now regenerating')
(root_id, root_perceptual_hash) = self._PopBestRootNode(
all_nodes) #HydrusData.RandomPop( all_nodes )
self._GenerateBranch(job_key, None, root_id, root_perceptual_hash,
all_nodes)
finally:
job_key.SetVariable('popup_text_1', 'done!')
job_key.DeleteVariable('popup_text_2')
job_key.Finish()
job_key.Delete(5)
def ToString(self, name_method):
# this probably needs some params for 'short string' and stuff later on
if self.IsEmpty():
return 'nothing'
if self.IncludesCurrent() and self.IncludesDeleted():
if self.current_service_keys == self.deleted_service_keys:
prefix = 'current and deleted files of '
else:
prefix = 'a mix of current and deleted files of '
elif self.IncludesDeleted():
prefix = 'deleted files of '
else:
prefix = ''
if self.current_service_keys == self.deleted_service_keys:
service_keys_to_consider = self.current_service_keys
else:
service_keys_to_consider = self.current_service_keys.union(
self.deleted_service_keys)
if len(service_keys_to_consider) <= 2:
service_strings = sorted(
(name_method(service_key)
for service_key in service_keys_to_consider))
service_string = ', '.join(service_strings)
else:
service_string = '{} services'.format(
HydrusData.ToHumanInt(len(service_keys_to_consider)))
return prefix + service_string
def _DeleteSelected(self):
gallery_seeds_to_delete = self._list_ctrl.GetData(only_selected=True)
if len(gallery_seeds_to_delete) > 0:
message = 'Are you sure you want to delete the {} selected entries? This is only useful if you have a really really huge list.'.format(
HydrusData.ToHumanInt(len(gallery_seeds_to_delete)))
result = ClientGUIDialogsQuick.GetYesNo(self, message)
if result == QW.QDialog.Accepted:
self._gallery_seed_log.RemoveGallerySeeds(
gallery_seeds_to_delete)
def _UpdateAutoCreationHistoryText( self ):
( accounts_per_time_delta, time_delta ) = self._auto_create_velocity_control.GetValue()
text = ''
if accounts_per_time_delta == 0:
text = 'Auto-creation disabled. '
num_created = self._auto_create_history.GetUsage( HC.BANDWIDTH_TYPE_DATA, time_delta )
text += '{} auto-created in the past {}.'.format( HydrusData.ToHumanInt( num_created ), HydrusData.TimeDeltaToPrettyTimeDelta( time_delta ) )
self._auto_create_history_st.setText( text )
def ConvertServiceKeysToContentUpdatesToPrettyString( service_keys_to_content_updates ):
num_files = 0
actions = set()
locations = set()
extra_words = ''
for ( service_key, content_updates ) in list(service_keys_to_content_updates.items()):
if len( content_updates ) > 0:
name = HG.client_controller.services_manager.GetName( service_key )
locations.add( name )
for content_update in content_updates:
( data_type, action, row ) = content_update.ToTuple()
if data_type == HC.CONTENT_TYPE_MAPPINGS:
extra_words = ' tags for'
actions.add( HC.content_update_string_lookup[ action ] )
if action in ( HC.CONTENT_UPDATE_ARCHIVE, HC.CONTENT_UPDATE_INBOX ):
locations = set()
num_files += len( content_update.GetHashes() )
s = ''
if len( locations ) > 0:
s += ', '.join( locations ) + '->'
s += ', '.join( actions ) + extra_words + ' ' + HydrusData.ToHumanInt( num_files ) + ' files'
return s
def _ConvertFileSeedToListCtrlTuples(self, file_seed):
try:
file_seed_index = self._file_seed_cache.GetFileSeedIndex(file_seed)
pretty_file_seed_index = HydrusData.ToHumanInt(file_seed_index)
except:
file_seed_index = '--'
pretty_file_seed_index = '--'
file_seed_data = file_seed.file_seed_data
status = file_seed.status
added = file_seed.created
modified = file_seed.modified
source_time = file_seed.source_time
note = file_seed.note
pretty_file_seed_data = str(file_seed_data)
pretty_status = CC.status_string_lookup[status]
pretty_added = ClientData.TimestampToPrettyTimeDelta(added)
pretty_modified = ClientData.TimestampToPrettyTimeDelta(modified)
if source_time is None:
pretty_source_time = 'unknown'
else:
pretty_source_time = ClientData.TimestampToPrettyTimeDelta(
source_time)
sort_source_time = ClientGUIListCtrl.SafeNoneInt(source_time)
pretty_note = note.split(os.linesep)[0]
display_tuple = (pretty_file_seed_index, pretty_file_seed_data,
pretty_status, pretty_added, pretty_modified,
pretty_source_time, pretty_note)
sort_tuple = (file_seed_index, file_seed_data, status, added, modified,
sort_source_time, note)
return (display_tuple, sort_tuple)
def GetRowsOfPresentationTextsWithNamespaces(
self, render_for_user: bool, sibling_decoration_allowed: bool,
child_rows_allowed: bool
) -> typing.List[typing.List[typing.Tuple[str, str]]]:
rows_of_texts_and_namespaces = []
first_row_of_texts_and_namespaces = self._predicate.GetTextsAndNamespaces(
render_for_user,
or_under_construction=self._i_am_an_or_under_construction)
if sibling_decoration_allowed and self._predicate.HasIdealSibling():
ideal_sibling = self._predicate.GetIdealSibling()
(ideal_namespace,
ideal_subtag) = HydrusTags.SplitTag(ideal_sibling)
ideal_text = ' (displays as {})'.format(
ClientTags.RenderTag(ideal_sibling, render_for_user))
first_row_of_texts_and_namespaces.append(
(ideal_text, ideal_namespace))
rows_of_texts_and_namespaces.append(first_row_of_texts_and_namespaces)
parent_preds = self._predicate.GetParentPredicates()
if len(parent_preds) > 0:
if child_rows_allowed:
for parent_pred in self._predicate.GetParentPredicates():
rows_of_texts_and_namespaces.append(
parent_pred.GetTextsAndNamespaces(render_for_user))
elif sibling_decoration_allowed:
parents_text = ' ({} parents)'.format(
HydrusData.ToHumanInt(len(parent_preds)))
first_row_of_texts_and_namespaces.append((parents_text, ''))
return rows_of_texts_and_namespaces
def ImportFromDragDrop(self, paths):
from hydrus.client.gui import ClientGUIDialogsQuick
message = 'Try to import the {} dropped files to this list? I am expecting json or png files.'.format(
HydrusData.ToHumanInt(len(paths)))
result = ClientGUIDialogsQuick.GetYesNo(self, message)
if result == QW.QDialog.Accepted:
(jsons, pngs) = HydrusData.PartitionIteratorIntoLists(
lambda path: path.endswith('.png'), paths)
self._ImportPNGs(pngs)
self._ImportJSONs(jsons)
self._listctrl.Sort()
def ToString(self, simple=False, with_type=False) -> str:
if simple:
return 'splitter'
result = 'splitting by "{}"'.format(self._separator)
if self._max_splits is not None:
result = '{}, at most {} times'.format(
result, HydrusData.ToHumanInt(self._max_splits))
if with_type:
result = 'SPLIT: {}'.format(result)
return result
def RegenerateTree(self):
job_key = ClientThreading.JobKey()
try:
job_key.SetVariable('popup_title',
'regenerating similar file search data')
HG.client_controller.pub('modal_message', job_key)
job_key.SetVariable('popup_text_1',
'purging search info of orphans')
self._c.execute(
'DELETE FROM shape_perceptual_hash_map WHERE hash_id NOT IN ( SELECT hash_id FROM current_files );'
)
job_key.SetVariable('popup_text_1', 'gathering all leaves')
self._c.execute('DELETE FROM shape_vptree;')
all_nodes = self._c.execute(
'SELECT phash_id, phash FROM shape_perceptual_hashes;'
).fetchall()
job_key.SetVariable(
'popup_text_1',
HydrusData.ToHumanInt(len(all_nodes)) +
' leaves found, now regenerating')
(root_id, root_phash) = self._PopBestRootNode(
all_nodes) #HydrusData.RandomPop( all_nodes )
self._GenerateBranch(job_key, None, root_id, root_phash, all_nodes)
finally:
job_key.SetVariable('popup_text_1', 'done!')
job_key.DeleteVariable('popup_text_2')
job_key.Finish()
job_key.Delete(5)
def ResetPotentialSearch(win, hashes):
if len(hashes) == 1:
message = 'Are you sure you want to search this file for potential duplicates again?'
message += os.linesep * 2
message += 'This will not remove any existing potential pairs, and will typically not find any new relationships unless an error has occured.'
else:
message = 'Are you sure you want to search these {} files for potential duplicates again?'.format(
HydrusData.ToHumanInt(len(hashes)))
message += os.linesep * 2
message += 'This will not remove any existing potential pairs, and will typically not find any new relationships unless an error has occured.'
result = ClientGUIDialogsQuick.GetYesNo(win, message)
if result == QW.QDialog.Accepted:
HG.client_controller.Write('reset_potential_search_status', hashes)
def RemovePotentials(win, hashes):
if len(hashes) == 1:
message = 'Are you sure you want to remove all of this file\'s potentials?'
message += os.linesep * 2
message += 'This will mean it (or any of its duplicates) will not appear in the duplicate filter unless new potentials are found with new files. Use this command if the file has accidentally received many false positive potential relationships.'
else:
message = 'Are you sure you want to remove all of these {} files\' potentials?'.format(
HydrusData.ToHumanInt(len(hashes)))
message += os.linesep * 2
message += 'This will mean they (or any of their duplicates) will not appear in the duplicate filter unless new potentials are found with new files. Use this command if the files have accidentally received many false positive potential relationships.'
result = ClientGUIDialogsQuick.GetYesNo(win, message)
if result == QW.QDialog.Accepted:
HG.client_controller.Write('remove_potential_pairs', hashes)
def _GetTuples(self, shortcuts):
name = shortcuts.GetName()
if name in ClientGUIShortcuts.shortcut_names_to_descriptions:
pretty_name = ClientGUIShortcuts.shortcut_names_to_pretty_names[
name]
sort_name = ClientGUIShortcuts.shortcut_names_sorted.index(name)
else:
pretty_name = name
sort_name = name
size = len(shortcuts)
display_tuple = (pretty_name, HydrusData.ToHumanInt(size))
sort_tuple = (sort_name, size)
return (display_tuple, sort_tuple)
def _ConvertRuleToListCtrlTuples(self, rule):
(bandwidth_type, time_delta, max_allowed) = rule
pretty_time_delta = HydrusData.TimeDeltaToPrettyTimeDelta(time_delta)
if bandwidth_type == HC.BANDWIDTH_TYPE_DATA:
pretty_max_allowed = HydrusData.ToHumanBytes(max_allowed)
elif bandwidth_type == HC.BANDWIDTH_TYPE_REQUESTS:
pretty_max_allowed = HydrusData.ToHumanInt(
max_allowed) + ' requests'
sort_time_delta = ClientGUIListCtrl.SafeNoneInt(time_delta)
sort_tuple = (max_allowed, sort_time_delta)
display_tuple = (pretty_max_allowed, pretty_time_delta)
return (display_tuple, sort_tuple)
def RemoveFromAlternateGroup(win, hashes):
if len(hashes) == 1:
message = 'Are you sure you want to remove this file from its alternates group?'
message += os.linesep * 2
message += 'Alternate relationships are stored between duplicate groups, so this will pull any duplicates of your file with it.'
message += os.linesep * 2
message += 'The removed file (and any duplicates) will be queued up for another potential duplicates search, so you will likely see at least one again in the duplicate filter.'
else:
message = 'Are you sure you want to remove these {} files from their alternates groups?'.format(
HydrusData.ToHumanInt(len(hashes)))
message += os.linesep * 2
message += 'Alternate relationships are stored between duplicate groups, so this will pull any duplicates of these files with them.'
message += os.linesep * 2
message += 'The removed files (and any duplicates) will be queued up for another potential duplicates search, so you will likely see some again in the duplicate filter.'
result = ClientGUIDialogsQuick.GetYesNo(win, message)
if result == QW.QDialog.Accepted:
HG.client_controller.Write('remove_alternates_member', hashes)
def DissolveDuplicateGroup(win, hashes):
if len(hashes) == 1:
message = 'Are you sure you want to dissolve this file\'s duplicate group?'
message += os.linesep * 2
message += 'This will split the duplicates group back into individual files and remove any alternate relations they have. They will be queued back up in the duplicate filter for reprocessing.'
message += os.linesep * 2
message += 'This could be a big change that throws away many previous decisions and cannot be undone. If you can achieve your result just by removing one or two members, do that instead.'
else:
message = 'Are you sure you want to dissolve these {} files\' duplicate groups?'.format(
HydrusData.ToHumanInt(len(hashes)))
message += os.linesep * 2
message += 'This will split all the files\' duplicates groups back into individual files and remove any alternate relations they have. They will all be queued back up in the duplicate filter for reprocessing.'
message += os.linesep * 2
message += 'This could be a huge change that throws away many previous decisions and cannot be undone. If you can achieve your result just by removing some members, do that instead.'
result = ClientGUIDialogsQuick.GetYesNo(win, message)
if result == QW.QDialog.Accepted:
HG.client_controller.Write('dissolve_duplicates_group', hashes)
def DissolveAlternateGroup(win, hashes):
if len(hashes) == 1:
message = 'Are you sure you want to dissolve this file\'s entire alternates group?'
message += os.linesep * 2
message += 'This will completely remove all duplicate, alternate, and false-positive relations for all files in the group and set them to come up again in the duplicate filter.'
message += os.linesep * 2
message += 'This is a potentially big change that throws away many previous decisions and cannot be undone. If you can achieve your result just by removing some alternate members, do that instead.'
else:
message = 'Are you sure you want to dissolve these {} files\' entire alternates groups?'.format(
HydrusData.ToHumanInt(len(hashes)))
message += os.linesep * 2
message += 'This will completely remove all duplicate, alternate, and false-positive relations for all alternate groups of all files selected and set them to come up again in the duplicate filter.'
message += os.linesep * 2
message += 'This is a potentially huge change that throws away many previous decisions and cannot be undone. If you can achieve your result just by removing some alternate members, do that instead.'
result = ClientGUIDialogsQuick.GetYesNo(win, message)
if result == QW.QDialog.Accepted:
HG.client_controller.Write('dissolve_alternates_group', hashes)
def ClearFalsePositives(win, hashes):
if len(hashes) == 1:
message = 'Are you sure you want to clear this file of its false-positive relations?'
message += os.linesep * 2
message += 'False-positive relations are recorded between alternate groups, so this change will also affect any files this file is alternate to.'
message += os.linesep * 2
message += 'All affected files will be queued up for another potential duplicates search, so you will likely see at least one of them again in the duplicate filter.'
else:
message = 'Are you sure you want to clear these {} files of their false-positive relations?'.format(
HydrusData.ToHumanInt(len(hashes)))
message += os.linesep * 2
message += 'False-positive relations are recorded between alternate groups, so this change will also affect all alternate files to your selection.'
message += os.linesep * 2
message += 'All affected files will be queued up for another potential duplicates search, so you will likely see some of them again in the duplicate filter.'
result = ClientGUIDialogsQuick.GetYesNo(win, message)
if result == QW.QDialog.Accepted:
HG.client_controller.Write('clear_false_positive_relations', hashes)
def RemoveFromDuplicateGroup(win, hashes):
if len(hashes) == 1:
message = 'Are you sure you want to remove this file from its duplicate group?'
message += os.linesep * 2
message += 'The remaining group will be otherwise unaffected and will keep its alternate relationships.'
message += os.linesep * 2
message += 'The removed file will be queued up for another potential duplicates search, so you will likely see it again in the duplicate filter.'
else:
message = 'Are you sure you want to remove these {} files from their duplicate groups?'.format(
HydrusData.ToHumanInt(len(hashes)))
message += os.linesep * 2
message += 'The remaining groups will be otherwise unaffected and keep their alternate relationships.'
message += os.linesep * 2
message += 'The removed files will be queued up for another potential duplicates search, so you will likely see them again in the duplicate filter.'
result = ClientGUIDialogsQuick.GetYesNo(win, message)
if result == QW.QDialog.Accepted:
HG.client_controller.Write('remove_duplicates_member', hashes)
def THREADDownloadURLs( job_key, urls, title ):
job_key.SetVariable( 'popup_title', title )
job_key.SetVariable( 'popup_text_1', 'initialising' )
num_successful = 0
num_redundant = 0
num_deleted = 0
num_failed = 0
presentation_hashes = []
presentation_hashes_fast = set()
file_import_options = HG.client_controller.new_options.GetDefaultFileImportOptions( 'loud' )
def network_job_factory( *args, **kwargs ):
network_job = ClientNetworkingJobs.NetworkJob( *args, **kwargs )
network_job.OverrideBandwidth()
return network_job
def status_hook( text ):
if len( text ) > 0:
text = text.splitlines()[0]
job_key.SetVariable( 'popup_text_2', text )
network_job_presentation_context_factory = GenerateMultiplePopupNetworkJobPresentationContextFactory( job_key )
for ( i, url ) in enumerate( urls ):
( i_paused, should_quit ) = job_key.WaitIfNeeded()
if should_quit:
break
job_key.SetVariable( 'popup_text_1', HydrusData.ConvertValueRangeToPrettyString( i + 1, len( urls ) ) )
job_key.SetVariable( 'popup_gauge_1', ( i + 1, len( urls ) ) )
file_seed = ClientImportFileSeeds.FileSeed( ClientImportFileSeeds.FILE_SEED_TYPE_URL, url )
try:
file_seed.DownloadAndImportRawFile( url, file_import_options, network_job_factory, network_job_presentation_context_factory, status_hook )
status = file_seed.status
if status in CC.SUCCESSFUL_IMPORT_STATES:
if status == CC.STATUS_SUCCESSFUL_AND_NEW:
num_successful += 1
elif status == CC.STATUS_SUCCESSFUL_BUT_REDUNDANT:
num_redundant += 1
if file_seed.HasHash():
hash = file_seed.GetHash()
if hash not in presentation_hashes_fast:
presentation_hashes.append( hash )
presentation_hashes_fast.add( hash )
if len( presentation_hashes ) > 0:
job_key.SetVariable( 'popup_files', ( presentation_hashes, 'downloads' ) )
elif status == CC.STATUS_DELETED:
num_deleted += 1
except Exception as e:
num_failed += 1
HydrusData.Print( url + ' failed to import!' )
HydrusData.PrintException( e )
finally:
job_key.DeleteVariable( 'popup_text_2' )
job_key.DeleteVariable( 'popup_network_job' )
text_components = []
if num_successful > 0:
text_components.append( HydrusData.ToHumanInt( num_successful ) + ' successful' )
if num_redundant > 0:
text_components.append( HydrusData.ToHumanInt( num_redundant ) + ' already in db' )
if num_deleted > 0:
text_components.append( HydrusData.ToHumanInt( num_deleted ) + ' deleted' )
if num_failed > 0:
text_components.append( HydrusData.ToHumanInt( num_failed ) + ' failed (errors written to log)' )
job_key.SetVariable( 'popup_text_1', ', '.join( text_components ) )
if len( presentation_hashes ) > 0:
job_key.SetVariable( 'popup_files', ( presentation_hashes, 'downloads' ) )
job_key.DeleteVariable( 'popup_gauge_1' )
job_key.Finish()
def Process(self):
# only do one list of callables at a time
# we don't want to map a topic to its callables until the previous topic's callables have been fully executed
# e.g. when we start a message with a pubsub, it'll take a while (in independant thread-time) for Qt to create
# the dialog and hence map the new callable to the topic. this was leading to messages not being updated
# because the (short) processing thread finished and entirely pubsubbed before Qt had a chance to boot the
# message.
self._doing_work = True
try:
callable_tuples = []
with self._lock:
if len(self._pubsubs) == 0:
return
pubsubs = self._pubsubs
self._pubsubs = []
for (topic, args, kwargs) in pubsubs:
try:
# do all this _outside_ the lock, lol
callable_tuples = self._GetCallableTuples(topic)
# don't want to report the showtext we just send here!
not_a_report = topic != 'message'
if HG.pubsub_report_mode and not_a_report:
HydrusData.ShowText(
(topic, args, kwargs, callable_tuples))
if HG.pubsub_profile_mode and not_a_report:
summary = 'Profiling ' + HydrusData.ToHumanInt(
len(callable_tuples)) + ' x ' + topic
HydrusData.ShowText(summary)
per_summary = 'Profiling ' + topic
for (obj, callable) in callable_tuples:
try:
HydrusData.Profile(
per_summary, 'callable( *args, **kwargs )',
globals(), locals())
except HydrusExceptions.ShutdownException:
return False
else:
for (obj, callable) in callable_tuples:
try:
callable(*args, **kwargs)
except HydrusExceptions.ShutdownException:
return False
except Exception as e:
HydrusData.ShowException(e)
finally:
self._doing_work = False
def _ImportFiles(self, job_key):
did_work = False
time_to_save = HydrusData.GetNow() + 600
num_files_imported = 0
presentation_hashes = []
presentation_hashes_fast = set()
i = 0
num_total = len(self._file_seed_cache)
num_total_unknown = self._file_seed_cache.GetFileSeedCount(
CC.STATUS_UNKNOWN)
num_total_done = num_total - num_total_unknown
while True:
file_seed = self._file_seed_cache.GetNextFileSeed(
CC.STATUS_UNKNOWN)
p1 = HC.options['pause_import_folders_sync'] or self._paused
p2 = HydrusThreading.IsThreadShuttingDown()
p3 = job_key.IsCancelled()
if file_seed is None or p1 or p2 or p3:
break
did_work = True
if HydrusData.TimeHasPassed(time_to_save):
HG.client_controller.WriteSynchronous('serialisable', self)
time_to_save = HydrusData.GetNow() + 600
gauge_num_done = num_total_done + num_files_imported + 1
job_key.SetVariable(
'popup_text_1',
'importing file ' + HydrusData.ConvertValueRangeToPrettyString(
gauge_num_done, num_total))
job_key.SetVariable('popup_gauge_1', (gauge_num_done, num_total))
path = file_seed.file_seed_data
file_seed.ImportPath(self._file_seed_cache,
self._file_import_options,
limited_mimes=self._mimes)
if file_seed.status in CC.SUCCESSFUL_IMPORT_STATES:
if file_seed.HasHash():
hash = file_seed.GetHash()
if self._tag_import_options.HasAdditionalTags():
media_result = HG.client_controller.Read(
'media_result', hash)
downloaded_tags = []
service_keys_to_content_updates = self._tag_import_options.GetServiceKeysToContentUpdates(
file_seed.status, media_result,
downloaded_tags) # additional tags
if len(service_keys_to_content_updates) > 0:
HG.client_controller.WriteSynchronous(
'content_updates',
service_keys_to_content_updates)
service_keys_to_tags = ClientTags.ServiceKeysToTags()
for (tag_service_key, filename_tagging_options) in list(
self._tag_service_keys_to_filename_tagging_options.
items()):
if not HG.client_controller.services_manager.ServiceExists(
tag_service_key):
continue
try:
tags = filename_tagging_options.GetTags(
tag_service_key, path)
if len(tags) > 0:
service_keys_to_tags[tag_service_key] = tags
except Exception as e:
HydrusData.ShowText(
'Trying to parse filename tags in the import folder "'
+ self._name + '" threw an error!')
HydrusData.ShowException(e)
if len(service_keys_to_tags) > 0:
service_keys_to_content_updates = ClientData.ConvertServiceKeysToTagsToServiceKeysToContentUpdates(
{hash}, service_keys_to_tags)
HG.client_controller.WriteSynchronous(
'content_updates', service_keys_to_content_updates)
num_files_imported += 1
if hash not in presentation_hashes_fast:
if file_seed.ShouldPresent(self._file_import_options):
presentation_hashes.append(hash)
presentation_hashes_fast.add(hash)
elif file_seed.status == CC.STATUS_ERROR:
HydrusData.Print(
'A file failed to import from import folder ' +
self._name + ':' + path)
i += 1
if i % 10 == 0:
self._ActionPaths()
if num_files_imported > 0:
HydrusData.Print('Import folder ' + self._name + ' imported ' +
HydrusData.ToHumanInt(num_files_imported) +
' files.')
if len(presentation_hashes) > 0:
ClientImporting.PublishPresentationHashes(
self._name, presentation_hashes,
self._publish_files_to_popup_button,
self._publish_files_to_page)
self._ActionPaths()
return did_work
def AddKnownURLsViewCopyMenu(win, menu, focus_media, selected_media=None):
# figure out which urls this focused file has
focus_urls = focus_media.GetLocationsManager().GetURLs()
focus_matched_labels_and_urls = []
focus_unmatched_urls = []
focus_labels_and_urls = []
if len(focus_urls) > 0:
for url in focus_urls:
try:
url_class = HG.client_controller.network_engine.domain_manager.GetURLClass(
url)
except HydrusExceptions.URLClassException:
continue
if url_class is None:
focus_unmatched_urls.append(url)
else:
label = url_class.GetName() + ': ' + url
focus_matched_labels_and_urls.append((label, url))
focus_matched_labels_and_urls.sort()
focus_unmatched_urls.sort()
focus_labels_and_urls = list(focus_matched_labels_and_urls)
focus_labels_and_urls.extend(
((url, url) for url in focus_unmatched_urls))
# figure out which urls these selected files have
selected_media_url_classes = set()
multiple_or_unmatching_selection_url_classes = False
if selected_media is not None and len(selected_media) > 1:
selected_media = ClientMedia.FlattenMedia(selected_media)
SAMPLE_SIZE = 256
if len(selected_media) > SAMPLE_SIZE:
selected_media_sample = random.sample(selected_media, SAMPLE_SIZE)
else:
selected_media_sample = selected_media
for media in selected_media_sample:
media_urls = media.GetLocationsManager().GetURLs()
for url in media_urls:
try:
url_class = HG.client_controller.network_engine.domain_manager.GetURLClass(
url)
except HydrusExceptions.URLClassException:
continue
if url_class is None:
multiple_or_unmatching_selection_url_classes = True
else:
selected_media_url_classes.add(url_class)
if len(selected_media_url_classes) > 1:
multiple_or_unmatching_selection_url_classes = True
if len(focus_labels_and_urls) > 0 or len(
selected_media_url_classes
) > 0 or multiple_or_unmatching_selection_url_classes:
urls_menu = QW.QMenu(menu)
urls_visit_menu = QW.QMenu(urls_menu)
urls_copy_menu = QW.QMenu(urls_menu)
# copy each this file's urls (of a particular type)
if len(focus_labels_and_urls) > 0:
for (label, url) in focus_labels_and_urls:
ClientGUIMenus.AppendMenuItem(
urls_visit_menu, label,
'Open this url in your web browser.',
ClientPaths.LaunchURLInWebBrowser, url)
ClientGUIMenus.AppendMenuItem(
urls_copy_menu, label, 'Copy this url to your clipboard.',
HG.client_controller.pub, 'clipboard', 'text', url)
# copy this file's urls
there_are_focus_url_classes_to_action = len(
focus_matched_labels_and_urls) > 1
multiple_or_unmatching_focus_url_classes = len(
focus_unmatched_urls) > 0 and len(
focus_labels_and_urls
) > 1 # if there are unmatched urls and more than one thing total
if there_are_focus_url_classes_to_action or multiple_or_unmatching_focus_url_classes:
ClientGUIMenus.AppendSeparator(urls_visit_menu)
ClientGUIMenus.AppendSeparator(urls_copy_menu)
if there_are_focus_url_classes_to_action:
urls = [url for (label, url) in focus_matched_labels_and_urls]
label = 'open this file\'s ' + HydrusData.ToHumanInt(
len(urls)) + ' recognised urls in your web browser'
ClientGUIMenus.AppendMenuItem(
urls_visit_menu, label, 'Open these urls in your web browser.',
OpenURLs, urls)
urls_string = os.linesep.join(urls)
label = 'copy this file\'s ' + HydrusData.ToHumanInt(
len(urls)) + ' recognised urls to your clipboard'
ClientGUIMenus.AppendMenuItem(
urls_copy_menu, label, 'Copy these urls to your clipboard.',
HG.client_controller.pub, 'clipboard', 'text', urls_string)
if multiple_or_unmatching_focus_url_classes:
urls = [url for (label, url) in focus_labels_and_urls]
label = 'open this file\'s ' + HydrusData.ToHumanInt(
len(urls)) + ' urls in your web browser'
ClientGUIMenus.AppendMenuItem(
urls_visit_menu, label, 'Open these urls in your web browser.',
OpenURLs, urls)
urls_string = os.linesep.join(urls)
label = 'copy this file\'s ' + HydrusData.ToHumanInt(
len(urls)) + ' urls to your clipboard'
ClientGUIMenus.AppendMenuItem(urls_copy_menu, label,
'Copy this url to your clipboard.',
HG.client_controller.pub,
'clipboard', 'text', urls_string)
# now by url match type
there_are_selection_url_classes_to_action = len(
selected_media_url_classes) > 0
if there_are_selection_url_classes_to_action or multiple_or_unmatching_selection_url_classes:
ClientGUIMenus.AppendSeparator(urls_visit_menu)
ClientGUIMenus.AppendSeparator(urls_copy_menu)
if there_are_selection_url_classes_to_action:
selected_media_url_classes = list(selected_media_url_classes)
selected_media_url_classes.sort(
key=lambda url_class: url_class.GetName())
for url_class in selected_media_url_classes:
label = 'open files\' ' + url_class.GetName(
) + ' urls in your web browser'
ClientGUIMenus.AppendMenuItem(
urls_visit_menu, label,
'Open this url class in your web browser for all files.',
OpenMediaURLClassURLs, selected_media, url_class)
label = 'copy files\' ' + url_class.GetName() + ' urls'
ClientGUIMenus.AppendMenuItem(
urls_copy_menu, label,
'Copy this url class for all files.',
CopyMediaURLClassURLs, selected_media, url_class)
# now everything
if multiple_or_unmatching_selection_url_classes:
label = 'open all files\' urls'
ClientGUIMenus.AppendMenuItem(
urls_visit_menu, label,
'Open urls in your web browser for all files.', OpenMediaURLs,
selected_media)
label = 'copy all files\' urls'
ClientGUIMenus.AppendMenuItem(urls_copy_menu, label,
'Copy urls for all files.',
CopyMediaURLs, selected_media)
#
ClientGUIMenus.AppendMenu(urls_menu, urls_visit_menu, 'open')
ClientGUIMenus.AppendMenu(urls_menu, urls_copy_menu, 'copy')
ClientGUIMenus.AppendMenu(menu, urls_menu, 'known urls')
def PopulateGallerySeedLogButton(
win: QW.QWidget, menu: QW.QMenu,
gallery_seed_log: ClientImportGallerySeeds.GallerySeedLog,
read_only: bool, can_generate_more_pages: bool,
gallery_type_string: str):
num_successful = gallery_seed_log.GetGallerySeedCount(
CC.STATUS_SUCCESSFUL_AND_NEW)
num_vetoed = gallery_seed_log.GetGallerySeedCount(CC.STATUS_VETOED)
num_errors = gallery_seed_log.GetGallerySeedCount(CC.STATUS_ERROR)
num_skipped = gallery_seed_log.GetGallerySeedCount(CC.STATUS_SKIPPED)
if num_successful > 0:
ClientGUIMenus.AppendMenuItem(
menu,
'delete {} \'successful\' gallery log entries from the log'.format(
HydrusData.ToHumanInt(num_successful)),
'Tell this log to clear out successful records, reducing the size of the queue.',
ClearGallerySeeds, win, gallery_seed_log,
(CC.STATUS_SUCCESSFUL_AND_NEW, ), gallery_type_string)
if num_errors > 0:
ClientGUIMenus.AppendMenuItem(
menu,
'delete {} \'failed\' gallery log entries from the log'.format(
HydrusData.ToHumanInt(num_errors)),
'Tell this log to clear out errored records, reducing the size of the queue.',
ClearGallerySeeds, win, gallery_seed_log, (CC.STATUS_ERROR, ),
gallery_type_string)
if num_vetoed > 0:
ClientGUIMenus.AppendMenuItem(
menu,
'delete {} \'ignored\' gallery log entries from the log'.format(
HydrusData.ToHumanInt(num_vetoed)),
'Tell this log to clear out ignored records, reducing the size of the queue.',
ClearGallerySeeds, win, gallery_seed_log, (CC.STATUS_VETOED, ),
gallery_type_string)
if num_skipped > 0:
ClientGUIMenus.AppendMenuItem(
menu,
'delete {} \'skipped\' gallery log entries from the log'.format(
HydrusData.ToHumanInt(num_skipped)),
'Tell this log to clear out skipped records, reducing the size of the queue.',
ClearGallerySeeds, win, gallery_seed_log, (CC.STATUS_SKIPPED, ),
gallery_type_string)
ClientGUIMenus.AppendSeparator(menu)
if len(gallery_seed_log) > 0:
if not read_only and gallery_seed_log.CanRestartFailedSearch():
ClientGUIMenus.AppendMenuItem(
menu, 'restart and resume failed search',
'Requeue the last failed attempt and resume search from there.',
gallery_seed_log.RestartFailedSearch)
ClientGUIMenus.AppendSeparator(menu)
submenu = QW.QMenu(menu)
ClientGUIMenus.AppendMenuItem(
submenu, 'to clipboard',
'Copy all the urls in this list to the clipboard.',
ExportToClipboard, gallery_seed_log)
ClientGUIMenus.AppendMenuItem(
submenu, 'to png',
'Export all the urls in this list to a png file.', ExportToPNG,
win, gallery_seed_log)
ClientGUIMenus.AppendMenu(menu, submenu, 'export all urls')
if not read_only:
submenu = QW.QMenu(menu)
ClientGUIMenus.AppendMenuItem(
submenu, 'from clipboard',
'Import new urls to this list from the clipboard.',
ImportFromClipboard, win, gallery_seed_log,
can_generate_more_pages)
ClientGUIMenus.AppendMenuItem(
submenu, 'from png',
'Import new urls to this list from a png file.', ImportFromPNG,
win, gallery_seed_log, can_generate_more_pages)
ClientGUIMenus.AppendMenu(menu, submenu, 'import new urls')
def GetCurrentJobSummary(self):
with self._waiting_lock:
return HydrusData.ToHumanInt(len(self._waiting)) + ' jobs'
