def _UpdateSerialisableInfo( self, version, old_serialisable_info ):
if version == 1:
( serialisable_file_seed_cache, serialisable_file_import_options, paused ) = old_serialisable_info
gallery_seed_log = ClientImportGallerySeeds.GallerySeedLog()
serialisable_gallery_seed_log = gallery_seed_log.GetSerialisableTuple()
new_serialisable_info = ( serialisable_gallery_seed_log, serialisable_file_seed_cache, serialisable_file_import_options, paused )
return ( 2, new_serialisable_info )
if version == 2:
( serialisable_gallery_seed_log, serialisable_file_seed_cache, serialisable_file_import_options, paused ) = old_serialisable_info
tag_import_options = TagImportOptions.TagImportOptions( is_default = True )
serialisable_tag_import_options = tag_import_options.GetSerialisableTuple()
new_serialisable_info = ( serialisable_gallery_seed_log, serialisable_file_seed_cache, serialisable_file_import_options, serialisable_tag_import_options, paused )
return ( 3, new_serialisable_info )
def __init__( self ):
HydrusSerialisable.SerialisableBase.__init__( self )
self._gallery_seed_log = ClientImportGallerySeeds.GallerySeedLog()
self._file_seed_cache = ClientImportFileSeeds.FileSeedCache()
self._file_import_options = HG.client_controller.new_options.GetDefaultFileImportOptions( 'loud' )
self._tag_import_options = TagImportOptions.TagImportOptions( is_default = True )
self._paused = False
self._no_work_until = 0
self._no_work_until_reason = ''
self._page_key = b'initialising page key'
self._downloader_key = HydrusData.GenerateKey()
self._lock = threading.Lock()
self._have_started = False
self._files_status = ''
self._gallery_status = ''
self._files_network_job = None
self._gallery_network_job = None
self._files_repeating_job = None
self._gallery_repeating_job = None
self._last_serialisable_change_timestamp = 0
HG.client_controller.sub( self, 'NotifyFileSeedsUpdated', 'file_seed_cache_file_seeds_updated' )
HG.client_controller.sub( self, 'NotifyGallerySeedsUpdated', 'gallery_seed_log_gallery_seeds_updated' )
def _WorkOnFiles( self ):
file_seed = self._file_seed_cache.GetNextFileSeed( CC.STATUS_UNKNOWN )
if file_seed is None:
return
did_substantial_work = False
def status_hook( text ):
with self._lock:
self._files_status = ClientImportControl.NeatenStatusText( text )
tag_import_options = TagImportOptions.TagImportOptions( is_default = True )
try:
did_substantial_work = file_seed.WorkOnURL( self._file_seed_cache, status_hook, self._NetworkJobFactory, self._FileNetworkJobPresentationContextFactory, self._file_import_options, tag_import_options )
except HydrusExceptions.NetworkException as e:
delay = HG.client_controller.new_options.GetInteger( 'downloader_network_error_delay' )
self._DelayWork( delay, str( e ) )
file_seed.SetStatus( CC.STATUS_ERROR, str( e ) )
HydrusData.PrintException( e )
except Exception as e:
status = CC.STATUS_ERROR
file_seed.SetStatus( status, exception = e )
time.sleep( 3 )
if file_seed.ShouldPresent( self._file_import_options.GetPresentationImportOptions() ):
file_seed.PresentToPage( self._page_key )
did_substantial_work = True
with self._lock:
self._files_status = ''
if did_substantial_work:
time.sleep( ClientImporting.DID_SUBSTANTIAL_FILE_WORK_MINIMUM_SLEEP_TIME )
def __init__( self, name, path = '', file_import_options = None, tag_import_options = None, tag_service_keys_to_filename_tagging_options = None, mimes = None, actions = None, action_locations = None, period = 3600, check_regularly = True, show_working_popup = True, publish_files_to_popup_button = True, publish_files_to_page = False ):
if mimes is None:
mimes = HC.ALLOWED_MIMES
if file_import_options is None:
file_import_options = HG.client_controller.new_options.GetDefaultFileImportOptions( 'quiet' )
if tag_import_options is None:
tag_import_options = TagImportOptions.TagImportOptions()
if tag_service_keys_to_filename_tagging_options is None:
tag_service_keys_to_filename_tagging_options = {}
if actions is None:
actions = {}
actions[ CC.STATUS_SUCCESSFUL_AND_NEW ] = CC.IMPORT_FOLDER_IGNORE
actions[ CC.STATUS_SUCCESSFUL_BUT_REDUNDANT ] = CC.IMPORT_FOLDER_IGNORE
actions[ CC.STATUS_DELETED ] = CC.IMPORT_FOLDER_IGNORE
actions[ CC.STATUS_ERROR ] = CC.IMPORT_FOLDER_IGNORE
if action_locations is None:
action_locations = {}
HydrusSerialisable.SerialisableBaseNamed.__init__( self, name )
self._path = path
self._mimes = mimes
self._file_import_options = file_import_options
self._tag_import_options = tag_import_options
self._tag_service_keys_to_filename_tagging_options = tag_service_keys_to_filename_tagging_options
self._actions = actions
self._action_locations = action_locations
self._period = period
self._check_regularly = check_regularly
self._file_seed_cache = ClientImportFileSeeds.FileSeedCache()
self._last_checked = 0
self._paused = False
self._check_now = False
self._show_working_popup = show_working_popup
self._publish_files_to_popup_button = publish_files_to_popup_button
self._publish_files_to_page = publish_files_to_page
def _WorkOnFiles(self, page_key):
file_seed = self._file_seed_cache.GetNextFileSeed(CC.STATUS_UNKNOWN)
if file_seed is None:
return
did_substantial_work = False
def status_hook(text):
with self._lock:
if len(text) > 0:
text = text.splitlines()[0]
self._current_action = text
tag_import_options = TagImportOptions.TagImportOptions(is_default=True)
did_substantial_work = file_seed.WorkOnURL(
self._file_seed_cache, status_hook, self._NetworkJobFactory,
self._FileNetworkJobPresentationContextFactory,
self._file_import_options, tag_import_options)
if file_seed.ShouldPresent(
self._file_import_options.GetPresentationImportOptions()):
file_seed.PresentToPage(page_key)
did_substantial_work = True
with self._lock:
self._current_action = ''
if did_substantial_work:
time.sleep(
ClientImporting.DID_SUBSTANTIAL_FILE_WORK_MINIMUM_SLEEP_TIME)
def __init__(self):
HydrusSerialisable.SerialisableBase.__init__(self)
self._query_log_container_name = GenerateQueryLogContainerName()
self._query_text = 'query'
self._display_name = None
self._check_now = False
self._last_check_time = 0
self._next_check_time = 0
self._paused = False
self._checker_status = ClientImporting.CHECKER_STATUS_OK
self._query_log_container_status = LOG_CONTAINER_UNSYNCED
self._file_seed_cache_status = ClientImportFileSeeds.FileSeedCacheStatus(
)
self._tag_import_options = TagImportOptions.TagImportOptions()
self._raw_file_velocity = (0, 1)
self._pretty_file_velocity = 'unknown'
self._example_file_seed = None
self._example_gallery_seed = None
def _UpdateSerialisableInfo(self, version, old_serialisable_info):
if version == 1:
(path, mimes, serialisable_file_import_options, action_pairs,
action_location_pairs, period, open_popup, tag,
serialisable_file_seed_cache, last_checked,
paused) = old_serialisable_info
# edited out tag carry-over to tio due to bit rot
tag_import_options = TagImportOptions.TagImportOptions()
serialisable_tag_import_options = tag_import_options.GetSerialisableTuple(
)
new_serialisable_info = (path, mimes,
serialisable_file_import_options,
serialisable_tag_import_options,
action_pairs, action_location_pairs,
period, open_popup,
serialisable_file_seed_cache,
last_checked, paused)
return (2, new_serialisable_info)
if version == 2:
(path, mimes, serialisable_file_import_options,
serialisable_tag_import_options, action_pairs,
action_location_pairs, period, open_popup,
serialisable_file_seed_cache, last_checked,
paused) = old_serialisable_info
serialisable_txt_parse_tag_service_keys = []
new_serialisable_info = (path, mimes,
serialisable_file_import_options,
serialisable_tag_import_options,
serialisable_txt_parse_tag_service_keys,
action_pairs, action_location_pairs,
period, open_popup,
serialisable_file_seed_cache,
last_checked, paused)
return (3, new_serialisable_info)
if version == 3:
(path, mimes, serialisable_file_import_options,
serialisable_tag_import_options,
serialisable_txt_parse_tag_service_keys, action_pairs,
action_location_pairs, period, open_popup,
serialisable_file_seed_cache, last_checked,
paused) = old_serialisable_info
check_now = False
new_serialisable_info = (path, mimes,
serialisable_file_import_options,
serialisable_tag_import_options,
serialisable_txt_parse_tag_service_keys,
action_pairs, action_location_pairs,
period, open_popup,
serialisable_file_seed_cache,
last_checked, paused, check_now)
return (4, new_serialisable_info)
if version == 4:
(path, mimes, serialisable_file_import_options,
serialisable_tag_import_options,
serialisable_txt_parse_tag_service_keys, action_pairs,
action_location_pairs, period, open_popup,
serialisable_file_seed_cache, last_checked, paused,
check_now) = old_serialisable_info
txt_parse_tag_service_keys = [
bytes.fromhex(service_key)
for service_key in serialisable_txt_parse_tag_service_keys
]
tag_service_keys_to_filename_tagging_options = {}
for service_key in txt_parse_tag_service_keys:
filename_tagging_options = TagImportOptions.FilenameTaggingOptions(
)
filename_tagging_options._load_from_neighbouring_txt_files = True
tag_service_keys_to_filename_tagging_options[
service_key] = filename_tagging_options
serialisable_tag_service_keys_to_filename_tagging_options = [
(service_key.hex(),
filename_tagging_options.GetSerialisableTuple())
for (service_key, filename_tagging_options) in list(
tag_service_keys_to_filename_tagging_options.items())
]
new_serialisable_info = (
path, mimes, serialisable_file_import_options,
serialisable_tag_import_options,
serialisable_tag_service_keys_to_filename_tagging_options,
action_pairs, action_location_pairs, period, open_popup,
serialisable_file_seed_cache, last_checked, paused, check_now)
return (5, new_serialisable_info)
if version == 5:
(path, mimes, serialisable_file_import_options,
serialisable_tag_import_options,
serialisable_tag_service_keys_to_filename_tagging_options,
action_pairs, action_location_pairs, period, open_popup,
serialisable_file_seed_cache, last_checked, paused,
check_now) = old_serialisable_info
check_regularly = not paused
show_working_popup = True
publish_files_to_page = False
publish_files_to_popup_button = open_popup
new_serialisable_info = (
path, mimes, serialisable_file_import_options,
serialisable_tag_import_options,
serialisable_tag_service_keys_to_filename_tagging_options,
action_pairs, action_location_pairs, period, check_regularly,
serialisable_file_seed_cache, last_checked, paused, check_now,
show_working_popup, publish_files_to_popup_button,
publish_files_to_page)
return (6, new_serialisable_info)
def test_SERIALISABLE_TYPE_SUBSCRIPTION(self):
def test(obj, dupe_obj):
self.assertEqual(obj.GetName(), dupe_obj.GetName())
self.assertEqual(obj._gug_key_and_name, dupe_obj._gug_key_and_name)
self.assertEqual(len(obj._query_headers),
len(dupe_obj._query_headers))
self.assertEqual(obj._initial_file_limit,
dupe_obj._initial_file_limit)
self.assertEqual(obj._periodic_file_limit,
dupe_obj._periodic_file_limit)
self.assertEqual(obj._paused, dupe_obj._paused)
self.assertEqual(
obj._file_import_options.GetSerialisableTuple(),
dupe_obj._file_import_options.GetSerialisableTuple())
self.assertEqual(
obj._tag_import_options.GetSerialisableTuple(),
dupe_obj._tag_import_options.GetSerialisableTuple())
self.assertEqual(obj._no_work_until, dupe_obj._no_work_until)
sub = ClientImportSubscriptions.Subscription('test sub')
self._dump_and_load_and_test(sub, test)
gug_key_and_name = (HydrusData.GenerateKey(), 'muh test gug')
query_headers = []
q = ClientImportSubscriptionQuery.SubscriptionQueryHeader()
q.SetQueryText('test query')
query_headers.append(q)
q = ClientImportSubscriptionQuery.SubscriptionQueryHeader()
q.SetQueryText('test query 2')
query_headers.append(q)
checker_options = ClientImportOptions.CheckerOptions()
initial_file_limit = 100
periodic_file_limit = 50
paused = False
file_import_options = FileImportOptions.FileImportOptions()
service_tag_import_options = TagImportOptions.ServiceTagImportOptions(
get_tags=False,
additional_tags={'test additional tag', 'and another'})
tag_import_options = TagImportOptions.TagImportOptions(
service_keys_to_service_tag_import_options={
HydrusData.GenerateKey(): service_tag_import_options
})
no_work_until = HydrusData.GetNow() - 86400 * 20
sub.SetTuple(gug_key_and_name, checker_options, initial_file_limit,
periodic_file_limit, paused, file_import_options,
tag_import_options, no_work_until)
sub.SetQueryHeaders(query_headers)
self.assertEqual(sub.GetGUGKeyAndName(), gug_key_and_name)
self.assertEqual(sub.GetTagImportOptions(), tag_import_options)
self.assertEqual(sub.GetQueryHeaders(), query_headers)
self.assertEqual(sub._paused, False)
sub.PauseResume()
self.assertEqual(sub._paused, True)
sub.PauseResume()
self.assertEqual(sub._paused, False)
self._dump_and_load_and_test(sub, test)
