def _InitialiseFromSerialisableInfo( self, serialisable_info ):
(
self._query_log_container_name,
self._query_text,
self._display_name,
self._check_now,
self._last_check_time,
self._next_check_time,
self._paused,
self._checker_status,
self._query_log_container_status,
serialisable_file_seed_cache_status,
serialisable_tag_import_options,
self._raw_file_velocity,
self._pretty_file_velocity,
serialisable_example_file_seed,
serialisable_example_gallery_seed
) = serialisable_info
self._file_seed_cache_status = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_file_seed_cache_status )
self._tag_import_options = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_tag_import_options )
self._example_file_seed = HydrusSerialisable.CreateFromNoneableSerialisableTuple( serialisable_example_file_seed )
self._example_gallery_seed = HydrusSerialisable.CreateFromNoneableSerialisableTuple( serialisable_example_gallery_seed )
def _InitialiseFromSerialisableInfo( self, serialisable_info ):
( self._query, self._display_name, self._check_now, self._last_check_time, self._next_check_time, self._paused, self._status, serialisable_gallery_seed_log, serialisable_file_seed_cache, serialisable_tag_import_options ) = serialisable_info
self._gallery_seed_log = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_gallery_seed_log )
self._file_seed_cache = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_file_seed_cache )
self._tag_import_options = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_tag_import_options )
def _GetSerialisableInfo( self ):
serialisable_file_seed_cache_status = self._file_seed_cache_status.GetSerialisableTuple()
serialisable_tag_import_options = self._tag_import_options.GetSerialisableTuple()
serialisable_example_file_seed = HydrusSerialisable.GetNoneableSerialisableTuple( self._example_file_seed )
serialisable_example_gallery_seed = HydrusSerialisable.GetNoneableSerialisableTuple( self._example_gallery_seed )
return (
self._query_log_container_name,
self._query_text,
self._display_name,
self._check_now,
self._last_check_time,
self._next_check_time,
self._paused,
self._checker_status,
self._query_log_container_status,
serialisable_file_seed_cache_status,
serialisable_tag_import_options,
self._raw_file_velocity,
self._pretty_file_velocity,
serialisable_example_file_seed,
serialisable_example_gallery_seed
)
def _InitialiseFromSerialisableInfo(self, serialisable_info):
(all_serialisable_trackers, all_serialisable_rules) = serialisable_info
for (serialisable_network_context,
serialisable_tracker) in all_serialisable_trackers:
network_context = HydrusSerialisable.CreateFromSerialisableTuple(
serialisable_network_context)
tracker = HydrusSerialisable.CreateFromSerialisableTuple(
serialisable_tracker)
self._network_contexts_to_bandwidth_trackers[
network_context] = tracker
for (serialisable_network_context,
serialisable_rules) in all_serialisable_rules:
network_context = HydrusSerialisable.CreateFromSerialisableTuple(
serialisable_network_context)
rules = HydrusSerialisable.CreateFromSerialisableTuple(
serialisable_rules)
if network_context.context_type == CC.NETWORK_CONTEXT_DOWNLOADER: # no longer use this
continue
self._network_contexts_to_bandwidth_rules[network_context] = rules
def _InitialiseFromSerialisableInfo(self, serialisable_info):
(self._path, self._mimes, serialisable_file_import_options,
serialisable_tag_import_options,
serialisable_tag_service_keys_to_filename_tagging_options,
action_pairs, action_location_pairs, self._period,
self._check_regularly, serialisable_file_seed_cache,
self._last_checked, self._paused, self._check_now,
self._show_working_popup, self._publish_files_to_popup_button,
self._publish_files_to_page) = serialisable_info
self._actions = dict(action_pairs)
self._action_locations = dict(action_location_pairs)
self._file_import_options = HydrusSerialisable.CreateFromSerialisableTuple(
serialisable_file_import_options)
self._tag_import_options = HydrusSerialisable.CreateFromSerialisableTuple(
serialisable_tag_import_options)
self._tag_service_keys_to_filename_tagging_options = dict([
(bytes.fromhex(encoded_service_key),
HydrusSerialisable.CreateFromSerialisableTuple(
serialisable_filename_tagging_options))
for (encoded_service_key, serialisable_filename_tagging_options
) in serialisable_tag_service_keys_to_filename_tagging_options
])
self._file_seed_cache = HydrusSerialisable.CreateFromSerialisableTuple(
serialisable_file_seed_cache)
def test_basics(self):
def test(obj, dupe_obj):
self.assertEqual(len(list(obj.items())),
len(list(dupe_obj.items())))
for (key, value) in list(obj.items()):
self.assertEqual(value, dupe_obj[key])
#
d = HydrusSerialisable.SerialisableDictionary()
d[1] = 2
d[3] = 'test1'
d['test2'] = 4
d['test3'] = 5
d[6] = HydrusSerialisable.SerialisableDictionary(
{i: 'test' + str(i)
for i in range(20)})
d[ClientSearch.Predicate(ClientSearch.PREDICATE_TYPE_TAG,
'test pred 1')] = 56
d[ClientSearch.Predicate(
ClientSearch.PREDICATE_TYPE_TAG,
'test pred 2')] = HydrusSerialisable.SerialisableList([
ClientSearch.Predicate(ClientSearch.PREDICATE_TYPE_TAG,
'test' + str(i)) for i in range(10)
])
self.assertEqual(len(list(d.keys())), 7)
for (key, value) in list(d.items()):
self.assertEqual(d[key], value)
self._dump_and_load_and_test(d, test)
#
db = HydrusSerialisable.SerialisableBytesDictionary()
db[HydrusData.GenerateKey()] = HydrusData.GenerateKey()
db[HydrusData.GenerateKey()] = [
HydrusData.GenerateKey() for i in range(10)
]
db[1] = HydrusData.GenerateKey()
db[2] = [HydrusData.GenerateKey() for i in range(10)]
self.assertEqual(len(list(db.keys())), 4)
for (key, value) in list(db.items()):
self.assertEqual(db[key], value)
self._dump_and_load_and_test(db, test)
def _InitialiseFromSerialisableInfo( self, serialisable_info ):
( serialisable_gallery_seed_log, serialisable_file_seed_cache, serialisable_file_import_options, serialisable_tag_import_options, self._paused ) = serialisable_info
self._gallery_seed_log = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_gallery_seed_log )
self._file_seed_cache = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_file_seed_cache )
self._file_import_options = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_file_import_options )
self._tag_import_options = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_tag_import_options )
def _InitialiseFromSerialisableInfo(self, serialisable_info):
(serialisable_network_context,
serialisable_bandwidth_tracker) = serialisable_info
self.network_context = HydrusSerialisable.CreateFromSerialisableTuple(
serialisable_network_context)
self.bandwidth_tracker = HydrusSerialisable.CreateFromSerialisableTuple(
serialisable_bandwidth_tracker)
def _InitialiseFromSerialisableInfo( self, serialisable_info ):
( serialisable_pending_jobs, serialisable_gallery_seed_log, serialisable_file_seed_cache, serialisable_file_import_options, self._formula_name, self._gallery_paused, self._files_paused ) = serialisable_info
self._pending_jobs = [ ( url, HydrusSerialisable.CreateFromSerialisableTuple( serialisable_simple_downloader_formula ) ) for ( url, serialisable_simple_downloader_formula ) in serialisable_pending_jobs ]
self._gallery_seed_log = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_gallery_seed_log )
self._file_seed_cache = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_file_seed_cache )
self._file_import_options = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_file_import_options )
def _InitialiseFromSerialisableInfo(self, serialisable_info):
(serialisable_file_seed_cache, serialisable_options,
self._delete_after_success, self._paused) = serialisable_info
self._file_seed_cache = HydrusSerialisable.CreateFromSerialisableTuple(
serialisable_file_seed_cache)
self._file_import_options = HydrusSerialisable.CreateFromSerialisableTuple(
serialisable_options)
def _InitialiseFromSerialisableInfo(self, serialisable_info):
(serialisable_gallery_seed_log,
serialisable_file_seed_cache) = serialisable_info
self._gallery_seed_log = HydrusSerialisable.CreateFromSerialisableTuple(
serialisable_gallery_seed_log)
self._file_seed_cache = HydrusSerialisable.CreateFromSerialisableTuple(
serialisable_file_seed_cache)
def _UpdateSerialisableInfo( self, version, old_serialisable_info ):
if version == 1:
serialisable_tag_display_types_to_service_keys_to_tag_filters = old_serialisable_info
tag_autocomplete_options_list = HydrusSerialisable.SerialisableList()
new_serialisable_info = [
serialisable_tag_display_types_to_service_keys_to_tag_filters,
tag_autocomplete_options_list.GetSerialisableTuple()
]
return ( 2, new_serialisable_info )
if version == 2:
[
serialisable_tag_display_types_to_service_keys_to_tag_filters,
serialisable_tag_autocomplete_options
] = old_serialisable_info
service_keys_to_ordered_sibling_service_keys = collections.defaultdict( list )
service_keys_to_ordered_parent_service_keys = collections.defaultdict( list )
serialisable_service_keys_to_ordered_sibling_service_keys = HydrusSerialisable.SerialisableBytesDictionary( service_keys_to_ordered_sibling_service_keys ).GetSerialisableTuple()
serialisable_service_keys_to_ordered_parent_service_keys = HydrusSerialisable.SerialisableBytesDictionary( service_keys_to_ordered_parent_service_keys ).GetSerialisableTuple()
new_serialisable_info = [
serialisable_tag_display_types_to_service_keys_to_tag_filters,
serialisable_tag_autocomplete_options,
serialisable_service_keys_to_ordered_sibling_service_keys,
serialisable_service_keys_to_ordered_parent_service_keys
]
return ( 3, new_serialisable_info )
if version == 3:
# took it out again lmao, down to the db
[
serialisable_tag_display_types_to_service_keys_to_tag_filters,
serialisable_tag_autocomplete_options,
serialisable_service_keys_to_ordered_sibling_service_keys,
serialisable_service_keys_to_ordered_parent_service_keys
] = old_serialisable_info
new_serialisable_info = [
serialisable_tag_display_types_to_service_keys_to_tag_filters,
serialisable_tag_autocomplete_options
]
return ( 4, new_serialisable_info )
def _InitialiseFromSerialisableInfo(self, serialisable_info):
for (serialisable_shortcut, serialisable_command) in serialisable_info:
shortcut = HydrusSerialisable.CreateFromSerialisableTuple(
serialisable_shortcut)
command = HydrusSerialisable.CreateFromSerialisableTuple(
serialisable_command)
self._shortcuts_to_commands[shortcut] = command
def _InitialiseFromSerialisableInfo( self, serialisable_info ):
( serialisable_watchers, self._highlighted_watcher_url, serialisable_checker_options, serialisable_file_import_options, serialisable_tag_import_options ) = serialisable_info
self._watchers = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_watchers )
self._watcher_keys_to_watchers = { watcher.GetWatcherKey() : watcher for watcher in self._watchers }
self._checker_options = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_checker_options )
self._file_import_options = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_file_import_options )
self._tag_import_options = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_tag_import_options )
def DumpHydrusArgsToNetworkBytes(args):
if not isinstance(args, HydrusSerialisable.SerialisableBase):
args = HydrusSerialisable.SerialisableDictionary(args)
for param_name in BYTE_PARAMS:
if param_name in args:
args[param_name] = args[param_name].hex()
for param_name in JSON_BYTE_LIST_PARAMS:
if param_name in args:
args[param_name] = [item.hex() for item in args[param_name]]
if 'account_types' in args:
args['account_types'] = HydrusSerialisable.SerialisableList(
args['account_types'])
if 'account' in args:
args[
'account'] = HydrusNetwork.Account.GenerateSerialisableTupleFromAccount(
args['account'])
if 'accounts' in args:
args['accounts'] = list(
map(HydrusNetwork.Account.GenerateSerialisableTupleFromAccount,
args['accounts']))
if 'service_keys_to_access_keys' in args:
args['service_keys_to_access_keys'] = [
(service_key.hex(), access_key.hex())
for (service_key, access_key
) in list(args['service_keys_to_access_keys'].items())
]
if 'services' in args:
args['services'] = [
service.ToSerialisableTuple() for service in args['services']
]
network_bytes = args.DumpToNetworkBytes()
return network_bytes
def _InitialiseFromSerialisableInfo(self, serialisable_info):
(self._get_tags, serialisable_get_tags_filter, self._additional_tags,
self._to_new_files, self._to_already_in_inbox,
self._to_already_in_archive, self._only_add_existing_tags,
serialisable_only_add_existing_tags_filter,
self._get_tags_overwrite_deleted,
self._additional_tags_overwrite_deleted) = serialisable_info
self._get_tags_filter = HydrusSerialisable.CreateFromSerialisableTuple(
serialisable_get_tags_filter)
self._only_add_existing_tags_filter = HydrusSerialisable.CreateFromSerialisableTuple(
serialisable_only_add_existing_tags_filter)
def _dump_and_load_and_test(self, obj, test_func):
serialisable_tuple = obj.GetSerialisableTuple()
self.assertIsInstance(serialisable_tuple, tuple)
if isinstance(obj, HydrusSerialisable.SerialisableBaseNamed):
(serialisable_type, name, version,
serialisable_info) = serialisable_tuple
elif isinstance(obj, HydrusSerialisable.SerialisableBase):
(serialisable_type, version,
serialisable_info) = serialisable_tuple
self.assertEqual(serialisable_type, obj.SERIALISABLE_TYPE)
self.assertEqual(version, obj.SERIALISABLE_VERSION)
dupe_obj = HydrusSerialisable.CreateFromSerialisableTuple(
serialisable_tuple)
self.assertIsNot(obj, dupe_obj)
test_func(obj, dupe_obj)
#
json_string = obj.DumpToString()
self.assertIsInstance(json_string, str)
dupe_obj = HydrusSerialisable.CreateFromString(json_string)
self.assertIsNot(obj, dupe_obj)
test_func(obj, dupe_obj)
#
network_bytes = obj.DumpToNetworkBytes()
self.assertIsInstance(network_bytes, bytes)
dupe_obj = HydrusSerialisable.CreateFromNetworkBytes(network_bytes)
self.assertIsNot(obj, dupe_obj)
test_func(obj, dupe_obj)
def _UpdateSerialisableInfo( self, version, old_serialisable_info ):
if version == 1:
( serialisable_file_seed_cache, serialisable_options, serialisable_paths_to_tags, delete_after_success, paused ) = old_serialisable_info
file_seed_cache = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_file_seed_cache )
paths_to_additional_service_keys_to_tags = { path : { bytes.fromhex( service_key ) : tags for ( service_key, tags ) in service_keys_to_tags.items() } for ( path, service_keys_to_tags ) in serialisable_paths_to_tags.items() }
for file_seed in file_seed_cache.GetFileSeeds():
path = file_seed.file_seed_data
if path in paths_to_additional_service_keys_to_tags:
file_seed.SetExternalAdditionalServiceKeysToTags( paths_to_additional_service_keys_to_tags[ path ] )
serialisable_file_seed_cache = file_seed_cache.GetSerialisableTuple()
new_serialisable_info = ( serialisable_file_seed_cache, serialisable_options, delete_after_success, paused )
return ( 2, new_serialisable_info )
def _InitialiseFromSerialisableInfo( self, serialisable_info ):
( pre_import_options, post_import_options, serialisable_presentation_import_options ) = serialisable_info
( self._exclude_deleted, self._do_not_check_known_urls_before_importing, self._do_not_check_hashes_before_importing, self._allow_decompression_bombs, self._min_size, self._max_size, self._max_gif_size, self._min_resolution, self._max_resolution ) = pre_import_options
( self._automatic_archive, self._associate_primary_urls, self._associate_source_urls ) = post_import_options
self._presentation_import_options = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_presentation_import_options )
def _ImportPngs( self, paths ):
for path in paths:
try:
payload = ClientSerialisable.LoadFromPng( path )
except Exception as e:
QW.QMessageBox.critical( self, 'Error', str(e) )
return
try:
obj = HydrusSerialisable.CreateFromNetworkBytes( payload )
self._ImportObject( obj )
except:
QW.QMessageBox.critical( self, 'Error', 'I could not understand what was encoded in the file!' )
return
def _ImportFromClipboard( self ):
try:
raw_text = HG.client_controller.GetClipboardText()
except HydrusExceptions.DataMissing as e:
QW.QMessageBox.critical( self, 'Error', str(e) )
return
try:
obj = HydrusSerialisable.CreateFromString( raw_text )
self._ImportObject( obj )
except Exception as e:
QW.QMessageBox.critical( self, 'Error', 'I could not understand what was in the clipboard' )
self._listctrl.Sort()
def _GetExportObject( self ):
if self._custom_get_callable is None:
to_export = HydrusSerialisable.SerialisableList()
for obj in self._listctrl.GetData( only_selected = True ):
to_export.append( obj )
else:
to_export = [ self._custom_get_callable() ]
if len( to_export ) == 0:
return None
elif len( to_export ) == 1:
return to_export[0]
else:
return to_export
def _InitCaches(self):
if self._c.execute('SELECT 1 FROM sqlite_master WHERE name = ?;',
('services', )).fetchone() is not None:
all_data = self._c.execute(
'SELECT service_id, service_key, service_type, name, dictionary_string FROM services;'
).fetchall()
for (service_id, service_key, service_type, name,
dictionary_string) in all_data:
dictionary = HydrusSerialisable.CreateFromString(
dictionary_string)
service = ClientServices.GenerateService(
service_key, service_type, name, dictionary)
self._service_ids_to_services[service_id] = service
self._service_keys_to_service_ids[service_key] = service_id
self.local_file_service_id = self.GetServiceId(
CC.LOCAL_FILE_SERVICE_KEY)
self.local_update_service_id = self.GetServiceId(
CC.LOCAL_UPDATE_SERVICE_KEY)
self.trash_service_id = self.GetServiceId(CC.TRASH_SERVICE_KEY)
self.combined_local_file_service_id = self.GetServiceId(
CC.COMBINED_LOCAL_FILE_SERVICE_KEY)
self.combined_file_service_id = self.GetServiceId(
CC.COMBINED_FILE_SERVICE_KEY)
self.combined_tag_service_id = self.GetServiceId(
CC.COMBINED_TAG_SERVICE_KEY)
def _ImportJSONs(self, paths):
for path in paths:
try:
with open(path, 'r', encoding='utf-8') as f:
payload = f.read()
except Exception as e:
QW.QMessageBox.critical(self, 'Error', str(e))
return
try:
obj = HydrusSerialisable.CreateFromString(payload)
self._ImportObject(obj)
except:
QW.QMessageBox.critical(
self, 'Error',
'I could not understand what was encoded in "{}"!'.format(
path))
return
def _InitialiseFromSerialisableInfo(self, serialisable_info):
serialisable_processing_steps = serialisable_info
self._processing_steps = list(
HydrusSerialisable.CreateFromSerialisableTuple(
serialisable_processing_steps))
def Compact( self, compact_before_this_source_time ):
with self._lock:
if len( self._gallery_seeds ) <= self.COMPACT_NUMBER:
return
new_gallery_seeds = HydrusSerialisable.SerialisableList()
for gallery_seed in self._gallery_seeds[:-self.COMPACT_NUMBER]:
still_to_do = gallery_seed.status == CC.STATUS_UNKNOWN
still_relevant = gallery_seed.created > compact_before_this_source_time
if still_to_do or still_relevant:
new_gallery_seeds.append( gallery_seed )
new_gallery_seeds.extend( self._gallery_seeds[-self.COMPACT_NUMBER:] )
self._gallery_seeds = new_gallery_seeds
self._gallery_seeds_to_indices = { gallery_seed : index for ( index, gallery_seed ) in enumerate( self._gallery_seeds ) }
self._SetStatusDirty()
def _InitialiseFromSerialisableInfo( self, serialisable_info ):
with self._lock:
self._gallery_seeds = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_info )
self._gallery_seeds_to_indices = { gallery_seed : index for ( index, gallery_seed ) in enumerate( self._gallery_seeds ) }
def _GetSerialisableInfo(self):
serialisable_tag_display_types_to_service_keys_to_tag_filters = []
for (
tag_display_type, service_keys_to_tag_filters
) in self._tag_display_types_to_service_keys_to_tag_filters.items():
serialisable_service_keys_to_tag_filters = [
(service_key.hex(), tag_filter.GetSerialisableTuple())
for (service_key,
tag_filter) in service_keys_to_tag_filters.items()
]
serialisable_tag_display_types_to_service_keys_to_tag_filters.append(
(tag_display_type, serialisable_service_keys_to_tag_filters))
serialisable_tag_autocomplete_options = HydrusSerialisable.SerialisableList(
self._tag_service_keys_to_tag_autocomplete_options.values(
)).GetSerialisableTuple()
serialisable_info = [
serialisable_tag_display_types_to_service_keys_to_tag_filters,
serialisable_tag_autocomplete_options
]
return serialisable_info
def _InitialiseFromSerialisableInfo(self, serialisable_info):
[
serialisable_tag_display_types_to_service_keys_to_tag_filters,
serialisable_tag_autocomplete_options
] = serialisable_info
for (
tag_display_type, serialisable_service_keys_to_tag_filters
) in serialisable_tag_display_types_to_service_keys_to_tag_filters:
for (serialisable_service_key, serialisable_tag_filter
) in serialisable_service_keys_to_tag_filters:
service_key = bytes.fromhex(serialisable_service_key)
tag_filter = HydrusSerialisable.CreateFromSerialisableTuple(
serialisable_tag_filter)
self._tag_display_types_to_service_keys_to_tag_filters[
tag_display_type][service_key] = tag_filter
self._tag_service_keys_to_tag_autocomplete_options = {
tag_autocomplete_options.GetServiceKey(): tag_autocomplete_options
for tag_autocomplete_options in HydrusSerialisable.
CreateFromSerialisableTuple(serialisable_tag_autocomplete_options)
}
def _InitialiseFromSerialisableInfo( self, serialisable_info ):
serialisable_api_permissions_objects = serialisable_info
api_permissions_objects = [ HydrusSerialisable.CreateFromSerialisableTuple( serialisable_api_permissions ) for serialisable_api_permissions in serialisable_api_permissions_objects ]
self._access_keys_to_permissions = { api_permissions.GetAccessKey() : api_permissions for api_permissions in api_permissions_objects }
