def _InitialiseFromSerialisableInfo(self, serialisable_info):
(self._path, self._export_type, self._delete_from_client_after_export,
serialisable_file_search_context, self._run_regularly, self._period,
self._phrase, self._last_checked, self._paused, self._run_now,
self._last_error) = serialisable_info
if self._export_type == HC.EXPORT_FOLDER_TYPE_SYNCHRONISE:
self._delete_from_client_after_export = False
self._file_search_context = HydrusSerialisable.CreateFromSerialisableTuple(
serialisable_file_search_context)
def _InitialiseFromSerialisableInfo(self, serialisable_info):
for (tag_display_type,
serialisable_service_keys_to_tag_filters) in serialisable_info:
for (serialisable_service_key, serialisable_tag_filter
) in serialisable_service_keys_to_tag_filters:
service_key = bytes.fromhex(serialisable_service_key)
tag_filter = HydrusSerialisable.CreateFromSerialisableTuple(
serialisable_tag_filter)
self._tag_display_types_to_service_keys_to_tag_filters[
tag_display_type][service_key] = tag_filter
def _InitialiseFromSerialisableInfo(self, serialisable_info):
serialisable_api_permissions_objects = serialisable_info
api_permissions_objects = [
HydrusSerialisable.CreateFromSerialisableTuple(
serialisable_api_permissions) for serialisable_api_permissions
in serialisable_api_permissions_objects
]
self._access_keys_to_permissions = {
api_permissions.GetAccessKey(): api_permissions
for api_permissions in api_permissions_objects
}
def _dump_and_load_and_test(self, obj, test_func):
serialisable_tuple = obj.GetSerialisableTuple()
self.assertIsInstance(serialisable_tuple, tuple)
if isinstance(obj, HydrusSerialisable.SerialisableBaseNamed):
(serialisable_type, name, version,
serialisable_info) = serialisable_tuple
elif isinstance(obj, HydrusSerialisable.SerialisableBase):
(serialisable_type, version,
serialisable_info) = serialisable_tuple
self.assertEqual(serialisable_type, obj.SERIALISABLE_TYPE)
self.assertEqual(version, obj.SERIALISABLE_VERSION)
dupe_obj = HydrusSerialisable.CreateFromSerialisableTuple(
serialisable_tuple)
self.assertIsNot(obj, dupe_obj)
test_func(obj, dupe_obj)
#
json_string = obj.DumpToString()
self.assertIsInstance(json_string, str)
dupe_obj = HydrusSerialisable.CreateFromString(json_string)
self.assertIsNot(obj, dupe_obj)
test_func(obj, dupe_obj)
#
network_bytes = obj.DumpToNetworkBytes()
self.assertIsInstance(network_bytes, bytes)
dupe_obj = HydrusSerialisable.CreateFromNetworkBytes(network_bytes)
self.assertIsNot(obj, dupe_obj)
test_func(obj, dupe_obj)
def _InitialiseFromSerialisableInfo( self, serialisable_info ):
(
self.url,
self._can_generate_more_pages,
serialisable_external_filterable_tags,
serialisable_external_additional_service_keys_to_tags,
self.created,
self.modified,
self.status,
self.note,
self._referral_url
) = serialisable_info
self._external_filterable_tags = set( serialisable_external_filterable_tags )
self._external_additional_service_keys_to_tags = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_external_additional_service_keys_to_tags )
def GetJSONDump( self, dump_type ):
result = self._c.execute( 'SELECT version, dump FROM json_dumps WHERE dump_type = ?;', ( dump_type, ) ).fetchone()
if result is None:
return result
else:
( version, dump ) = result
try:
if isinstance( dump, bytes ):
dump = str( dump, 'utf-8' )
serialisable_info = json.loads( dump )
except:
self._c.execute( 'DELETE FROM json_dumps WHERE dump_type = ?;', ( dump_type, ) )
self._cursor_transaction_wrapper.CommitAndBegin()
DealWithBrokenJSONDump( self._db_dir, dump, 'dump_type {}'.format( dump_type ) )
obj = HydrusSerialisable.CreateFromSerialisableTuple( ( dump_type, version, serialisable_info ) )
if dump_type == HydrusSerialisable.SERIALISABLE_TYPE_NETWORK_SESSION_MANAGER:
session_containers = self.GetJSONDumpNamed( HydrusSerialisable.SERIALISABLE_TYPE_NETWORK_SESSION_MANAGER_SESSION_CONTAINER )
obj.SetSessionContainers( session_containers )
elif dump_type == HydrusSerialisable.SERIALISABLE_TYPE_NETWORK_BANDWIDTH_MANAGER:
tracker_containers = self.GetJSONDumpNamed( HydrusSerialisable.SERIALISABLE_TYPE_NETWORK_BANDWIDTH_MANAGER_TRACKER_CONTAINER )
obj.SetTrackerContainers( tracker_containers )
return obj
def _InitialiseFromSerialisableInfo(self, serialisable_info):
(serialisable_tag_service_actions, serialisable_rating_service_actions,
self._sync_archive, self._sync_urls_action) = serialisable_info
self._tag_service_actions = [
(bytes.fromhex(serialisable_service_key), action,
HydrusSerialisable.CreateFromSerialisableTuple(
serialisable_tag_filter))
for (serialisable_service_key, action,
serialisable_tag_filter) in serialisable_tag_service_actions
]
self._rating_service_actions = [
(bytes.fromhex(serialisable_service_key), action)
for (serialisable_service_key,
action) in serialisable_rating_service_actions
]
def _InitialiseFromSerialisableInfo( self, serialisable_info ):
( self._url, serialisable_gallery_seed_log, serialisable_file_seed_cache, serialisable_fixed_service_keys_to_tags, serialisable_checker_options, serialisable_file_import_options, serialisable_tag_import_options, self._last_check_time, self._files_paused, self._checking_paused, self._checking_status, self._subject, self._no_work_until, self._no_work_until_reason, self._creation_time ) = serialisable_info
self._fixed_service_keys_to_tags = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_fixed_service_keys_to_tags )
self._gallery_seed_log = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_gallery_seed_log )
self._file_seed_cache = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_file_seed_cache )
self._checker_options = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_checker_options )
self._file_import_options = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_file_import_options )
self._tag_import_options = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_tag_import_options )
def _InitialiseFromSerialisableInfo(self, serialisable_info):
(serialisable_network_context, pickled_session_hex) = serialisable_info
self.network_context = HydrusSerialisable.CreateFromSerialisableTuple(
serialisable_network_context)
try:
self.session = pickle.loads(bytes.fromhex(pickled_session_hex))
except:
# a new version of requests messed this up lad, so reset
self._InitialiseEmptySession()
self.session.cookies.clear_session_cookies()
def _InitialiseFromSerialisableInfo(self, serialisable_info):
[
serialisable_service_key,
serialisable_write_autocomplete_tag_domain,
self._override_write_autocomplete_location_context,
serialisable_write_autocomplete_location_context,
self._search_namespaces_into_full_tags,
self._namespace_bare_fetch_all_allowed,
self._namespace_fetch_all_allowed, self._fetch_all_allowed,
self._fetch_results_automatically,
self._exact_match_character_threshold
] = serialisable_info
self._service_key = bytes.fromhex(serialisable_service_key)
self._write_autocomplete_tag_domain = bytes.fromhex(
serialisable_write_autocomplete_tag_domain)
self._write_autocomplete_location_context = HydrusSerialisable.CreateFromSerialisableTuple(
serialisable_write_autocomplete_location_context)
def _InitialiseFromSerialisableInfo( self, serialisable_info ):
[
serialisable_tag_display_types_to_service_keys_to_tag_filters,
serialisable_tag_autocomplete_options
] = serialisable_info
for ( tag_display_type, serialisable_service_keys_to_tag_filters ) in serialisable_tag_display_types_to_service_keys_to_tag_filters:
for ( serialisable_service_key, serialisable_tag_filter ) in serialisable_service_keys_to_tag_filters:
service_key = bytes.fromhex( serialisable_service_key )
tag_filter = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_tag_filter )
self._tag_display_types_to_service_keys_to_tag_filters[ tag_display_type ][ service_key ] = tag_filter
self._tag_service_keys_to_tag_autocomplete_options = { tag_autocomplete_options.GetServiceKey() : tag_autocomplete_options for tag_autocomplete_options in HydrusSerialisable.CreateFromSerialisableTuple( serialisable_tag_autocomplete_options ) }
def _InitialiseFromSerialisableInfo( self, serialisable_info ):
( serialisable_network_context, pickled_cookies_hex ) = serialisable_info
self.network_context = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_network_context )
self._InitialiseEmptySession()
try:
cookies = pickle.loads( bytes.fromhex( pickled_cookies_hex ) )
self.session.cookies = cookies
except:
HydrusData.Print( "Could not load and set cookies for session {}".format( self.network_context ) )
self.session.cookies.clear_session_cookies()
def _InitialiseFromSerialisableInfo( self, serialisable_info ):
serialisable_network_contexts_to_sessions = serialisable_info
for ( serialisable_network_context, pickled_session_hex ) in serialisable_network_contexts_to_sessions:
network_context = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_network_context )
try:
session = pickle.loads( bytes.fromhex( pickled_session_hex ) )
except:
# new version of requests uses a diff format, wew
continue
session.cookies.clear_session_cookies()
self._network_contexts_to_sessions[ network_context ] = session
def GetPageTuple(serialisable_page_tuple):
(page_type, serialisable_page_data) = serialisable_page_tuple
if page_type == 'pages':
(name, serialisable_page_tuples) = serialisable_page_data
page_tuples = []
for spt in serialisable_page_tuples:
try:
page_tuples.append(GetPageTuple(spt))
except Exception as e:
handle_e(spt, e)
page_data = (name, page_tuples)
elif page_type == 'page':
(serialisable_management_controller,
serialisable_hashes) = serialisable_page_data
management_controller = HydrusSerialisable.CreateFromSerialisableTuple(
serialisable_management_controller)
hashes = [bytes.fromhex(hash) for hash in serialisable_hashes]
page_data = (management_controller, hashes)
page_tuple = (page_type, page_data)
return page_tuple
def _UpdateSerialisableInfo(self, version, old_serialisable_info):
if version == 1:
(serialisable_file_seed_cache, serialisable_options,
serialisable_paths_to_tags, delete_after_success,
paused) = old_serialisable_info
file_seed_cache = HydrusSerialisable.CreateFromSerialisableTuple(
serialisable_file_seed_cache)
paths_to_service_keys_to_tags = {
path: {
bytes.fromhex(service_key): tags
for (service_key, tags) in service_keys_to_tags.items()
}
for (path, service_keys_to_tags
) in serialisable_paths_to_tags.items()
}
for file_seed in file_seed_cache.GetFileSeeds():
path = file_seed.file_seed_data
if path in paths_to_service_keys_to_tags:
file_seed.SetFixedServiceKeysToTags(
paths_to_service_keys_to_tags[path])
serialisable_file_seed_cache = file_seed_cache.GetSerialisableTuple(
)
new_serialisable_info = (serialisable_file_seed_cache,
serialisable_options,
delete_after_success, paused)
return (2, new_serialisable_info)
def GetJSONDumpNamed(self, dump_type, dump_name=None, timestamp=None):
if dump_name is None:
results = self._c.execute(
'SELECT dump_name, version, dump, timestamp FROM json_dumps_named WHERE dump_type = ?;',
(dump_type, )).fetchall()
objs = []
for (dump_name, version, dump, object_timestamp) in results:
try:
if isinstance(dump, bytes):
dump = str(dump, 'utf-8')
serialisable_info = json.loads(dump)
objs.append(
HydrusSerialisable.CreateFromSerialisableTuple(
(dump_type, dump_name, version,
serialisable_info)))
except:
self._c.execute(
'DELETE FROM json_dumps_named WHERE dump_type = ? AND dump_name = ? AND timestamp = ?;',
(dump_type, dump_name, object_timestamp))
self._cursor_transaction_wrapper.CommitAndBegin()
DealWithBrokenJSONDump(
self._db_dir, dump,
'dump_type {} dump_name {} timestamp {}'.format(
dump_type, dump_name[:10], timestamp))
return objs
else:
if timestamp is None:
result = self._c.execute(
'SELECT version, dump, timestamp FROM json_dumps_named WHERE dump_type = ? AND dump_name = ? ORDER BY timestamp DESC;',
(dump_type, dump_name)).fetchone()
else:
result = self._c.execute(
'SELECT version, dump, timestamp FROM json_dumps_named WHERE dump_type = ? AND dump_name = ? AND timestamp = ?;',
(dump_type, dump_name, timestamp)).fetchone()
if result is None:
raise HydrusExceptions.DataMissing(
'Could not find the object of type "{}" and name "{}" and timestamp "{}".'
.format(dump_type, dump_name, str(timestamp)))
(version, dump, object_timestamp) = result
try:
if isinstance(dump, bytes):
dump = str(dump, 'utf-8')
serialisable_info = json.loads(dump)
except:
self._c.execute(
'DELETE FROM json_dumps_named WHERE dump_type = ? AND dump_name = ? AND timestamp = ?;',
(dump_type, dump_name, object_timestamp))
self._cursor_transaction_wrapper.CommitAndBegin()
DealWithBrokenJSONDump(
self._db_dir, dump,
'dump_type {} dump_name {} timestamp {}'.format(
dump_type, dump_name[:10], object_timestamp))
return HydrusSerialisable.CreateFromSerialisableTuple(
(dump_type, dump_name, version, serialisable_info))
def GetHashedJSONDumps(self, hashes):
shown_missing_dump_message = False
shown_broken_dump_message = False
hashes_to_objs = {}
for hash in hashes:
result = self._c.execute(
'SELECT version, dump_type, dump FROM json_dumps_hashed WHERE hash = ?;',
(sqlite3.Binary(hash), )).fetchone()
if result is None:
if not shown_missing_dump_message:
message = 'A hashed serialised object was missing! Its hash is "{}".'.format(
hash.hex())
message += os.linesep * 2
message += 'This error could be due to several factors, but is most likely a hard drive fault (perhaps your computer recently had a bad power cut?).'
message += os.linesep * 2
message += 'Your client may have lost one or more session pages.'
message += os.linesep * 2
message += 'Please review the \'help my db is broke.txt\' file in your install_dir/db directory as background reading, and if the situation or fix here is not obvious, please contact hydrus dev.'
HydrusData.ShowText(message)
shown_missing_dump_message = True
HydrusData.Print(
'Was asked to fetch named JSON object "{}", but it was missing!'
.format(hash.hex()))
continue
(version, dump_type, dump) = result
try:
if isinstance(dump, bytes):
dump = str(dump, 'utf-8')
serialisable_info = json.loads(dump)
except:
self._c.execute(
'DELETE FROM json_dumps_hashed WHERE hash = ?;',
(sqlite3.Binary(hash), ))
self._cursor_transaction_wrapper.CommitAndBegin()
ExportBrokenHashedJSONDump(
self._db_dir, dump,
'hash {} dump_type {}'.format(hash.hex(), dump_type))
if not shown_broken_dump_message:
message = 'A hashed serialised object failed to load! Its hash is "{}".'.format(
hash.hex())
message += os.linesep * 2
message += 'This error could be due to several factors, but is most likely a hard drive fault (perhaps your computer recently had a bad power cut?).'
message += os.linesep * 2
message += 'The database has attempted to delete the broken object, and the object\'s dump written to your database directory. Your client may have lost one or more session pages.'
message += os.linesep * 2
message += 'Please review the \'help my db is broke.txt\' file in your install_dir/db directory as background reading, and if the situation or fix here is not obvious, please contact hydrus dev.'
HydrusData.ShowText(message)
shown_broken_dump_message = True
HydrusData.Print(
'Was asked to fetch named JSON object "{}", but it was malformed!'
.format(hash.hex()))
obj = HydrusSerialisable.CreateFromSerialisableTuple(
(dump_type, version, serialisable_info))
hashes_to_objs[hash] = obj
return hashes_to_objs
def _InitialiseFromSerialisableInfo( self, serialisable_info ):
serialisable_service_keys_and_tag_data = serialisable_info
self._service_keys_to_tag_data = { bytes.fromhex( service_key_hex ) : ( HydrusSerialisable.CreateFromSerialisableTuple( serialisable_tag_filter ), tag_display_type ) for ( service_key_hex, serialisable_tag_filter, tag_display_type ) in serialisable_service_keys_and_tag_data }
def _InitialiseFromSerialisableInfo(self, serialisable_info):
serialisable_pages = serialisable_info
self._page_containers = HydrusSerialisable.CreateFromSerialisableTuple(
serialisable_pages)
def _InitialiseFromSerialisableInfo( self, serialisable_info ):
( serialisable_gallery_seed_log, serialisable_file_seed_cache ) = serialisable_info
self._gallery_seed_log = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_gallery_seed_log )
self._file_seed_cache = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_file_seed_cache )
def _InitialiseFromSerialisableInfo(self, serialisable_info):
serialisable_column_list_types_to_statuses = serialisable_info
self._column_list_types_to_statuses = HydrusSerialisable.CreateFromSerialisableTuple(
serialisable_column_list_types_to_statuses)
def _InitialiseFromSerialisableInfo( self, serialisable_info ):
( serialisable_network_context, serialisable_bandwidth_tracker ) = serialisable_info
self.network_context = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_network_context )
self.bandwidth_tracker = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_bandwidth_tracker )
def _InitialiseFromSerialisableInfo(self, serialisable_info):
serialisable_top_notebook = serialisable_info
self._top_notebook_container = HydrusSerialisable.CreateFromSerialisableTuple(
serialisable_top_notebook)
def _UpdateSerialisableInfo(self, version, old_serialisable_info):
if version == 1:
new_serialisable_info = []
for (page_name, serialisable_management_controller,
serialisable_hashes) in old_serialisable_info:
management_controller = HydrusSerialisable.CreateFromSerialisableTuple(
serialisable_management_controller)
management_controller.SetPageName(page_name)
serialisable_management_controller = management_controller.GetSerialisableTuple(
)
new_serialisable_info.append(
(serialisable_management_controller, serialisable_hashes))
return (2, new_serialisable_info)
if version == 2:
new_serialisable_info = []
for (serialisable_management_controller,
serialisable_hashes) in old_serialisable_info:
new_serialisable_info.append(
('page', (serialisable_management_controller,
serialisable_hashes)))
return (3, new_serialisable_info)
if version == 3:
def clean_tuple(spt):
(page_type, serialisable_page_data) = spt
if page_type == 'pages':
(name,
pages_serialisable_page_tuples) = serialisable_page_data
if name.startswith('[USER]') and len(name) > 6:
name = name[6:]
pages_serialisable_page_tuples = [
clean_tuple(pages_spt)
for pages_spt in pages_serialisable_page_tuples
]
return ('pages', (name, pages_serialisable_page_tuples))
else:
return spt
new_serialisable_info = []
serialisable_page_tuples = old_serialisable_info
for serialisable_page_tuple in serialisable_page_tuples:
serialisable_page_tuple = clean_tuple(serialisable_page_tuple)
new_serialisable_info.append(serialisable_page_tuple)
return (4, new_serialisable_info)
def _InitialiseFromSerialisableInfo( self, serialisable_info ):
( serialisable_file_seed_cache, serialisable_options, self._delete_after_success, self._paused ) = serialisable_info
self._file_seed_cache = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_file_seed_cache )
self._file_import_options = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_options )
def _InitialiseFromSerialisableInfo( self, serialisable_info ):
( serialisable_location_context, self._presentation_status, self._presentation_inbox ) = serialisable_info
self._location_context = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_location_context )
