def _InitialiseFromSerialisableInfo(self, serialisable_info): (self._path, self._export_type, self._delete_from_client_after_export, serialisable_file_search_context, self._run_regularly, self._period, self._phrase, self._last_checked, self._paused, self._run_now, self._last_error) = serialisable_info if self._export_type == HC.EXPORT_FOLDER_TYPE_SYNCHRONISE: self._delete_from_client_after_export = False self._file_search_context = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_file_search_context)
def _InitialiseFromSerialisableInfo(self, serialisable_info): for (tag_display_type, serialisable_service_keys_to_tag_filters) in serialisable_info: for (serialisable_service_key, serialisable_tag_filter ) in serialisable_service_keys_to_tag_filters: service_key = bytes.fromhex(serialisable_service_key) tag_filter = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_tag_filter) self._tag_display_types_to_service_keys_to_tag_filters[ tag_display_type][service_key] = tag_filter
def _InitialiseFromSerialisableInfo(self, serialisable_info): serialisable_api_permissions_objects = serialisable_info api_permissions_objects = [ HydrusSerialisable.CreateFromSerialisableTuple( serialisable_api_permissions) for serialisable_api_permissions in serialisable_api_permissions_objects ] self._access_keys_to_permissions = { api_permissions.GetAccessKey(): api_permissions for api_permissions in api_permissions_objects }
def _dump_and_load_and_test(self, obj, test_func): serialisable_tuple = obj.GetSerialisableTuple() self.assertIsInstance(serialisable_tuple, tuple) if isinstance(obj, HydrusSerialisable.SerialisableBaseNamed): (serialisable_type, name, version, serialisable_info) = serialisable_tuple elif isinstance(obj, HydrusSerialisable.SerialisableBase): (serialisable_type, version, serialisable_info) = serialisable_tuple self.assertEqual(serialisable_type, obj.SERIALISABLE_TYPE) self.assertEqual(version, obj.SERIALISABLE_VERSION) dupe_obj = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_tuple) self.assertIsNot(obj, dupe_obj) test_func(obj, dupe_obj) # json_string = obj.DumpToString() self.assertIsInstance(json_string, str) dupe_obj = HydrusSerialisable.CreateFromString(json_string) self.assertIsNot(obj, dupe_obj) test_func(obj, dupe_obj) # network_bytes = obj.DumpToNetworkBytes() self.assertIsInstance(network_bytes, bytes) dupe_obj = HydrusSerialisable.CreateFromNetworkBytes(network_bytes) self.assertIsNot(obj, dupe_obj) test_func(obj, dupe_obj)
def _InitialiseFromSerialisableInfo( self, serialisable_info ): ( self.url, self._can_generate_more_pages, serialisable_external_filterable_tags, serialisable_external_additional_service_keys_to_tags, self.created, self.modified, self.status, self.note, self._referral_url ) = serialisable_info self._external_filterable_tags = set( serialisable_external_filterable_tags ) self._external_additional_service_keys_to_tags = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_external_additional_service_keys_to_tags )
def GetJSONDump( self, dump_type ): result = self._c.execute( 'SELECT version, dump FROM json_dumps WHERE dump_type = ?;', ( dump_type, ) ).fetchone() if result is None: return result else: ( version, dump ) = result try: if isinstance( dump, bytes ): dump = str( dump, 'utf-8' ) serialisable_info = json.loads( dump ) except: self._c.execute( 'DELETE FROM json_dumps WHERE dump_type = ?;', ( dump_type, ) ) self._cursor_transaction_wrapper.CommitAndBegin() DealWithBrokenJSONDump( self._db_dir, dump, 'dump_type {}'.format( dump_type ) ) obj = HydrusSerialisable.CreateFromSerialisableTuple( ( dump_type, version, serialisable_info ) ) if dump_type == HydrusSerialisable.SERIALISABLE_TYPE_NETWORK_SESSION_MANAGER: session_containers = self.GetJSONDumpNamed( HydrusSerialisable.SERIALISABLE_TYPE_NETWORK_SESSION_MANAGER_SESSION_CONTAINER ) obj.SetSessionContainers( session_containers ) elif dump_type == HydrusSerialisable.SERIALISABLE_TYPE_NETWORK_BANDWIDTH_MANAGER: tracker_containers = self.GetJSONDumpNamed( HydrusSerialisable.SERIALISABLE_TYPE_NETWORK_BANDWIDTH_MANAGER_TRACKER_CONTAINER ) obj.SetTrackerContainers( tracker_containers ) return obj
def _InitialiseFromSerialisableInfo(self, serialisable_info): (serialisable_tag_service_actions, serialisable_rating_service_actions, self._sync_archive, self._sync_urls_action) = serialisable_info self._tag_service_actions = [ (bytes.fromhex(serialisable_service_key), action, HydrusSerialisable.CreateFromSerialisableTuple( serialisable_tag_filter)) for (serialisable_service_key, action, serialisable_tag_filter) in serialisable_tag_service_actions ] self._rating_service_actions = [ (bytes.fromhex(serialisable_service_key), action) for (serialisable_service_key, action) in serialisable_rating_service_actions ]
def _InitialiseFromSerialisableInfo( self, serialisable_info ): ( self._url, serialisable_gallery_seed_log, serialisable_file_seed_cache, serialisable_fixed_service_keys_to_tags, serialisable_checker_options, serialisable_file_import_options, serialisable_tag_import_options, self._last_check_time, self._files_paused, self._checking_paused, self._checking_status, self._subject, self._no_work_until, self._no_work_until_reason, self._creation_time ) = serialisable_info self._fixed_service_keys_to_tags = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_fixed_service_keys_to_tags ) self._gallery_seed_log = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_gallery_seed_log ) self._file_seed_cache = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_file_seed_cache ) self._checker_options = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_checker_options ) self._file_import_options = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_file_import_options ) self._tag_import_options = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_tag_import_options )
def _InitialiseFromSerialisableInfo(self, serialisable_info): (serialisable_network_context, pickled_session_hex) = serialisable_info self.network_context = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_network_context) try: self.session = pickle.loads(bytes.fromhex(pickled_session_hex)) except: # a new version of requests messed this up lad, so reset self._InitialiseEmptySession() self.session.cookies.clear_session_cookies()
def _InitialiseFromSerialisableInfo(self, serialisable_info): [ serialisable_service_key, serialisable_write_autocomplete_tag_domain, self._override_write_autocomplete_location_context, serialisable_write_autocomplete_location_context, self._search_namespaces_into_full_tags, self._namespace_bare_fetch_all_allowed, self._namespace_fetch_all_allowed, self._fetch_all_allowed, self._fetch_results_automatically, self._exact_match_character_threshold ] = serialisable_info self._service_key = bytes.fromhex(serialisable_service_key) self._write_autocomplete_tag_domain = bytes.fromhex( serialisable_write_autocomplete_tag_domain) self._write_autocomplete_location_context = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_write_autocomplete_location_context)
def _InitialiseFromSerialisableInfo( self, serialisable_info ): [ serialisable_tag_display_types_to_service_keys_to_tag_filters, serialisable_tag_autocomplete_options ] = serialisable_info for ( tag_display_type, serialisable_service_keys_to_tag_filters ) in serialisable_tag_display_types_to_service_keys_to_tag_filters: for ( serialisable_service_key, serialisable_tag_filter ) in serialisable_service_keys_to_tag_filters: service_key = bytes.fromhex( serialisable_service_key ) tag_filter = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_tag_filter ) self._tag_display_types_to_service_keys_to_tag_filters[ tag_display_type ][ service_key ] = tag_filter self._tag_service_keys_to_tag_autocomplete_options = { tag_autocomplete_options.GetServiceKey() : tag_autocomplete_options for tag_autocomplete_options in HydrusSerialisable.CreateFromSerialisableTuple( serialisable_tag_autocomplete_options ) }
def _InitialiseFromSerialisableInfo( self, serialisable_info ): ( serialisable_network_context, pickled_cookies_hex ) = serialisable_info self.network_context = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_network_context ) self._InitialiseEmptySession() try: cookies = pickle.loads( bytes.fromhex( pickled_cookies_hex ) ) self.session.cookies = cookies except: HydrusData.Print( "Could not load and set cookies for session {}".format( self.network_context ) ) self.session.cookies.clear_session_cookies()
def _InitialiseFromSerialisableInfo( self, serialisable_info ): serialisable_network_contexts_to_sessions = serialisable_info for ( serialisable_network_context, pickled_session_hex ) in serialisable_network_contexts_to_sessions: network_context = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_network_context ) try: session = pickle.loads( bytes.fromhex( pickled_session_hex ) ) except: # new version of requests uses a diff format, wew continue session.cookies.clear_session_cookies() self._network_contexts_to_sessions[ network_context ] = session
def GetPageTuple(serialisable_page_tuple): (page_type, serialisable_page_data) = serialisable_page_tuple if page_type == 'pages': (name, serialisable_page_tuples) = serialisable_page_data page_tuples = [] for spt in serialisable_page_tuples: try: page_tuples.append(GetPageTuple(spt)) except Exception as e: handle_e(spt, e) page_data = (name, page_tuples) elif page_type == 'page': (serialisable_management_controller, serialisable_hashes) = serialisable_page_data management_controller = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_management_controller) hashes = [bytes.fromhex(hash) for hash in serialisable_hashes] page_data = (management_controller, hashes) page_tuple = (page_type, page_data) return page_tuple
def _UpdateSerialisableInfo(self, version, old_serialisable_info): if version == 1: (serialisable_file_seed_cache, serialisable_options, serialisable_paths_to_tags, delete_after_success, paused) = old_serialisable_info file_seed_cache = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_file_seed_cache) paths_to_service_keys_to_tags = { path: { bytes.fromhex(service_key): tags for (service_key, tags) in service_keys_to_tags.items() } for (path, service_keys_to_tags ) in serialisable_paths_to_tags.items() } for file_seed in file_seed_cache.GetFileSeeds(): path = file_seed.file_seed_data if path in paths_to_service_keys_to_tags: file_seed.SetFixedServiceKeysToTags( paths_to_service_keys_to_tags[path]) serialisable_file_seed_cache = file_seed_cache.GetSerialisableTuple( ) new_serialisable_info = (serialisable_file_seed_cache, serialisable_options, delete_after_success, paused) return (2, new_serialisable_info)
def GetJSONDumpNamed(self, dump_type, dump_name=None, timestamp=None): if dump_name is None: results = self._c.execute( 'SELECT dump_name, version, dump, timestamp FROM json_dumps_named WHERE dump_type = ?;', (dump_type, )).fetchall() objs = [] for (dump_name, version, dump, object_timestamp) in results: try: if isinstance(dump, bytes): dump = str(dump, 'utf-8') serialisable_info = json.loads(dump) objs.append( HydrusSerialisable.CreateFromSerialisableTuple( (dump_type, dump_name, version, serialisable_info))) except: self._c.execute( 'DELETE FROM json_dumps_named WHERE dump_type = ? AND dump_name = ? AND timestamp = ?;', (dump_type, dump_name, object_timestamp)) self._cursor_transaction_wrapper.CommitAndBegin() DealWithBrokenJSONDump( self._db_dir, dump, 'dump_type {} dump_name {} timestamp {}'.format( dump_type, dump_name[:10], timestamp)) return objs else: if timestamp is None: result = self._c.execute( 'SELECT version, dump, timestamp FROM json_dumps_named WHERE dump_type = ? AND dump_name = ? ORDER BY timestamp DESC;', (dump_type, dump_name)).fetchone() else: result = self._c.execute( 'SELECT version, dump, timestamp FROM json_dumps_named WHERE dump_type = ? AND dump_name = ? AND timestamp = ?;', (dump_type, dump_name, timestamp)).fetchone() if result is None: raise HydrusExceptions.DataMissing( 'Could not find the object of type "{}" and name "{}" and timestamp "{}".' .format(dump_type, dump_name, str(timestamp))) (version, dump, object_timestamp) = result try: if isinstance(dump, bytes): dump = str(dump, 'utf-8') serialisable_info = json.loads(dump) except: self._c.execute( 'DELETE FROM json_dumps_named WHERE dump_type = ? AND dump_name = ? AND timestamp = ?;', (dump_type, dump_name, object_timestamp)) self._cursor_transaction_wrapper.CommitAndBegin() DealWithBrokenJSONDump( self._db_dir, dump, 'dump_type {} dump_name {} timestamp {}'.format( dump_type, dump_name[:10], object_timestamp)) return HydrusSerialisable.CreateFromSerialisableTuple( (dump_type, dump_name, version, serialisable_info))
def GetHashedJSONDumps(self, hashes): shown_missing_dump_message = False shown_broken_dump_message = False hashes_to_objs = {} for hash in hashes: result = self._c.execute( 'SELECT version, dump_type, dump FROM json_dumps_hashed WHERE hash = ?;', (sqlite3.Binary(hash), )).fetchone() if result is None: if not shown_missing_dump_message: message = 'A hashed serialised object was missing! Its hash is "{}".'.format( hash.hex()) message += os.linesep * 2 message += 'This error could be due to several factors, but is most likely a hard drive fault (perhaps your computer recently had a bad power cut?).' message += os.linesep * 2 message += 'Your client may have lost one or more session pages.' message += os.linesep * 2 message += 'Please review the \'help my db is broke.txt\' file in your install_dir/db directory as background reading, and if the situation or fix here is not obvious, please contact hydrus dev.' HydrusData.ShowText(message) shown_missing_dump_message = True HydrusData.Print( 'Was asked to fetch named JSON object "{}", but it was missing!' .format(hash.hex())) continue (version, dump_type, dump) = result try: if isinstance(dump, bytes): dump = str(dump, 'utf-8') serialisable_info = json.loads(dump) except: self._c.execute( 'DELETE FROM json_dumps_hashed WHERE hash = ?;', (sqlite3.Binary(hash), )) self._cursor_transaction_wrapper.CommitAndBegin() ExportBrokenHashedJSONDump( self._db_dir, dump, 'hash {} dump_type {}'.format(hash.hex(), dump_type)) if not shown_broken_dump_message: message = 'A hashed serialised object failed to load! Its hash is "{}".'.format( hash.hex()) message += os.linesep * 2 message += 'This error could be due to several factors, but is most likely a hard drive fault (perhaps your computer recently had a bad power cut?).' message += os.linesep * 2 message += 'The database has attempted to delete the broken object, and the object\'s dump written to your database directory. Your client may have lost one or more session pages.' message += os.linesep * 2 message += 'Please review the \'help my db is broke.txt\' file in your install_dir/db directory as background reading, and if the situation or fix here is not obvious, please contact hydrus dev.' HydrusData.ShowText(message) shown_broken_dump_message = True HydrusData.Print( 'Was asked to fetch named JSON object "{}", but it was malformed!' .format(hash.hex())) obj = HydrusSerialisable.CreateFromSerialisableTuple( (dump_type, version, serialisable_info)) hashes_to_objs[hash] = obj return hashes_to_objs
def _InitialiseFromSerialisableInfo( self, serialisable_info ): serialisable_service_keys_and_tag_data = serialisable_info self._service_keys_to_tag_data = { bytes.fromhex( service_key_hex ) : ( HydrusSerialisable.CreateFromSerialisableTuple( serialisable_tag_filter ), tag_display_type ) for ( service_key_hex, serialisable_tag_filter, tag_display_type ) in serialisable_service_keys_and_tag_data }
def _InitialiseFromSerialisableInfo(self, serialisable_info): serialisable_pages = serialisable_info self._page_containers = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_pages)
def _InitialiseFromSerialisableInfo( self, serialisable_info ): ( serialisable_gallery_seed_log, serialisable_file_seed_cache ) = serialisable_info self._gallery_seed_log = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_gallery_seed_log ) self._file_seed_cache = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_file_seed_cache )
def _InitialiseFromSerialisableInfo(self, serialisable_info): serialisable_column_list_types_to_statuses = serialisable_info self._column_list_types_to_statuses = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_column_list_types_to_statuses)
def _InitialiseFromSerialisableInfo( self, serialisable_info ): ( serialisable_network_context, serialisable_bandwidth_tracker ) = serialisable_info self.network_context = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_network_context ) self.bandwidth_tracker = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_bandwidth_tracker )
def _InitialiseFromSerialisableInfo(self, serialisable_info): serialisable_top_notebook = serialisable_info self._top_notebook_container = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_top_notebook)
def _UpdateSerialisableInfo(self, version, old_serialisable_info): if version == 1: new_serialisable_info = [] for (page_name, serialisable_management_controller, serialisable_hashes) in old_serialisable_info: management_controller = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_management_controller) management_controller.SetPageName(page_name) serialisable_management_controller = management_controller.GetSerialisableTuple( ) new_serialisable_info.append( (serialisable_management_controller, serialisable_hashes)) return (2, new_serialisable_info) if version == 2: new_serialisable_info = [] for (serialisable_management_controller, serialisable_hashes) in old_serialisable_info: new_serialisable_info.append( ('page', (serialisable_management_controller, serialisable_hashes))) return (3, new_serialisable_info) if version == 3: def clean_tuple(spt): (page_type, serialisable_page_data) = spt if page_type == 'pages': (name, pages_serialisable_page_tuples) = serialisable_page_data if name.startswith('[USER]') and len(name) > 6: name = name[6:] pages_serialisable_page_tuples = [ clean_tuple(pages_spt) for pages_spt in pages_serialisable_page_tuples ] return ('pages', (name, pages_serialisable_page_tuples)) else: return spt new_serialisable_info = [] serialisable_page_tuples = old_serialisable_info for serialisable_page_tuple in serialisable_page_tuples: serialisable_page_tuple = clean_tuple(serialisable_page_tuple) new_serialisable_info.append(serialisable_page_tuple) return (4, new_serialisable_info)
def _InitialiseFromSerialisableInfo( self, serialisable_info ): ( serialisable_file_seed_cache, serialisable_options, self._delete_after_success, self._paused ) = serialisable_info self._file_seed_cache = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_file_seed_cache ) self._file_import_options = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_options )
def _InitialiseFromSerialisableInfo( self, serialisable_info ): ( serialisable_location_context, self._presentation_status, self._presentation_inbox ) = serialisable_info self._location_context = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_location_context )