def GenerateLoginProcess(self, network_context): with self._lock: if network_context.context_type == CC.NETWORK_CONTEXT_DOMAIN: login_network_context = self._GetLoginNetworkContext( network_context) if login_network_context is None: raise HydrusExceptions.DataMissing() (login_script, credentials) = self._domains_to_login_scripts_and_credentials[ login_network_context.context_data] login_process = LoginProcessDomain(self.engine, login_network_context, login_script, credentials) elif network_context.context_type == CC.NETWORK_CONTEXT_HYDRUS: login_process = LoginProcessHydrus(self.engine, network_context, self._hydrus_login_script) return login_process
def GetCookie(cookies, search_domain, name): existing_domains = cookies.list_domains() for existing_domain in existing_domains: # blah.com is viewable by blah.com matches_exactly = existing_domain == search_domain # .blah.com is viewable by blah.com matches_dot = existing_domain == '.' + search_domain # .blah.com applies to subdomain.blah.com, blah.com does not valid_subdomain = existing_domain.startwith( '.') and search_domain.endswith(existing_domain) if matches_exactly or matches_dot or valid_subdomain: cookie_dict = cookies.get_dict(existing_domain) if name in cookie_dict: return cookie_dict[name] raise HydrusExceptions.DataMissing('Cookie ' + name + ' not found for domain ' + search_domain + '!')
def _Start( self ): try: name = self.login_script if name == 'hentai foundry': self.LoginHF( self.network_context ) elif name == 'pixiv': result = self.engine.controller.Read( 'serialisable_simple', 'pixiv_account' ) if result is None: raise HydrusExceptions.DataMissing( 'You need to set up your pixiv credentials in services->manage pixiv account.' ) ( pixiv_id, password ) = result self.LoginPixiv( self.network_context, pixiv_id, password ) except: global LEGACY_LOGIN_OK LEGACY_LOGIN_OK = False
def GenerateLoginProcess( self, network_context ): with self._lock: if network_context.context_type == CC.NETWORK_CONTEXT_DOMAIN: domain = network_context.context_data if 'pixiv.net' in domain: return LoginProcessLegacy( self.engine, PIXIV_NETWORK_CONTEXT, 'pixiv' ) elif 'hentai-foundry.com' in domain: return LoginProcessLegacy( self.engine, HENTAI_FOUNDRY_NETWORK_CONTEXT, 'hentai foundry' ) login_network_context = self._GetLoginNetworkContext( network_context ) if login_network_context is None: raise HydrusExceptions.DataMissing() ( login_script, credentials ) = self._domains_to_login_scripts_and_credentials[ login_network_context.context_data ] login_process = LoginProcessDomain( self.engine, login_network_context, login_script, credentials ) elif network_context.context_type == CC.NETWORK_CONTEXT_HYDRUS: login_process = LoginProcessHydrus( self.engine, network_context, self._hydrus_login_script ) return login_process
def EnsureLoggedIn( self, name ): with self._lock: if name in self._error_names: raise Exception( name + ' could not establish a session! This ugly error is temporary due to the network engine rewrite. Please restart the client to reattempt this network context.' ) if name == 'hentai foundry': network_context = ClientNetworking.NetworkContext( CC.NETWORK_CONTEXT_DOMAIN, 'hentai-foundry.com' ) required_cookies = [ 'PHPSESSID', 'YII_CSRF_TOKEN' ] elif name == 'pixiv': network_context = ClientNetworking.NetworkContext( CC.NETWORK_CONTEXT_DOMAIN, 'pixiv.net' ) required_cookies = [ 'PHPSESSID' ] if self._IsLoggedIn( network_context, required_cookies ): return try: if name == 'hentai foundry': self.LoginHF( network_context ) elif name == 'pixiv': result = self.engine.controller.Read( 'serialisable_simple', 'pixiv_account' ) if result is None: raise HydrusExceptions.DataMissing( 'You need to set up your pixiv credentials in services->manage pixiv account.' ) ( pixiv_id, password ) = result self.LoginPixiv( network_context, pixiv_id, password ) if not self._IsLoggedIn( network_context, required_cookies ): raise Exception( name + ' login did not work correctly!' ) HydrusData.Print( 'Successfully logged into ' + name + '.' ) except: self._error_names.add( name ) raise
def _GetListCtrlMenu(self): selected_file_seeds = self._list_ctrl.GetData(only_selected=True) if len(selected_file_seeds) == 0: raise HydrusExceptions.DataMissing() menu = wx.Menu() can_show_files_in_new_page = True in ( file_seed.HasHash() for file_seed in selected_file_seeds) if can_show_files_in_new_page: ClientGUIMenus.AppendMenuItem( self, menu, 'open selected import files in a new page', 'Show all the known selected files in a new thumbnail page. This is complicated, so cannot always be guaranteed, even if the import says \'success\'.', self._ShowSelectionInNewPage) ClientGUIMenus.AppendSeparator(menu) ClientGUIMenus.AppendMenuItem( self, menu, 'copy sources', 'Copy all the selected sources to clipboard.', self._CopySelectedFileSeedData) ClientGUIMenus.AppendMenuItem( self, menu, 'copy notes', 'Copy all the selected notes to clipboard.', self._CopySelectedNotes) ClientGUIMenus.AppendSeparator(menu) ClientGUIMenus.AppendMenuItem( self, menu, 'open sources', 'Open all the selected sources in your file explorer or web browser.', self._OpenSelectedFileSeedData) ClientGUIMenus.AppendSeparator(menu) ClientGUIMenus.AppendMenuItem( self, menu, 'try again', 'Reset the progress of all the selected imports.', HydrusData.Call(self._SetSelected, CC.STATUS_UNKNOWN)) ClientGUIMenus.AppendMenuItem( self, menu, 'skip', 'Skip all the selected imports.', HydrusData.Call(self._SetSelected, CC.STATUS_SKIPPED)) ClientGUIMenus.AppendMenuItem(self, menu, 'delete from list', 'Remove all the selected imports.', self._DeleteSelected) return menu
def GetIndexFromObject(self, obj): try: data_index = self._objects_to_data_indices[obj] index = self._GetIndexFromDataIndex(data_index) return index except KeyError: raise HydrusExceptions.DataMissing('Data not found!')
def _GetIndexFromDataIndex( self, data_index ): if self._data_indices_to_sort_indices_dirty: self._data_indices_to_sort_indices = { self.GetItemData( index ) : index for index in range( self.GetItemCount() ) } self._data_indices_to_sort_indices_dirty = False try: return self._data_indices_to_sort_indices[ data_index ] except KeyError: raise HydrusExceptions.DataMissing( 'Data not found!' )
def _GetListCtrlMenu(self): selected_gallery_seeds = self._list_ctrl.GetData(only_selected=True) if len(selected_gallery_seeds) == 0: raise HydrusExceptions.DataMissing() menu = wx.Menu() ClientGUIMenus.AppendMenuItem( self, menu, 'copy urls', 'Copy all the selected urls to clipboard.', self._CopySelectedGalleryURLs) ClientGUIMenus.AppendMenuItem( self, menu, 'copy notes', 'Copy all the selected notes to clipboard.', self._CopySelectedNotes) ClientGUIMenus.AppendSeparator(menu) ClientGUIMenus.AppendMenuItem( self, menu, 'open urls', 'Open all the selected urls in your web browser.', self._OpenSelectedGalleryURLs) ClientGUIMenus.AppendSeparator(menu) if not self._read_only: ClientGUIMenus.AppendMenuItem( self, menu, 'try again (just this one page)', 'Schedule this url to occur again.', HydrusData.Call(self._TrySelectedAgain, False)) if self._can_generate_more_pages: ClientGUIMenus.AppendMenuItem( self, menu, 'try again (and allow search to continue)', 'Schedule this url to occur again and continue.', HydrusData.Call(self._TrySelectedAgain, True)) ClientGUIMenus.AppendMenuItem( self, menu, 'skip', 'Skip all the selected urls.', HydrusData.Call(self._SetSelected, CC.STATUS_SKIPPED)) return menu
def GetIndexFromClientData(self, data, column_index=None): for index in range(self.GetItemCount()): client_data = self.GetClientData(index) if column_index is None: comparison_data = client_data else: comparison_data = client_data[column_index] if comparison_data == data: return index raise HydrusExceptions.DataMissing('Data not found!')
def _ParseImagePage(self, html, url_base): (search_url, search_separator, advance_by_page_num, thumb_classname, image_id, image_data, tag_classnames_to_namespaces) = self._booru.GetData() soup = ClientParsing.GetSoup(html) image_url = None try: if image_id is not None: image = soup.find(id=image_id) if image is None: image_string = soup.find(text=re.compile('Save this file')) if image_string is None: image_string = soup.find( text=re.compile('Save this video')) if image_string is None: # catchall for rule34hentai.net's webms if image_url is None: a_tags = soup.find_all('a') for a_tag in a_tags: href = a_tag['href'] if href is not None: if href.endswith('.webm'): image_url = href break # catchall for rule34hentai.net's mp4s, which are loaded in a mickey-mouse flv player if image_url is None: magic_phrase = 'document.write("<source src=\'' if magic_phrase in html: # /image/252605' type='video/mp4... image_url_and_gumpf = html.split( magic_phrase, 1)[1] image_url = image_url_and_gumpf.split('\'', 1)[0] else: image = image_string.parent image_url = image['href'] else: if image.name in ('img', 'video'): image_url = image['src'] if 'Running Danbooru' in html: # possible danbooru resized image possible_better_image = soup.find( id='image-resize-link') if possible_better_image is not None: image_url = possible_better_image['href'] elif image.name == 'a': image_url = image['href'] if image_data is not None: links = soup.find_all('a') ok_link = None better_link = None for link in links: if link.string is not None: if link.string.startswith( image_data) or link.string.endswith( image_data): ok_link = link['href'] if link.string.startswith('Download PNG'): better_link = link['href'] break if better_link is not None: image_url = better_link else: image_url = ok_link except Exception as e: raise HydrusExceptions.DataMissing( 'Could not parse a download link for ' + url_base + '!' + os.linesep + HydrusData.ToUnicode(e)) if image_url is None: raise HydrusExceptions.DataMissing( 'Could not parse a download link for ' + url_base + '!') image_url = urlparse.urljoin(url_base, image_url) if 'gelbooru.com' in url_base: # giving 404 on some content servers for http, no redirect for some reason image_url = ClientNetworkingDomain.ConvertHTTPToHTTPS(image_url) tags = [] for (tag_classname, namespace) in tag_classnames_to_namespaces.items(): tag_list_entries = soup.find_all(class_=tag_classname) for tag_list_entry in tag_list_entries: links = tag_list_entry.find_all('a') if tag_list_entry.name == 'a': links.append(tag_list_entry) for link in links: if link.string is None: continue try: tag_string = HydrusData.ToUnicode(link.string) tag_string = HydrusTags.CleanTag(tag_string) if tag_string in ( '?', '-', '+', u'\xe2\x80\x93', u'\u2013' ): # last two are a couple of amusing encodings of en-dash '-' from danbooru continue tag = HydrusTags.CombineTag(namespace, tag_string) tags.append(tag) except Exception as e: HydrusData.Print('Could not parse tag "' + repr(link.string) + '":') HydrusData.PrintException(e) return (image_url, tags)
def _ActionPaths( self ): for status in ( CC.STATUS_SUCCESSFUL_AND_NEW, CC.STATUS_SUCCESSFUL_BUT_REDUNDANT, CC.STATUS_DELETED, CC.STATUS_ERROR ): action = self._actions[ status ] if action == CC.IMPORT_FOLDER_DELETE: while True: file_seed = self._file_seed_cache.GetNextFileSeed( status ) if file_seed is None or HG.view_shutdown: break path = file_seed.file_seed_data try: if os.path.exists( path ): ClientPaths.DeletePath( path ) txt_path = path + '.txt' if os.path.exists( txt_path ): ClientPaths.DeletePath( txt_path ) self._file_seed_cache.RemoveFileSeeds( ( file_seed, ) ) except Exception as e: HydrusData.ShowText( 'Import folder tried to delete ' + path + ', but could not:' ) HydrusData.ShowException( e ) HydrusData.ShowText( 'Import folder has been paused.' ) self._paused = True return elif action == CC.IMPORT_FOLDER_MOVE: while True: file_seed = self._file_seed_cache.GetNextFileSeed( status ) if file_seed is None or HG.view_shutdown: break path = file_seed.file_seed_data try: dest_dir = self._action_locations[ status ] if not os.path.exists( dest_dir ): raise HydrusExceptions.DataMissing( 'The move location "' + dest_dir + '" does not exist!' ) if os.path.exists( path ): filename = os.path.basename( path ) dest_path = os.path.join( dest_dir, filename ) dest_path = HydrusPaths.AppendPathUntilNoConflicts( dest_path ) HydrusPaths.MergeFile( path, dest_path ) txt_path = path + '.txt' if os.path.exists( txt_path ): txt_filename = os.path.basename( txt_path ) txt_dest_path = os.path.join( dest_dir, txt_filename ) txt_dest_path = HydrusPaths.AppendPathUntilNoConflicts( txt_dest_path ) HydrusPaths.MergeFile( txt_path, txt_dest_path ) self._file_seed_cache.RemoveFileSeeds( ( file_seed, ) ) except Exception as e: HydrusData.ShowText( 'Import folder tried to move ' + path + ', but could not:' ) HydrusData.ShowException( e ) HydrusData.ShowText( 'Import folder has been paused.' ) self._paused = True return elif status == CC.IMPORT_FOLDER_IGNORE: pass
def CheckCanLogin( self, network_context ): with self._lock: if network_context.context_type == CC.NETWORK_CONTEXT_DOMAIN: domain = network_context.context_data if 'pixiv.net' in domain: if not LEGACY_LOGIN_OK: raise Exception( 'Legacy login broke last time--please either restart the client or contact hydrus dev!' ) result = self.engine.controller.Read( 'serialisable_simple', 'pixiv_account' ) if result is None: raise HydrusExceptions.DataMissing( 'You need to set up your pixiv credentials in services->manage pixiv account.' ) return elif 'hentai-foundry.com' in domain: if not LEGACY_LOGIN_OK: raise Exception( 'Legacy login broke last time--please either restart the client or contact hydrus dev!' ) return login_network_context = self._GetLoginNetworkContext( network_context ) if login_network_context is None: raise HydrusExceptions.LoginException( 'Could not find a network context to login with!' ) ( login_script, credentials ) = self._domains_to_login_scripts_and_credentials[ login_network_context.context_data ] login_script.CheckCanLogin( credentials ) elif network_context.context_type == CC.NETWORK_CONTEXT_HYDRUS: service_key = network_context.context_data services_manager = self.engine.controller.services_manager if not services_manager.ServiceExists( service_key ): raise HydrusExceptions.LoginException( 'Service does not exist!' ) service = services_manager.GetService( service_key ) try: service.CheckFunctional( including_account = False ) except Exception as e: message = 'Service has had a recent error or is otherwise not functional! Specific error was:' message += os.linesep * 2 message += HydrusData.ToUnicode( e ) message += os.linesep * 2 message += 'You might like to try refreshing its account in \'review services\'.' raise HydrusExceptions.LoginException( message )