def GenerateLoginProcess(self, network_context):
with self._lock:
if network_context.context_type == CC.NETWORK_CONTEXT_DOMAIN:
login_network_context = self._GetLoginNetworkContext(
network_context)
if login_network_context is None:
raise HydrusExceptions.DataMissing()
(login_script,
credentials) = self._domains_to_login_scripts_and_credentials[
login_network_context.context_data]
login_process = LoginProcessDomain(self.engine,
login_network_context,
login_script, credentials)
elif network_context.context_type == CC.NETWORK_CONTEXT_HYDRUS:
login_process = LoginProcessHydrus(self.engine,
network_context,
self._hydrus_login_script)
return login_process
def GetCookie(cookies, search_domain, name):
existing_domains = cookies.list_domains()
for existing_domain in existing_domains:
# blah.com is viewable by blah.com
matches_exactly = existing_domain == search_domain
# .blah.com is viewable by blah.com
matches_dot = existing_domain == '.' + search_domain
# .blah.com applies to subdomain.blah.com, blah.com does not
valid_subdomain = existing_domain.startwith(
'.') and search_domain.endswith(existing_domain)
if matches_exactly or matches_dot or valid_subdomain:
cookie_dict = cookies.get_dict(existing_domain)
if name in cookie_dict:
return cookie_dict[name]
raise HydrusExceptions.DataMissing('Cookie ' + name +
' not found for domain ' +
search_domain + '!')
def _Start( self ):
try:
name = self.login_script
if name == 'hentai foundry':
self.LoginHF( self.network_context )
elif name == 'pixiv':
result = self.engine.controller.Read( 'serialisable_simple', 'pixiv_account' )
if result is None:
raise HydrusExceptions.DataMissing( 'You need to set up your pixiv credentials in services->manage pixiv account.' )
( pixiv_id, password ) = result
self.LoginPixiv( self.network_context, pixiv_id, password )
except:
global LEGACY_LOGIN_OK
LEGACY_LOGIN_OK = False
def GenerateLoginProcess( self, network_context ):
with self._lock:
if network_context.context_type == CC.NETWORK_CONTEXT_DOMAIN:
domain = network_context.context_data
if 'pixiv.net' in domain:
return LoginProcessLegacy( self.engine, PIXIV_NETWORK_CONTEXT, 'pixiv' )
elif 'hentai-foundry.com' in domain:
return LoginProcessLegacy( self.engine, HENTAI_FOUNDRY_NETWORK_CONTEXT, 'hentai foundry' )
login_network_context = self._GetLoginNetworkContext( network_context )
if login_network_context is None:
raise HydrusExceptions.DataMissing()
( login_script, credentials ) = self._domains_to_login_scripts_and_credentials[ login_network_context.context_data ]
login_process = LoginProcessDomain( self.engine, login_network_context, login_script, credentials )
elif network_context.context_type == CC.NETWORK_CONTEXT_HYDRUS:
login_process = LoginProcessHydrus( self.engine, network_context, self._hydrus_login_script )
return login_process
def EnsureLoggedIn( self, name ):
with self._lock:
if name in self._error_names:
raise Exception( name + ' could not establish a session! This ugly error is temporary due to the network engine rewrite. Please restart the client to reattempt this network context.' )
if name == 'hentai foundry':
network_context = ClientNetworking.NetworkContext( CC.NETWORK_CONTEXT_DOMAIN, 'hentai-foundry.com' )
required_cookies = [ 'PHPSESSID', 'YII_CSRF_TOKEN' ]
elif name == 'pixiv':
network_context = ClientNetworking.NetworkContext( CC.NETWORK_CONTEXT_DOMAIN, 'pixiv.net' )
required_cookies = [ 'PHPSESSID' ]
if self._IsLoggedIn( network_context, required_cookies ):
return
try:
if name == 'hentai foundry':
self.LoginHF( network_context )
elif name == 'pixiv':
result = self.engine.controller.Read( 'serialisable_simple', 'pixiv_account' )
if result is None:
raise HydrusExceptions.DataMissing( 'You need to set up your pixiv credentials in services->manage pixiv account.' )
( pixiv_id, password ) = result
self.LoginPixiv( network_context, pixiv_id, password )
if not self._IsLoggedIn( network_context, required_cookies ):
raise Exception( name + ' login did not work correctly!' )
HydrusData.Print( 'Successfully logged into ' + name + '.' )
except:
self._error_names.add( name )
raise
def _GetListCtrlMenu(self):
selected_file_seeds = self._list_ctrl.GetData(only_selected=True)
if len(selected_file_seeds) == 0:
raise HydrusExceptions.DataMissing()
menu = wx.Menu()
can_show_files_in_new_page = True in (
file_seed.HasHash() for file_seed in selected_file_seeds)
if can_show_files_in_new_page:
ClientGUIMenus.AppendMenuItem(
self, menu, 'open selected import files in a new page',
'Show all the known selected files in a new thumbnail page. This is complicated, so cannot always be guaranteed, even if the import says \'success\'.',
self._ShowSelectionInNewPage)
ClientGUIMenus.AppendSeparator(menu)
ClientGUIMenus.AppendMenuItem(
self, menu, 'copy sources',
'Copy all the selected sources to clipboard.',
self._CopySelectedFileSeedData)
ClientGUIMenus.AppendMenuItem(
self, menu, 'copy notes',
'Copy all the selected notes to clipboard.',
self._CopySelectedNotes)
ClientGUIMenus.AppendSeparator(menu)
ClientGUIMenus.AppendMenuItem(
self, menu, 'open sources',
'Open all the selected sources in your file explorer or web browser.',
self._OpenSelectedFileSeedData)
ClientGUIMenus.AppendSeparator(menu)
ClientGUIMenus.AppendMenuItem(
self, menu, 'try again',
'Reset the progress of all the selected imports.',
HydrusData.Call(self._SetSelected, CC.STATUS_UNKNOWN))
ClientGUIMenus.AppendMenuItem(
self, menu, 'skip', 'Skip all the selected imports.',
HydrusData.Call(self._SetSelected, CC.STATUS_SKIPPED))
ClientGUIMenus.AppendMenuItem(self, menu, 'delete from list',
'Remove all the selected imports.',
self._DeleteSelected)
return menu
def GetIndexFromObject(self, obj):
try:
data_index = self._objects_to_data_indices[obj]
index = self._GetIndexFromDataIndex(data_index)
return index
except KeyError:
raise HydrusExceptions.DataMissing('Data not found!')
def _GetIndexFromDataIndex( self, data_index ):
if self._data_indices_to_sort_indices_dirty:
self._data_indices_to_sort_indices = { self.GetItemData( index ) : index for index in range( self.GetItemCount() ) }
self._data_indices_to_sort_indices_dirty = False
try:
return self._data_indices_to_sort_indices[ data_index ]
except KeyError:
raise HydrusExceptions.DataMissing( 'Data not found!' )
def _GetListCtrlMenu(self):
selected_gallery_seeds = self._list_ctrl.GetData(only_selected=True)
if len(selected_gallery_seeds) == 0:
raise HydrusExceptions.DataMissing()
menu = wx.Menu()
ClientGUIMenus.AppendMenuItem(
self, menu, 'copy urls',
'Copy all the selected urls to clipboard.',
self._CopySelectedGalleryURLs)
ClientGUIMenus.AppendMenuItem(
self, menu, 'copy notes',
'Copy all the selected notes to clipboard.',
self._CopySelectedNotes)
ClientGUIMenus.AppendSeparator(menu)
ClientGUIMenus.AppendMenuItem(
self, menu, 'open urls',
'Open all the selected urls in your web browser.',
self._OpenSelectedGalleryURLs)
ClientGUIMenus.AppendSeparator(menu)
if not self._read_only:
ClientGUIMenus.AppendMenuItem(
self, menu, 'try again (just this one page)',
'Schedule this url to occur again.',
HydrusData.Call(self._TrySelectedAgain, False))
if self._can_generate_more_pages:
ClientGUIMenus.AppendMenuItem(
self, menu, 'try again (and allow search to continue)',
'Schedule this url to occur again and continue.',
HydrusData.Call(self._TrySelectedAgain, True))
ClientGUIMenus.AppendMenuItem(
self, menu, 'skip', 'Skip all the selected urls.',
HydrusData.Call(self._SetSelected, CC.STATUS_SKIPPED))
return menu
def GetIndexFromClientData(self, data, column_index=None):
for index in range(self.GetItemCount()):
client_data = self.GetClientData(index)
if column_index is None:
comparison_data = client_data
else:
comparison_data = client_data[column_index]
if comparison_data == data:
return index
raise HydrusExceptions.DataMissing('Data not found!')
def _ParseImagePage(self, html, url_base):
(search_url, search_separator, advance_by_page_num, thumb_classname,
image_id, image_data,
tag_classnames_to_namespaces) = self._booru.GetData()
soup = ClientParsing.GetSoup(html)
image_url = None
try:
if image_id is not None:
image = soup.find(id=image_id)
if image is None:
image_string = soup.find(text=re.compile('Save this file'))
if image_string is None:
image_string = soup.find(
text=re.compile('Save this video'))
if image_string is None:
# catchall for rule34hentai.net's webms
if image_url is None:
a_tags = soup.find_all('a')
for a_tag in a_tags:
href = a_tag['href']
if href is not None:
if href.endswith('.webm'):
image_url = href
break
# catchall for rule34hentai.net's mp4s, which are loaded in a mickey-mouse flv player
if image_url is None:
magic_phrase = 'document.write("<source src=\''
if magic_phrase in html:
# /image/252605' type='video/mp4...
image_url_and_gumpf = html.split(
magic_phrase, 1)[1]
image_url = image_url_and_gumpf.split('\'',
1)[0]
else:
image = image_string.parent
image_url = image['href']
else:
if image.name in ('img', 'video'):
image_url = image['src']
if 'Running Danbooru' in html:
# possible danbooru resized image
possible_better_image = soup.find(
id='image-resize-link')
if possible_better_image is not None:
image_url = possible_better_image['href']
elif image.name == 'a':
image_url = image['href']
if image_data is not None:
links = soup.find_all('a')
ok_link = None
better_link = None
for link in links:
if link.string is not None:
if link.string.startswith(
image_data) or link.string.endswith(
image_data):
ok_link = link['href']
if link.string.startswith('Download PNG'):
better_link = link['href']
break
if better_link is not None:
image_url = better_link
else:
image_url = ok_link
except Exception as e:
raise HydrusExceptions.DataMissing(
'Could not parse a download link for ' + url_base + '!' +
os.linesep + HydrusData.ToUnicode(e))
if image_url is None:
raise HydrusExceptions.DataMissing(
'Could not parse a download link for ' + url_base + '!')
image_url = urlparse.urljoin(url_base, image_url)
if 'gelbooru.com' in url_base:
# giving 404 on some content servers for http, no redirect for some reason
image_url = ClientNetworkingDomain.ConvertHTTPToHTTPS(image_url)
tags = []
for (tag_classname, namespace) in tag_classnames_to_namespaces.items():
tag_list_entries = soup.find_all(class_=tag_classname)
for tag_list_entry in tag_list_entries:
links = tag_list_entry.find_all('a')
if tag_list_entry.name == 'a':
links.append(tag_list_entry)
for link in links:
if link.string is None:
continue
try:
tag_string = HydrusData.ToUnicode(link.string)
tag_string = HydrusTags.CleanTag(tag_string)
if tag_string in (
'?', '-', '+', u'\xe2\x80\x93', u'\u2013'
): # last two are a couple of amusing encodings of en-dash '-' from danbooru
continue
tag = HydrusTags.CombineTag(namespace, tag_string)
tags.append(tag)
except Exception as e:
HydrusData.Print('Could not parse tag "' +
repr(link.string) + '":')
HydrusData.PrintException(e)
return (image_url, tags)
def _ActionPaths( self ):
for status in ( CC.STATUS_SUCCESSFUL_AND_NEW, CC.STATUS_SUCCESSFUL_BUT_REDUNDANT, CC.STATUS_DELETED, CC.STATUS_ERROR ):
action = self._actions[ status ]
if action == CC.IMPORT_FOLDER_DELETE:
while True:
file_seed = self._file_seed_cache.GetNextFileSeed( status )
if file_seed is None or HG.view_shutdown:
break
path = file_seed.file_seed_data
try:
if os.path.exists( path ):
ClientPaths.DeletePath( path )
txt_path = path + '.txt'
if os.path.exists( txt_path ):
ClientPaths.DeletePath( txt_path )
self._file_seed_cache.RemoveFileSeeds( ( file_seed, ) )
except Exception as e:
HydrusData.ShowText( 'Import folder tried to delete ' + path + ', but could not:' )
HydrusData.ShowException( e )
HydrusData.ShowText( 'Import folder has been paused.' )
self._paused = True
return
elif action == CC.IMPORT_FOLDER_MOVE:
while True:
file_seed = self._file_seed_cache.GetNextFileSeed( status )
if file_seed is None or HG.view_shutdown:
break
path = file_seed.file_seed_data
try:
dest_dir = self._action_locations[ status ]
if not os.path.exists( dest_dir ):
raise HydrusExceptions.DataMissing( 'The move location "' + dest_dir + '" does not exist!' )
if os.path.exists( path ):
filename = os.path.basename( path )
dest_path = os.path.join( dest_dir, filename )
dest_path = HydrusPaths.AppendPathUntilNoConflicts( dest_path )
HydrusPaths.MergeFile( path, dest_path )
txt_path = path + '.txt'
if os.path.exists( txt_path ):
txt_filename = os.path.basename( txt_path )
txt_dest_path = os.path.join( dest_dir, txt_filename )
txt_dest_path = HydrusPaths.AppendPathUntilNoConflicts( txt_dest_path )
HydrusPaths.MergeFile( txt_path, txt_dest_path )
self._file_seed_cache.RemoveFileSeeds( ( file_seed, ) )
except Exception as e:
HydrusData.ShowText( 'Import folder tried to move ' + path + ', but could not:' )
HydrusData.ShowException( e )
HydrusData.ShowText( 'Import folder has been paused.' )
self._paused = True
return
elif status == CC.IMPORT_FOLDER_IGNORE:
pass
def CheckCanLogin( self, network_context ):
with self._lock:
if network_context.context_type == CC.NETWORK_CONTEXT_DOMAIN:
domain = network_context.context_data
if 'pixiv.net' in domain:
if not LEGACY_LOGIN_OK:
raise Exception( 'Legacy login broke last time--please either restart the client or contact hydrus dev!' )
result = self.engine.controller.Read( 'serialisable_simple', 'pixiv_account' )
if result is None:
raise HydrusExceptions.DataMissing( 'You need to set up your pixiv credentials in services->manage pixiv account.' )
return
elif 'hentai-foundry.com' in domain:
if not LEGACY_LOGIN_OK:
raise Exception( 'Legacy login broke last time--please either restart the client or contact hydrus dev!' )
return
login_network_context = self._GetLoginNetworkContext( network_context )
if login_network_context is None:
raise HydrusExceptions.LoginException( 'Could not find a network context to login with!' )
( login_script, credentials ) = self._domains_to_login_scripts_and_credentials[ login_network_context.context_data ]
login_script.CheckCanLogin( credentials )
elif network_context.context_type == CC.NETWORK_CONTEXT_HYDRUS:
service_key = network_context.context_data
services_manager = self.engine.controller.services_manager
if not services_manager.ServiceExists( service_key ):
raise HydrusExceptions.LoginException( 'Service does not exist!' )
service = services_manager.GetService( service_key )
try:
service.CheckFunctional( including_account = False )
except Exception as e:
message = 'Service has had a recent error or is otherwise not functional! Specific error was:'
message += os.linesep * 2
message += HydrusData.ToUnicode( e )
message += os.linesep * 2
message += 'You might like to try refreshing its account in \'review services\'.'
raise HydrusExceptions.LoginException( message )
