def ParseFileArguments( path ):
HydrusImageHandling.ConvertToPngIfBmp( path )
hash = HydrusFileHandling.GetHashFromPath( path )
try:
mime = HydrusFileHandling.GetMime( path )
if mime in HC.IMAGES and HydrusImageHandling.IsDecompressionBomb( path ):
raise HydrusExceptions.ForbiddenException( 'File seemed to be a Decompression Bomb!' )
( size, mime, width, height, duration, num_frames, num_words ) = HydrusFileHandling.GetFileInfo( path, mime )
except HydrusExceptions.SizeException:
raise HydrusExceptions.ForbiddenException( 'File is of zero length!' )
except HydrusExceptions.MimeException:
raise HydrusExceptions.ForbiddenException( 'Filetype is not permitted!' )
except Exception as e:
raise HydrusExceptions.ForbiddenException( HydrusData.ToUnicode( e ) )
args = {}
args[ 'path' ] = path
args[ 'hash' ] = hash
args[ 'size' ] = size
args[ 'mime' ] = mime
if width is not None: args[ 'width' ] = width
if height is not None: args[ 'height' ] = height
if duration is not None: args[ 'duration' ] = duration
if num_frames is not None: args[ 'num_frames' ] = num_frames
if num_words is not None: args[ 'num_words' ] = num_words
if mime in HC.MIMES_WITH_THUMBNAILS:
try:
thumbnail = HydrusFileHandling.GenerateThumbnail( path, mime )
except Exception as e:
tb = traceback.format_exc()
raise HydrusExceptions.ForbiddenException( 'Could not generate thumbnail from that file:' + os.linesep + tb )
args[ 'thumbnail' ] = thumbnail
return args
def _callbackParsePOSTArgs( self, request ):
request.content.seek( 0 )
if not request.requestHeaders.hasHeader( 'Content-Type' ):
hydrus_args = {}
else:
content_types = request.requestHeaders.getRawHeaders( 'Content-Type' )
content_type = content_types[0]
try:
mime = HC.mime_enum_lookup[ content_type ]
except:
raise HydrusExceptions.ForbiddenException( 'Did not recognise Content-Type header!' )
total_bytes_read = 0
if mime == HC.APPLICATION_JSON:
json_string = request.content.read()
total_bytes_read += len( json_string )
hydrus_args = HydrusNetwork.ParseBodyString( json_string )
else:
( os_file_handle, temp_path ) = HydrusPaths.GetTempPath()
request.temp_file_info = ( os_file_handle, temp_path )
with open( temp_path, 'wb' ) as f:
for block in HydrusPaths.ReadFileLikeAsBlocks( request.content ):
f.write( block )
total_bytes_read += len( block )
decompression_bombs_ok = self._DecompressionBombsOK( request )
hydrus_args = ParseFileArguments( temp_path, decompression_bombs_ok )
self._reportDataUsed( request, total_bytes_read )
request.hydrus_args = hydrus_args
return request
def _callbackParseGETArgs( self, request ):
hydrus_args = {}
for name in request.args:
values = request.args[ name ]
value = values[0]
if name in ( 'begin', 'expires', 'lifetime', 'num', 'service_type', 'service_port', 'since', 'subindex', 'timespan' ):
try: hydrus_args[ name ] = int( value )
except: raise HydrusExceptions.ForbiddenException( 'I was expecting to parse \'' + name + '\' as an integer, but it failed.' )
elif name in ( 'access_key', 'title', 'subject_account_key', 'contact_key', 'hash', 'subject_hash', 'subject_tag', 'message_key', 'share_key' ):
try: hydrus_args[ name ] = value.decode( 'hex' )
except: raise HydrusExceptions.ForbiddenException( 'I was expecting to parse \'' + name + '\' as a hex-encoded string, but it failed.' )
if 'subject_account_key' in hydrus_args:
hydrus_args[ 'subject_identifier' ] = HydrusData.AccountIdentifier( account_key = hydrus_args[ 'subject_account_key' ] )
elif 'subject_hash' in hydrus_args:
hash = hydrus_args[ 'subject_hash' ]
if 'subject_tag' in hydrus_args:
tag = hydrus_args[ 'subject_tag' ]
content = HydrusData.Content( HC.CONTENT_TYPE_MAPPING, ( tag, hash ) )
else:
content = HydrusData.Content( HC.CONTENT_TYPE_FILES, [ hash ] )
hydrus_args[ 'subject_identifier' ] = HydrusData.AccountIdentifier( content = content )
request.hydrus_args = hydrus_args
return request
def _callbackParsePOSTArgs( self, request ):
request.content.seek( 0 )
if not request.requestHeaders.hasHeader( 'Content-Type' ):
hydrus_args = {}
else:
content_types = request.requestHeaders.getRawHeaders( 'Content-Type' )
content_type = content_types[0]
try: mime = HC.mime_enum_lookup[ content_type ]
except: raise HydrusExceptions.ForbiddenException( 'Did not recognise Content-Type header!' )
if mime == HC.APPLICATION_YAML:
yaml_string = request.content.read()
request.hydrus_request_data_usage += len( yaml_string )
hydrus_args = yaml.safe_load( yaml_string )
elif mime == HC.APPLICATION_JSON:
json_string = request.content.read()
request.hydrus_request_data_usage += len( json_string )
hydrus_args = HydrusSerialisable.CreateFromNetworkString( json_string )
else:
( os_file_handle, temp_path ) = HydrusPaths.GetTempPath()
request.temp_file_info = ( os_file_handle, temp_path )
with open( temp_path, 'wb' ) as f:
for block in HydrusPaths.ReadFileLikeAsBlocks( request.content ):
f.write( block )
request.hydrus_request_data_usage += len( block )
hydrus_args = ParseFileArguments( temp_path )
request.hydrus_args = hydrus_args
return request
def _callbackParsePOSTArgs(self, request):
request.content.seek(0)
if not request.requestHeaders.hasHeader('Content-Type'):
raise HydrusExceptions.ForbiddenException(
'No Content-Type header found!')
content_types = request.requestHeaders.getRawHeaders('Content-Type')
content_type = content_types[0]
try:
mime = HC.mime_enum_lookup[content_type]
except:
raise HydrusExceptions.ForbiddenException(
'Did not recognise Content-Type header!')
if mime == HC.APPLICATION_YAML:
yaml_string = request.content.read()
request.hydrus_request_data_usage += len(yaml_string)
hydrus_args = yaml.safe_load(yaml_string)
else:
temp_path = HC.GetTempPath()
with open(temp_path, 'wb') as f:
for block in HC.ReadFileLikeAsBlocks(request.content, 65536):
f.write(block)
request.hydrus_request_data_usage += len(block)
hydrus_args = ParseFileArguments(temp_path)
request.hydrus_args = hydrus_args
return request
def _parseAccessKey( self, request ):
if not request.requestHeaders.hasHeader( 'Hydrus-Key' ): raise HydrusExceptions.PermissionException( 'No hydrus key header found!' )
hex_keys = request.requestHeaders.getRawHeaders( 'Hydrus-Key' )
hex_key = hex_keys[0]
try: access_key = hex_key.decode( 'hex' )
except: raise HydrusExceptions.ForbiddenException( 'Could not parse the hydrus key!' )
return access_key
def ParseFileArguments(path):
HydrusImageHandling.ConvertToPngIfBmp(path)
hash = HydrusFileHandling.GetHashFromPath(path)
try:
(size, mime, width, height, duration, num_frames,
num_words) = HydrusFileHandling.GetFileInfo(path)
except HydrusExceptions.SizeException:
raise HydrusExceptions.ForbiddenException('File is of zero length!')
except HydrusExceptions.MimeException:
raise HydrusExceptions.ForbiddenException('Filetype is not permitted!')
except Exception as e:
raise HydrusExceptions.ForbiddenException(HC.u(e))
args = {}
args['path'] = path
args['hash'] = hash
args['size'] = size
args['mime'] = mime
if width is not None: args['width'] = width
if height is not None: args['height'] = height
if duration is not None: args['duration'] = duration
if num_frames is not None: args['num_frames'] = num_frames
if num_words is not None: args['num_words'] = num_words
if mime in HC.IMAGES:
try:
thumbnail = HydrusFileHandling.GenerateThumbnail(path)
except:
raise HydrusExceptions.ForbiddenException(
'Could not generate thumbnail from that file.')
args['thumbnail'] = thumbnail
return args
def LoginPixiv(self, network_context, pixiv_id, password):
session = self.engine.session_manager.GetSession(network_context)
response = session.get('https://accounts.pixiv.net/login')
soup = ClientParsing.GetSoup(response.content)
# some whocking 20kb bit of json tucked inside a hidden form input wew lad
i = soup.find('input', id='init-config')
raw_json = i['value']
j = json.loads(raw_json)
if 'pixivAccount.postKey' not in j:
raise HydrusExceptions.ForbiddenException(
'When trying to log into Pixiv, I could not find the POST key! This is a problem with hydrus\'s pixiv parsing, not your login! Please contact hydrus dev!'
)
post_key = j['pixivAccount.postKey']
form_fields = {}
form_fields['pixiv_id'] = pixiv_id
form_fields['password'] = password
form_fields['captcha'] = ''
form_fields['g_recaptcha_response'] = ''
form_fields['return_to'] = 'https://www.pixiv.net'
form_fields['lang'] = 'en'
form_fields['post_key'] = post_key
form_fields['source'] = 'pc'
headers = {}
headers[
'referer'] = "https://accounts.pixiv.net/login?lang=en^source=pc&view_type=page&ref=wwwtop_accounts_index"
headers['origin'] = "https://accounts.pixiv.net"
session.post('https://accounts.pixiv.net/api/login?lang=en',
data=form_fields,
headers=headers)
time.sleep(1)
def CheckValid( self, client_ip ):
if self._local_only and client_ip != '127.0.0.1': raise HydrusExceptions.ForbiddenException( 'Only local access allowed!' )
def Request( self, method, path_and_query, request_headers, body, report_hooks = None, temp_path = None ):
if report_hooks is None: report_hooks = []
if method == HC.GET: method_string = 'GET'
elif method == HC.POST: method_string = 'POST'
if 'User-Agent' not in request_headers:
request_headers[ 'User-Agent' ] = 'hydrus/' + str( HC.NETWORK_VERSION )
path_and_query = HydrusData.ToByteString( path_and_query )
request_headers = { str( k ) : str( v ) for ( k, v ) in request_headers.items() }
response = self._GetResponse( method_string, path_and_query, request_headers, body )
( parsed_response, size_of_response ) = self._ReadResponse( response, report_hooks, temp_path )
response_headers = { k : v for ( k, v ) in response.getheaders() if k != 'set-cookie' }
cookies = self._ParseCookies( response.getheader( 'set-cookie' ) )
self._last_request_time = HydrusData.GetNow()
if response.status == 200:
return ( parsed_response, None, size_of_response, response_headers, cookies )
elif response.status in ( 301, 302, 303, 307 ):
location = response.getheader( 'Location' )
if location is None: raise Exception( 'Received an invalid redirection response.' )
else:
url = location
if ', ' in url:
url = url.split( ', ' )[0]
elif ' ' in url:
# some booru is giving daft redirect responses
HydrusData.Print( url )
url = urllib.quote( HydrusData.ToByteString( url ), safe = '/?=&' )
HydrusData.Print( url )
if not url.startswith( self._scheme ):
# assume it is like 'index.php' or '/index.php', rather than 'http://blah.com/index.php'
if url.startswith( '/' ): slash_sep = ''
else: slash_sep = '/'
url = self._scheme + '://' + self._host + slash_sep + url
if response.status in ( 301, 307 ):
# 301: moved permanently, repeat request
# 307: moved temporarily, repeat request
redirect_info = ( method, url )
elif response.status in ( 302, 303 ):
# 302: moved temporarily, repeat request (except everyone treats it like 303 for no good f*****g reason)
# 303: thanks, now go here with GET
redirect_info = ( HC.GET, url )
return ( parsed_response, redirect_info, size_of_response, response_headers, cookies )
elif response.status == 304: raise HydrusExceptions.NotModifiedException()
else:
if response.status == 401: raise HydrusExceptions.PermissionException( parsed_response )
elif response.status == 403: raise HydrusExceptions.ForbiddenException( parsed_response )
elif response.status == 404: raise HydrusExceptions.NotFoundException( parsed_response )
elif response.status == 419: raise HydrusExceptions.SessionException( parsed_response )
elif response.status == 426: raise HydrusExceptions.NetworkVersionException( parsed_response )
elif response.status in ( 500, 501, 502, 503 ):
server_header = response.getheader( 'Server' )
if server_header is not None and 'hydrus' in server_header:
hydrus_service = True
else:
hydrus_service = False
if response.status == 503 and hydrus_service:
raise HydrusExceptions.ServerBusyException( 'Server is busy, please try again later.' )
else:
raise Exception( parsed_response )
else: raise Exception( parsed_response )
def Request(self,
method,
path_and_query,
request_headers,
body,
report_hooks=[],
response_to_path=False):
if method == HC.GET: method_string = 'GET'
elif method == HC.POST: method_string = 'POST'
if 'User-Agent' not in request_headers:
request_headers['User-Agent'] = 'hydrus/' + HC.u(
HC.NETWORK_VERSION)
# it is important to only send str, not unicode, to httplib
# it uses += to extend the message body, which propagates the unicode (and thus fails) when
# you try to push non-ascii bytes as the body (e.g. during a file upload!)
method_string = str(method_string)
path_and_query = str(path_and_query)
request_headers = {
str(k): str(v)
for (k, v) in request_headers.items()
}
try:
self._connection.request(method_string,
path_and_query,
headers=request_headers,
body=body)
response = self._connection.getresponse()
except (httplib.CannotSendRequest, httplib.BadStatusLine):
# for some reason, we can't send a request on the current connection, so let's make a new one and try again!
self._RefreshConnection()
self._connection.request(method_string,
path_and_query,
headers=request_headers,
body=body)
response = self._connection.getresponse()
if response.status == 200 and response_to_path:
(temp_path, size_of_response) = self._WriteResponseToPath(
response, report_hooks)
parsed_response = temp_path
else:
(parsed_response,
size_of_response) = self._ParseResponse(response, report_hooks)
response_headers = {
k: v
for (k, v) in response.getheaders() if k != 'set-cookie'
}
cookies = self._ParseCookies(response.getheader('set-cookie'))
self._last_request_time = HC.GetNow()
if response.status == 200:
return (parsed_response, None, size_of_response, response_headers,
cookies)
elif response.status in (301, 302, 303, 307):
location = response.getheader('Location')
if location is None: raise Exception(parsed_response)
else:
url = location
if ' ' in url:
# some booru is giving daft redirect responses
print(url)
url = urllib.quote(url, safe='/?=&')
print(url)
if not url.startswith(self._scheme):
# assume it is like 'index.php' or '/index.php', rather than 'http://blah.com/index.php'
if url.startswith('/'): slash_sep = ''
else: slash_sep = '/'
url = self._scheme + '://' + self._host + slash_sep + url
if response.status in (301, 307):
# 301: moved permanently, repeat request
# 307: moved temporarily, repeat request
redirect_info = (method, url)
elif response.status in (302, 303):
# 302: moved temporarily, repeat request (except everyone treats it like 303 for no good f*****g reason)
# 303: thanks, now go here with GET
redirect_info = (HC.GET, url)
return (parsed_response, redirect_info, size_of_response,
response_headers, cookies)
elif response.status == 304:
raise HydrusExceptions.NotModifiedException()
else:
if response.status == 401:
raise HydrusExceptions.PermissionException(parsed_response)
elif response.status == 403:
raise HydrusExceptions.ForbiddenException(parsed_response)
elif response.status == 404:
raise HydrusExceptions.NotFoundException(parsed_response)
elif response.status == 426:
raise HydrusExceptions.NetworkVersionException(parsed_response)
elif response.status in (500, 501, 502, 503):
raise Exception(parsed_response)
else:
raise Exception(parsed_response)
