def ParseFileArguments( path ): HydrusImageHandling.ConvertToPngIfBmp( path ) hash = HydrusFileHandling.GetHashFromPath( path ) try: mime = HydrusFileHandling.GetMime( path ) if mime in HC.IMAGES and HydrusImageHandling.IsDecompressionBomb( path ): raise HydrusExceptions.ForbiddenException( 'File seemed to be a Decompression Bomb!' ) ( size, mime, width, height, duration, num_frames, num_words ) = HydrusFileHandling.GetFileInfo( path, mime ) except HydrusExceptions.SizeException: raise HydrusExceptions.ForbiddenException( 'File is of zero length!' ) except HydrusExceptions.MimeException: raise HydrusExceptions.ForbiddenException( 'Filetype is not permitted!' ) except Exception as e: raise HydrusExceptions.ForbiddenException( HydrusData.ToUnicode( e ) ) args = {} args[ 'path' ] = path args[ 'hash' ] = hash args[ 'size' ] = size args[ 'mime' ] = mime if width is not None: args[ 'width' ] = width if height is not None: args[ 'height' ] = height if duration is not None: args[ 'duration' ] = duration if num_frames is not None: args[ 'num_frames' ] = num_frames if num_words is not None: args[ 'num_words' ] = num_words if mime in HC.MIMES_WITH_THUMBNAILS: try: thumbnail = HydrusFileHandling.GenerateThumbnail( path, mime ) except Exception as e: tb = traceback.format_exc() raise HydrusExceptions.ForbiddenException( 'Could not generate thumbnail from that file:' + os.linesep + tb ) args[ 'thumbnail' ] = thumbnail return args
def _callbackParsePOSTArgs( self, request ): request.content.seek( 0 ) if not request.requestHeaders.hasHeader( 'Content-Type' ): hydrus_args = {} else: content_types = request.requestHeaders.getRawHeaders( 'Content-Type' ) content_type = content_types[0] try: mime = HC.mime_enum_lookup[ content_type ] except: raise HydrusExceptions.ForbiddenException( 'Did not recognise Content-Type header!' ) total_bytes_read = 0 if mime == HC.APPLICATION_JSON: json_string = request.content.read() total_bytes_read += len( json_string ) hydrus_args = HydrusNetwork.ParseBodyString( json_string ) else: ( os_file_handle, temp_path ) = HydrusPaths.GetTempPath() request.temp_file_info = ( os_file_handle, temp_path ) with open( temp_path, 'wb' ) as f: for block in HydrusPaths.ReadFileLikeAsBlocks( request.content ): f.write( block ) total_bytes_read += len( block ) decompression_bombs_ok = self._DecompressionBombsOK( request ) hydrus_args = ParseFileArguments( temp_path, decompression_bombs_ok ) self._reportDataUsed( request, total_bytes_read ) request.hydrus_args = hydrus_args return request
def _callbackParseGETArgs( self, request ): hydrus_args = {} for name in request.args: values = request.args[ name ] value = values[0] if name in ( 'begin', 'expires', 'lifetime', 'num', 'service_type', 'service_port', 'since', 'subindex', 'timespan' ): try: hydrus_args[ name ] = int( value ) except: raise HydrusExceptions.ForbiddenException( 'I was expecting to parse \'' + name + '\' as an integer, but it failed.' ) elif name in ( 'access_key', 'title', 'subject_account_key', 'contact_key', 'hash', 'subject_hash', 'subject_tag', 'message_key', 'share_key' ): try: hydrus_args[ name ] = value.decode( 'hex' ) except: raise HydrusExceptions.ForbiddenException( 'I was expecting to parse \'' + name + '\' as a hex-encoded string, but it failed.' ) if 'subject_account_key' in hydrus_args: hydrus_args[ 'subject_identifier' ] = HydrusData.AccountIdentifier( account_key = hydrus_args[ 'subject_account_key' ] ) elif 'subject_hash' in hydrus_args: hash = hydrus_args[ 'subject_hash' ] if 'subject_tag' in hydrus_args: tag = hydrus_args[ 'subject_tag' ] content = HydrusData.Content( HC.CONTENT_TYPE_MAPPING, ( tag, hash ) ) else: content = HydrusData.Content( HC.CONTENT_TYPE_FILES, [ hash ] ) hydrus_args[ 'subject_identifier' ] = HydrusData.AccountIdentifier( content = content ) request.hydrus_args = hydrus_args return request
def _callbackParsePOSTArgs( self, request ): request.content.seek( 0 ) if not request.requestHeaders.hasHeader( 'Content-Type' ): hydrus_args = {} else: content_types = request.requestHeaders.getRawHeaders( 'Content-Type' ) content_type = content_types[0] try: mime = HC.mime_enum_lookup[ content_type ] except: raise HydrusExceptions.ForbiddenException( 'Did not recognise Content-Type header!' ) if mime == HC.APPLICATION_YAML: yaml_string = request.content.read() request.hydrus_request_data_usage += len( yaml_string ) hydrus_args = yaml.safe_load( yaml_string ) elif mime == HC.APPLICATION_JSON: json_string = request.content.read() request.hydrus_request_data_usage += len( json_string ) hydrus_args = HydrusSerialisable.CreateFromNetworkString( json_string ) else: ( os_file_handle, temp_path ) = HydrusPaths.GetTempPath() request.temp_file_info = ( os_file_handle, temp_path ) with open( temp_path, 'wb' ) as f: for block in HydrusPaths.ReadFileLikeAsBlocks( request.content ): f.write( block ) request.hydrus_request_data_usage += len( block ) hydrus_args = ParseFileArguments( temp_path ) request.hydrus_args = hydrus_args return request
def _callbackParsePOSTArgs(self, request): request.content.seek(0) if not request.requestHeaders.hasHeader('Content-Type'): raise HydrusExceptions.ForbiddenException( 'No Content-Type header found!') content_types = request.requestHeaders.getRawHeaders('Content-Type') content_type = content_types[0] try: mime = HC.mime_enum_lookup[content_type] except: raise HydrusExceptions.ForbiddenException( 'Did not recognise Content-Type header!') if mime == HC.APPLICATION_YAML: yaml_string = request.content.read() request.hydrus_request_data_usage += len(yaml_string) hydrus_args = yaml.safe_load(yaml_string) else: temp_path = HC.GetTempPath() with open(temp_path, 'wb') as f: for block in HC.ReadFileLikeAsBlocks(request.content, 65536): f.write(block) request.hydrus_request_data_usage += len(block) hydrus_args = ParseFileArguments(temp_path) request.hydrus_args = hydrus_args return request
def _parseAccessKey( self, request ): if not request.requestHeaders.hasHeader( 'Hydrus-Key' ): raise HydrusExceptions.PermissionException( 'No hydrus key header found!' ) hex_keys = request.requestHeaders.getRawHeaders( 'Hydrus-Key' ) hex_key = hex_keys[0] try: access_key = hex_key.decode( 'hex' ) except: raise HydrusExceptions.ForbiddenException( 'Could not parse the hydrus key!' ) return access_key
def ParseFileArguments(path): HydrusImageHandling.ConvertToPngIfBmp(path) hash = HydrusFileHandling.GetHashFromPath(path) try: (size, mime, width, height, duration, num_frames, num_words) = HydrusFileHandling.GetFileInfo(path) except HydrusExceptions.SizeException: raise HydrusExceptions.ForbiddenException('File is of zero length!') except HydrusExceptions.MimeException: raise HydrusExceptions.ForbiddenException('Filetype is not permitted!') except Exception as e: raise HydrusExceptions.ForbiddenException(HC.u(e)) args = {} args['path'] = path args['hash'] = hash args['size'] = size args['mime'] = mime if width is not None: args['width'] = width if height is not None: args['height'] = height if duration is not None: args['duration'] = duration if num_frames is not None: args['num_frames'] = num_frames if num_words is not None: args['num_words'] = num_words if mime in HC.IMAGES: try: thumbnail = HydrusFileHandling.GenerateThumbnail(path) except: raise HydrusExceptions.ForbiddenException( 'Could not generate thumbnail from that file.') args['thumbnail'] = thumbnail return args
def LoginPixiv(self, network_context, pixiv_id, password): session = self.engine.session_manager.GetSession(network_context) response = session.get('https://accounts.pixiv.net/login') soup = ClientParsing.GetSoup(response.content) # some whocking 20kb bit of json tucked inside a hidden form input wew lad i = soup.find('input', id='init-config') raw_json = i['value'] j = json.loads(raw_json) if 'pixivAccount.postKey' not in j: raise HydrusExceptions.ForbiddenException( 'When trying to log into Pixiv, I could not find the POST key! This is a problem with hydrus\'s pixiv parsing, not your login! Please contact hydrus dev!' ) post_key = j['pixivAccount.postKey'] form_fields = {} form_fields['pixiv_id'] = pixiv_id form_fields['password'] = password form_fields['captcha'] = '' form_fields['g_recaptcha_response'] = '' form_fields['return_to'] = 'https://www.pixiv.net' form_fields['lang'] = 'en' form_fields['post_key'] = post_key form_fields['source'] = 'pc' headers = {} headers[ 'referer'] = "https://accounts.pixiv.net/login?lang=en^source=pc&view_type=page&ref=wwwtop_accounts_index" headers['origin'] = "https://accounts.pixiv.net" session.post('https://accounts.pixiv.net/api/login?lang=en', data=form_fields, headers=headers) time.sleep(1)
def CheckValid( self, client_ip ): if self._local_only and client_ip != '127.0.0.1': raise HydrusExceptions.ForbiddenException( 'Only local access allowed!' )
def Request( self, method, path_and_query, request_headers, body, report_hooks = None, temp_path = None ): if report_hooks is None: report_hooks = [] if method == HC.GET: method_string = 'GET' elif method == HC.POST: method_string = 'POST' if 'User-Agent' not in request_headers: request_headers[ 'User-Agent' ] = 'hydrus/' + str( HC.NETWORK_VERSION ) path_and_query = HydrusData.ToByteString( path_and_query ) request_headers = { str( k ) : str( v ) for ( k, v ) in request_headers.items() } response = self._GetResponse( method_string, path_and_query, request_headers, body ) ( parsed_response, size_of_response ) = self._ReadResponse( response, report_hooks, temp_path ) response_headers = { k : v for ( k, v ) in response.getheaders() if k != 'set-cookie' } cookies = self._ParseCookies( response.getheader( 'set-cookie' ) ) self._last_request_time = HydrusData.GetNow() if response.status == 200: return ( parsed_response, None, size_of_response, response_headers, cookies ) elif response.status in ( 301, 302, 303, 307 ): location = response.getheader( 'Location' ) if location is None: raise Exception( 'Received an invalid redirection response.' ) else: url = location if ', ' in url: url = url.split( ', ' )[0] elif ' ' in url: # some booru is giving daft redirect responses HydrusData.Print( url ) url = urllib.quote( HydrusData.ToByteString( url ), safe = '/?=&' ) HydrusData.Print( url ) if not url.startswith( self._scheme ): # assume it is like 'index.php' or '/index.php', rather than 'http://blah.com/index.php' if url.startswith( '/' ): slash_sep = '' else: slash_sep = '/' url = self._scheme + '://' + self._host + slash_sep + url if response.status in ( 301, 307 ): # 301: moved permanently, repeat request # 307: moved temporarily, repeat request redirect_info = ( method, url ) elif response.status in ( 302, 303 ): # 302: moved temporarily, repeat request (except everyone treats it like 303 for no good f*****g reason) # 303: thanks, now go here with GET redirect_info = ( HC.GET, url ) return ( parsed_response, redirect_info, size_of_response, response_headers, cookies ) elif response.status == 304: raise HydrusExceptions.NotModifiedException() else: if response.status == 401: raise HydrusExceptions.PermissionException( parsed_response ) elif response.status == 403: raise HydrusExceptions.ForbiddenException( parsed_response ) elif response.status == 404: raise HydrusExceptions.NotFoundException( parsed_response ) elif response.status == 419: raise HydrusExceptions.SessionException( parsed_response ) elif response.status == 426: raise HydrusExceptions.NetworkVersionException( parsed_response ) elif response.status in ( 500, 501, 502, 503 ): server_header = response.getheader( 'Server' ) if server_header is not None and 'hydrus' in server_header: hydrus_service = True else: hydrus_service = False if response.status == 503 and hydrus_service: raise HydrusExceptions.ServerBusyException( 'Server is busy, please try again later.' ) else: raise Exception( parsed_response ) else: raise Exception( parsed_response )
def Request(self, method, path_and_query, request_headers, body, report_hooks=[], response_to_path=False): if method == HC.GET: method_string = 'GET' elif method == HC.POST: method_string = 'POST' if 'User-Agent' not in request_headers: request_headers['User-Agent'] = 'hydrus/' + HC.u( HC.NETWORK_VERSION) # it is important to only send str, not unicode, to httplib # it uses += to extend the message body, which propagates the unicode (and thus fails) when # you try to push non-ascii bytes as the body (e.g. during a file upload!) method_string = str(method_string) path_and_query = str(path_and_query) request_headers = { str(k): str(v) for (k, v) in request_headers.items() } try: self._connection.request(method_string, path_and_query, headers=request_headers, body=body) response = self._connection.getresponse() except (httplib.CannotSendRequest, httplib.BadStatusLine): # for some reason, we can't send a request on the current connection, so let's make a new one and try again! self._RefreshConnection() self._connection.request(method_string, path_and_query, headers=request_headers, body=body) response = self._connection.getresponse() if response.status == 200 and response_to_path: (temp_path, size_of_response) = self._WriteResponseToPath( response, report_hooks) parsed_response = temp_path else: (parsed_response, size_of_response) = self._ParseResponse(response, report_hooks) response_headers = { k: v for (k, v) in response.getheaders() if k != 'set-cookie' } cookies = self._ParseCookies(response.getheader('set-cookie')) self._last_request_time = HC.GetNow() if response.status == 200: return (parsed_response, None, size_of_response, response_headers, cookies) elif response.status in (301, 302, 303, 307): location = response.getheader('Location') if location is None: raise Exception(parsed_response) else: url = location if ' ' in url: # some booru is giving daft redirect responses print(url) url = urllib.quote(url, safe='/?=&') print(url) if not url.startswith(self._scheme): # assume it is like 'index.php' or '/index.php', rather than 'http://blah.com/index.php' if url.startswith('/'): slash_sep = '' else: slash_sep = '/' url = self._scheme + '://' + self._host + slash_sep + url if response.status in (301, 307): # 301: moved permanently, repeat request # 307: moved temporarily, repeat request redirect_info = (method, url) elif response.status in (302, 303): # 302: moved temporarily, repeat request (except everyone treats it like 303 for no good f*****g reason) # 303: thanks, now go here with GET redirect_info = (HC.GET, url) return (parsed_response, redirect_info, size_of_response, response_headers, cookies) elif response.status == 304: raise HydrusExceptions.NotModifiedException() else: if response.status == 401: raise HydrusExceptions.PermissionException(parsed_response) elif response.status == 403: raise HydrusExceptions.ForbiddenException(parsed_response) elif response.status == 404: raise HydrusExceptions.NotFoundException(parsed_response) elif response.status == 426: raise HydrusExceptions.NetworkVersionException(parsed_response) elif response.status in (500, 501, 502, 503): raise Exception(parsed_response) else: raise Exception(parsed_response)