def _generic_send_error_handler(self, req, exception, grep, original_url): if not req.error_handling: msg = u'Raising HTTP error "%s" "%s". Reason: "%s". Error' \ u' handling was disabled for this request.' om.out.debug(msg % (req.get_method(), original_url, exception)) error_str = get_exception_reason(exception) or str(exception) raise HTTPRequestException(error_str, request=req) # Log the error msg = u'Failed to HTTP "%s" "%s". Reason: "%s", going to retry.' om.out.debug(msg % (req.get_method(), original_url, exception)) # Don't make a lot of noise on URLTimeoutError which is pretty common # and properly handled by this library if not isinstance(exception, URLTimeoutError): msg = 'Traceback for this error: %s' om.out.debug(msg % traceback.format_exc()) with self._count_lock: self._log_failed_response(exception, req) should_stop_scan = self._should_stop_scan(req) if should_stop_scan: self._handle_error_count_exceeded(exception) # Then retry! req._Request__original = original_url return self._retry(req, grep, exception)
def _check_uri(self, req): if req.get_full_url().startswith('http'): return True elif req.get_full_url().startswith('javascript:') or \ req.get_full_url().startswith('mailto:'): msg = 'Unsupported URL: "%s"' raise HTTPRequestException(msg % req.get_full_url(), request=req) else: return False
def connect(self): """ Connect using different SSL protocols """ for protocol in _protocols: ProxyHTTPConnection.connect(self) self.sock = self.make_ssl_aware(self.sock, protocol) if self.sock is not None: break else: msg = 'Unable to create a proxied SSL connection' raise HTTPRequestException(msg)
def connect(self): """ Test the different SSL protocols """ for protocol in _protocols: sock = self.connect_socket() sock = self.make_ssl_aware(sock, protocol) if sock is not None: break else: msg = 'Unable to create a SSL connection using protocols: %s' protocols = ', '.join([str(p) for p in _protocols]) raise HTTPRequestException(msg % protocols)
def _generic_send_error_handler(self, req, exception, grep, original_url): if req.ignore_errors: msg = 'Ignoring HTTP error "%s" "%s". Reason: "%s"' om.out.debug(msg % (req.get_method(), original_url, exception)) error_str = self.get_exception_reason(exception) or str(exception) raise HTTPRequestException(error_str, request=req) # Log the error msg = 'Failed to HTTP "%s" "%s". Reason: "%s", going to retry.' om.out.debug(msg % (req.get_method(), original_url, exception)) om.out.debug('Traceback for this error: %s' % traceback.format_exc()) # Then retry! req._Request__original = original_url return self._retry(req, grep, exception)
def get_remote_file_size(self, req, cache=True): """ This method was previously used in the framework to perform a HEAD request before each GET/POST (ouch!) and get the size of the response. The bad thing was that I was performing two requests for each resource... I moved the "protection against big files" to the keepalive.py module. I left it here because maybe I want to use it at some point. Mainly to call it directly. :return: The file size of the remote file. """ res = self.HEAD(req.get_full_url(), headers=req.headers, data=req.get_data(), cache=cache) resource_length = None for i in res.get_headers(): if i.lower() == 'content-length': resource_length = res.get_headers()[i] if resource_length.isdigit(): resource_length = int(resource_length) else: msg = 'The content length header value of the response'\ ' wasn\'t an integer, this is strange... The value'\ ' is: "%s".' om.out.error(msg % res.get_headers()[i]) raise HTTPRequestException(msg, request=req) if resource_length is not None: return resource_length else: msg = 'The response didn\'t contain a content-length header.'\ ' Unable to return the remote file size of request with'\ ' id: %s' % res.id om.out.debug(msg) # I prefer to fetch the file, before this om.out.debug was a # "raise BaseFrameworkException", but this didn't make much sense return 0
def _retry(self, req, grep, url_error): """ Try to send the request again while doing some error handling. """ req.retries_left -= 1 if req.retries_left > 0: msg = 'Re-sending request "%s" after initial exception: "%s"' om.out.debug(msg % (req, url_error)) return self._send(req, grep=grep) else: # Please note that I'm raising HTTPRequestException and not a # ScanMustStopException (or subclasses) since I don't want the # scan to stop because of a single HTTP request failing. # # Actually we get here if one request fails three times to be sent # but that might be because of the http request itself and not a # fault of the framework/server/network. error_str = self.get_exception_reason(url_error) or str(url_error) raise HTTPRequestException(error_str, request=req)
def assert_allowed_proto(self, req): full_url = req.get_full_url().lower() if not full_url.startswith('http'): msg = 'Unsupported URL: "%s"' raise HTTPRequestException(msg % req.get_full_url(), request=req)
class KeepAliveHandler(object): def __init__(self): # Create the connection pool instance # # Note: In the initial code this connection manager was created at # the module level and was shared between the HTTP and HTTPS # keep alive handlers. This was buggy since at any point if # a user requested http://host.tld and then https://host.tld # a connection to the HTTP one was returned from the manager # for the HTTPS request. # # This change lets us still keep a "persistent" connection # manager since our opener_settings and xurllib will only # create one instance for the KeepAliveHandler and use that # during the whole scan. self._cm = ConnectionManager() # Typically a urllib2.OpenerDirector instance. Set by the # urllib2 mechanism. self.parent = None self._pool_lock = threading.RLock() # Map hosts to a `collections.deque` of response status. self._hostresp = {} def close_connection(self, host): """ Close connection(s) to <host> host is the host:port spec, as in 'www.cnn.com:8080' as passed in. no error occurs if there is no connection to that host. """ for conn in self._cm.get_all(host): self._cm.remove_connection(conn, reason='close connection') def close_all(self): """ Close all open connections """ debug('Closing all connections') for conn in self._cm.get_all(): self._cm.remove_connection(conn, reason='close all connections') def _request_closed(self, connection): """ This request is now closed and that the connection is ready for another request """ debug('Add %s to free-to-use connection list' % connection) self._cm.free_connection(connection) def _remove_connection(self, conn): self._cm.remove_connection(conn, reason='remove connection') def do_open(self, req): """ Called by handler's url_open method. """ host = req.get_host() if not host: raise urllib2.URLError('no host given') conn_factory = self.get_connection try: conn = self._cm.get_available_connection(req, conn_factory) except ConnectionPoolException: # When `self._cm.get_available_connection(host, conn_factory)` does # not return a conn, it will raise this exception. So we either get # here and `raise`, or we have a connection and something else # failed and we get to the other error handlers. raise try: if conn.is_fresh: resp, start = self._get_response(conn, req) else: # We'll try to use a previously created connection start = time.time() resp = self._reuse_connection(conn, req, host) # If the resp is None it means that connection is bad. It was # possibly closed by the server. Replace it with a new one. if resp is None: conn = self._cm.replace_connection(conn, req, conn_factory) resp, start = self._get_response(conn, req) except socket.timeout: # We better discard this connection self._cm.remove_connection(conn, reason='socket timeout') raise URLTimeoutError() except OpenSSL.SSL.ZeroReturnError: # According to the pyOpenSSL docs ZeroReturnError means that the # SSL connection has been closed cleanly self._cm.remove_connection(conn, reason='ZeroReturnError') raise except (socket.error, httplib.HTTPException, OpenSSL.SSL.SysCallError): # We better discard this connection self._cm.remove_connection(conn, reason='socket error') raise except Exception, e: # We better discard this connection, we don't even know what happen! reason = 'unexpected exception "%s"' % e self._cm.remove_connection(conn, reason=reason) raise # How many requests were sent with this connection? conn.inc_req_count() # If not a persistent connection, or the user specified that he wanted # a new connection for this specific request, don't try to reuse it if resp.will_close or req.new_connection: self._cm.remove_connection(conn, reason='will close') # This response seems to be fine resp._handler = self resp._host = host resp._url = req.get_full_url() resp._connection = conn resp.code = resp.status resp.headers = resp.msg resp.msg = resp.reason try: resp.read() except AttributeError: # The rare case of: 'NoneType' object has no attribute 'recv', we # read the response here because we're closer to the error and can # better understand it. # # https://github.com/andresriancho/w3af/issues/2074 self._cm.remove_connection(conn, reason='http connection died') raise HTTPRequestException('The HTTP connection died') except Exception, e: # We better discard this connection, we don't even know what happen! reason = 'unexpected exception while reading "%s"' % e self._cm.remove_connection(conn, reason=reason) raise
def do_open(self, req): """ Called by handler's url_open method. """ host = req.get_host() if not host: raise urllib2.URLError('no host given') conn_factory = self._get_connection try: conn = self._cm.get_available_connection(host, conn_factory) except ConnectionPoolException: # When `self._cm.get_available_connection(host, conn_factory)` does # not return a conn, it will raise this exception. So we either get # here and `raise`, or we have a connection and something else # failed and we get to the other error handlers. raise try: if conn.is_fresh: # First of all, call the request method. This is needed for # HTTPS Proxy if isinstance(conn, ProxyHTTPConnection): conn.proxy_setup(req.get_full_url()) conn.is_fresh = False start = time.time() self._start_transaction(conn, req) resp = conn.getresponse() else: # We'll try to use a previously created connection start = time.time() resp = self._reuse_connection(conn, req, host) # If the resp is None it means that connection is bad. It was # possibly closed by the server. Replace it with a new one. if resp is None: conn.close() conn = self._cm.replace_connection(conn, host, conn_factory) # First of all, call the request method. This is needed for # HTTPS Proxy if isinstance(conn, ProxyHTTPConnection): conn.proxy_setup(req.get_full_url()) # Try again with the fresh one conn.is_fresh = False start = time.time() self._start_transaction(conn, req) resp = conn.getresponse() except socket.timeout: # We better discard this connection self._cm.remove_connection(conn, host) raise URLTimeoutError() except socket.error: # We better discard this connection self._cm.remove_connection(conn, host) raise except httplib.HTTPException: # We better discard this connection self._cm.remove_connection(conn, host) raise # This response seems to be fine # If not a persistent connection, don't try to reuse it if resp.will_close: self._cm.remove_connection(conn, host) resp._handler = self resp._host = host resp._url = req.get_full_url() resp._connection = conn resp.code = resp.status resp.headers = resp.msg resp.msg = resp.reason try: resp.read() except AttributeError: # The rare case of: 'NoneType' object has no attribute 'recv', we # read the response here because we're closer to the error and can # better understand it. # # https://github.com/andresriancho/w3af/issues/2074 self._cm.remove_connection(conn, host) raise HTTPRequestException('The HTTP connection died') # We measure time here because it's the best place we know of elapsed = time.time() - start resp.set_wait_time(elapsed) debug("HTTP response: %s, %s" % (resp.status, resp.reason)) return resp
def do_open(self, req): """ Called by handler's url_open method. """ host = req.get_host() if not host: raise urllib2.URLError('no host given') conn_factory = self.get_connection try: conn = self._cm.get_available_connection(req, conn_factory) except ConnectionPoolException: # When `self._cm.get_available_connection(host, conn_factory)` does # not return a conn, it will raise this exception. So we either get # here and `raise`, or we have a connection and something else # failed and we get to the other error handlers. raise try: if conn.is_fresh: resp, start = self._get_response(conn, req) else: # We'll try to use a previously created connection start = time.time() resp = self._reuse_connection(conn, req, host) # If the resp is None it means that connection is bad. It was # possibly closed by the server. Replace it with a new one. if resp is None: conn = self._cm.replace_connection(conn, req, conn_factory) resp, start = self._get_response(conn, req) except socket.timeout: # We better discard this connection self._cm.remove_connection(conn, host, reason='socket timeout') raise URLTimeoutError() except OpenSSL.SSL.ZeroReturnError: # According to the pyOpenSSL docs ZeroReturnError means that the # SSL connection has been closed cleanly self._cm.remove_connection(conn, host, reason='ZeroReturnError') raise except (socket.error, httplib.HTTPException, OpenSSL.SSL.SysCallError): # We better discard this connection self._cm.remove_connection(conn, host, reason='socket error') raise # If not a persistent connection, or the user specified that he wanted # a new connection for this specific request, don't try to reuse it if resp.will_close or req.new_connection: self._cm.remove_connection(conn, host, reason='will close') # How many requests were sent with this connection? conn.inc_req_count() # This response seems to be fine resp._handler = self resp._host = host resp._url = req.get_full_url() resp._connection = conn resp.code = resp.status resp.headers = resp.msg resp.msg = resp.reason try: resp.read() except AttributeError: # The rare case of: 'NoneType' object has no attribute 'recv', we # read the response here because we're closer to the error and can # better understand it. # # https://github.com/andresriancho/w3af/issues/2074 self._cm.remove_connection(conn, host, reason='http connection died') raise HTTPRequestException('The HTTP connection died') # We measure time here because it's the best place we know of elapsed = time.time() - start resp.set_wait_time(elapsed) msg = "HTTP response: %s - %s - %s with %s in %s seconds" args = (req.get_selector(), resp.status, resp.reason, conn, elapsed) debug(msg % args) return resp