def _generic_send_error_handler(self, req, exception, grep, original_url):
if not req.error_handling:
msg = u'Raising HTTP error "%s" "%s". Reason: "%s". Error' \
u' handling was disabled for this request.'
om.out.debug(msg % (req.get_method(), original_url, exception))
error_str = get_exception_reason(exception) or str(exception)
raise HTTPRequestException(error_str, request=req)
# Log the error
msg = u'Failed to HTTP "%s" "%s". Reason: "%s", going to retry.'
om.out.debug(msg % (req.get_method(), original_url, exception))
# Don't make a lot of noise on URLTimeoutError which is pretty common
# and properly handled by this library
if not isinstance(exception, URLTimeoutError):
msg = 'Traceback for this error: %s'
om.out.debug(msg % traceback.format_exc())
with self._count_lock:
self._log_failed_response(exception, req)
should_stop_scan = self._should_stop_scan(req)
if should_stop_scan:
self._handle_error_count_exceeded(exception)
# Then retry!
req._Request__original = original_url
return self._retry(req, grep, exception)
def _check_uri(self, req):
if req.get_full_url().startswith('http'):
return True
elif req.get_full_url().startswith('javascript:') or \
req.get_full_url().startswith('mailto:'):
msg = 'Unsupported URL: "%s"'
raise HTTPRequestException(msg % req.get_full_url(), request=req)
else:
return False
def connect(self):
"""
Connect using different SSL protocols
"""
for protocol in _protocols:
ProxyHTTPConnection.connect(self)
self.sock = self.make_ssl_aware(self.sock, protocol)
if self.sock is not None:
break
else:
msg = 'Unable to create a proxied SSL connection'
raise HTTPRequestException(msg)
def connect(self):
"""
Test the different SSL protocols
"""
for protocol in _protocols:
sock = self.connect_socket()
sock = self.make_ssl_aware(sock, protocol)
if sock is not None:
break
else:
msg = 'Unable to create a SSL connection using protocols: %s'
protocols = ', '.join([str(p) for p in _protocols])
raise HTTPRequestException(msg % protocols)
def _generic_send_error_handler(self, req, exception, grep, original_url):
if req.ignore_errors:
msg = 'Ignoring HTTP error "%s" "%s". Reason: "%s"'
om.out.debug(msg % (req.get_method(), original_url, exception))
error_str = self.get_exception_reason(exception) or str(exception)
raise HTTPRequestException(error_str, request=req)
# Log the error
msg = 'Failed to HTTP "%s" "%s". Reason: "%s", going to retry.'
om.out.debug(msg % (req.get_method(), original_url, exception))
om.out.debug('Traceback for this error: %s' % traceback.format_exc())
# Then retry!
req._Request__original = original_url
return self._retry(req, grep, exception)
def get_remote_file_size(self, req, cache=True):
"""
This method was previously used in the framework to perform a HEAD
request before each GET/POST (ouch!) and get the size of the response.
The bad thing was that I was performing two requests for each
resource... I moved the "protection against big files" to the
keepalive.py module.
I left it here because maybe I want to use it at some point. Mainly
to call it directly.
:return: The file size of the remote file.
"""
res = self.HEAD(req.get_full_url(),
headers=req.headers,
data=req.get_data(),
cache=cache)
resource_length = None
for i in res.get_headers():
if i.lower() == 'content-length':
resource_length = res.get_headers()[i]
if resource_length.isdigit():
resource_length = int(resource_length)
else:
msg = 'The content length header value of the response'\
' wasn\'t an integer, this is strange... The value'\
' is: "%s".'
om.out.error(msg % res.get_headers()[i])
raise HTTPRequestException(msg, request=req)
if resource_length is not None:
return resource_length
else:
msg = 'The response didn\'t contain a content-length header.'\
' Unable to return the remote file size of request with'\
' id: %s' % res.id
om.out.debug(msg)
# I prefer to fetch the file, before this om.out.debug was a
# "raise BaseFrameworkException", but this didn't make much sense
return 0
def _retry(self, req, grep, url_error):
"""
Try to send the request again while doing some error handling.
"""
req.retries_left -= 1
if req.retries_left > 0:
msg = 'Re-sending request "%s" after initial exception: "%s"'
om.out.debug(msg % (req, url_error))
return self._send(req, grep=grep)
else:
# Please note that I'm raising HTTPRequestException and not a
# ScanMustStopException (or subclasses) since I don't want the
# scan to stop because of a single HTTP request failing.
#
# Actually we get here if one request fails three times to be sent
# but that might be because of the http request itself and not a
# fault of the framework/server/network.
error_str = self.get_exception_reason(url_error) or str(url_error)
raise HTTPRequestException(error_str, request=req)
def assert_allowed_proto(self, req):
full_url = req.get_full_url().lower()
if not full_url.startswith('http'):
msg = 'Unsupported URL: "%s"'
raise HTTPRequestException(msg % req.get_full_url(), request=req)
class KeepAliveHandler(object):
def __init__(self):
# Create the connection pool instance
#
# Note: In the initial code this connection manager was created at
# the module level and was shared between the HTTP and HTTPS
# keep alive handlers. This was buggy since at any point if
# a user requested http://host.tld and then https://host.tld
# a connection to the HTTP one was returned from the manager
# for the HTTPS request.
#
# This change lets us still keep a "persistent" connection
# manager since our opener_settings and xurllib will only
# create one instance for the KeepAliveHandler and use that
# during the whole scan.
self._cm = ConnectionManager()
# Typically a urllib2.OpenerDirector instance. Set by the
# urllib2 mechanism.
self.parent = None
self._pool_lock = threading.RLock()
# Map hosts to a `collections.deque` of response status.
self._hostresp = {}
def close_connection(self, host):
"""
Close connection(s) to <host>
host is the host:port spec, as in 'www.cnn.com:8080' as passed in.
no error occurs if there is no connection to that host.
"""
for conn in self._cm.get_all(host):
self._cm.remove_connection(conn, reason='close connection')
def close_all(self):
"""
Close all open connections
"""
debug('Closing all connections')
for conn in self._cm.get_all():
self._cm.remove_connection(conn, reason='close all connections')
def _request_closed(self, connection):
"""
This request is now closed and that the connection is ready for another
request
"""
debug('Add %s to free-to-use connection list' % connection)
self._cm.free_connection(connection)
def _remove_connection(self, conn):
self._cm.remove_connection(conn, reason='remove connection')
def do_open(self, req):
"""
Called by handler's url_open method.
"""
host = req.get_host()
if not host:
raise urllib2.URLError('no host given')
conn_factory = self.get_connection
try:
conn = self._cm.get_available_connection(req, conn_factory)
except ConnectionPoolException:
# When `self._cm.get_available_connection(host, conn_factory)` does
# not return a conn, it will raise this exception. So we either get
# here and `raise`, or we have a connection and something else
# failed and we get to the other error handlers.
raise
try:
if conn.is_fresh:
resp, start = self._get_response(conn, req)
else:
# We'll try to use a previously created connection
start = time.time()
resp = self._reuse_connection(conn, req, host)
# If the resp is None it means that connection is bad. It was
# possibly closed by the server. Replace it with a new one.
if resp is None:
conn = self._cm.replace_connection(conn, req, conn_factory)
resp, start = self._get_response(conn, req)
except socket.timeout:
# We better discard this connection
self._cm.remove_connection(conn, reason='socket timeout')
raise URLTimeoutError()
except OpenSSL.SSL.ZeroReturnError:
# According to the pyOpenSSL docs ZeroReturnError means that the
# SSL connection has been closed cleanly
self._cm.remove_connection(conn, reason='ZeroReturnError')
raise
except (socket.error, httplib.HTTPException, OpenSSL.SSL.SysCallError):
# We better discard this connection
self._cm.remove_connection(conn, reason='socket error')
raise
except Exception, e:
# We better discard this connection, we don't even know what happen!
reason = 'unexpected exception "%s"' % e
self._cm.remove_connection(conn, reason=reason)
raise
# How many requests were sent with this connection?
conn.inc_req_count()
# If not a persistent connection, or the user specified that he wanted
# a new connection for this specific request, don't try to reuse it
if resp.will_close or req.new_connection:
self._cm.remove_connection(conn, reason='will close')
# This response seems to be fine
resp._handler = self
resp._host = host
resp._url = req.get_full_url()
resp._connection = conn
resp.code = resp.status
resp.headers = resp.msg
resp.msg = resp.reason
try:
resp.read()
except AttributeError:
# The rare case of: 'NoneType' object has no attribute 'recv', we
# read the response here because we're closer to the error and can
# better understand it.
#
# https://github.com/andresriancho/w3af/issues/2074
self._cm.remove_connection(conn, reason='http connection died')
raise HTTPRequestException('The HTTP connection died')
except Exception, e:
# We better discard this connection, we don't even know what happen!
reason = 'unexpected exception while reading "%s"' % e
self._cm.remove_connection(conn, reason=reason)
raise
def do_open(self, req):
"""
Called by handler's url_open method.
"""
host = req.get_host()
if not host:
raise urllib2.URLError('no host given')
conn_factory = self._get_connection
try:
conn = self._cm.get_available_connection(host, conn_factory)
except ConnectionPoolException:
# When `self._cm.get_available_connection(host, conn_factory)` does
# not return a conn, it will raise this exception. So we either get
# here and `raise`, or we have a connection and something else
# failed and we get to the other error handlers.
raise
try:
if conn.is_fresh:
# First of all, call the request method. This is needed for
# HTTPS Proxy
if isinstance(conn, ProxyHTTPConnection):
conn.proxy_setup(req.get_full_url())
conn.is_fresh = False
start = time.time()
self._start_transaction(conn, req)
resp = conn.getresponse()
else:
# We'll try to use a previously created connection
start = time.time()
resp = self._reuse_connection(conn, req, host)
# If the resp is None it means that connection is bad. It was
# possibly closed by the server. Replace it with a new one.
if resp is None:
conn.close()
conn = self._cm.replace_connection(conn, host,
conn_factory)
# First of all, call the request method. This is needed for
# HTTPS Proxy
if isinstance(conn, ProxyHTTPConnection):
conn.proxy_setup(req.get_full_url())
# Try again with the fresh one
conn.is_fresh = False
start = time.time()
self._start_transaction(conn, req)
resp = conn.getresponse()
except socket.timeout:
# We better discard this connection
self._cm.remove_connection(conn, host)
raise URLTimeoutError()
except socket.error:
# We better discard this connection
self._cm.remove_connection(conn, host)
raise
except httplib.HTTPException:
# We better discard this connection
self._cm.remove_connection(conn, host)
raise
# This response seems to be fine
# If not a persistent connection, don't try to reuse it
if resp.will_close:
self._cm.remove_connection(conn, host)
resp._handler = self
resp._host = host
resp._url = req.get_full_url()
resp._connection = conn
resp.code = resp.status
resp.headers = resp.msg
resp.msg = resp.reason
try:
resp.read()
except AttributeError:
# The rare case of: 'NoneType' object has no attribute 'recv', we
# read the response here because we're closer to the error and can
# better understand it.
#
# https://github.com/andresriancho/w3af/issues/2074
self._cm.remove_connection(conn, host)
raise HTTPRequestException('The HTTP connection died')
# We measure time here because it's the best place we know of
elapsed = time.time() - start
resp.set_wait_time(elapsed)
debug("HTTP response: %s, %s" % (resp.status, resp.reason))
return resp
def do_open(self, req):
"""
Called by handler's url_open method.
"""
host = req.get_host()
if not host:
raise urllib2.URLError('no host given')
conn_factory = self.get_connection
try:
conn = self._cm.get_available_connection(req, conn_factory)
except ConnectionPoolException:
# When `self._cm.get_available_connection(host, conn_factory)` does
# not return a conn, it will raise this exception. So we either get
# here and `raise`, or we have a connection and something else
# failed and we get to the other error handlers.
raise
try:
if conn.is_fresh:
resp, start = self._get_response(conn, req)
else:
# We'll try to use a previously created connection
start = time.time()
resp = self._reuse_connection(conn, req, host)
# If the resp is None it means that connection is bad. It was
# possibly closed by the server. Replace it with a new one.
if resp is None:
conn = self._cm.replace_connection(conn, req, conn_factory)
resp, start = self._get_response(conn, req)
except socket.timeout:
# We better discard this connection
self._cm.remove_connection(conn, host, reason='socket timeout')
raise URLTimeoutError()
except OpenSSL.SSL.ZeroReturnError:
# According to the pyOpenSSL docs ZeroReturnError means that the
# SSL connection has been closed cleanly
self._cm.remove_connection(conn, host, reason='ZeroReturnError')
raise
except (socket.error, httplib.HTTPException, OpenSSL.SSL.SysCallError):
# We better discard this connection
self._cm.remove_connection(conn, host, reason='socket error')
raise
# If not a persistent connection, or the user specified that he wanted
# a new connection for this specific request, don't try to reuse it
if resp.will_close or req.new_connection:
self._cm.remove_connection(conn, host, reason='will close')
# How many requests were sent with this connection?
conn.inc_req_count()
# This response seems to be fine
resp._handler = self
resp._host = host
resp._url = req.get_full_url()
resp._connection = conn
resp.code = resp.status
resp.headers = resp.msg
resp.msg = resp.reason
try:
resp.read()
except AttributeError:
# The rare case of: 'NoneType' object has no attribute 'recv', we
# read the response here because we're closer to the error and can
# better understand it.
#
# https://github.com/andresriancho/w3af/issues/2074
self._cm.remove_connection(conn, host, reason='http connection died')
raise HTTPRequestException('The HTTP connection died')
# We measure time here because it's the best place we know of
elapsed = time.time() - start
resp.set_wait_time(elapsed)
msg = "HTTP response: %s - %s - %s with %s in %s seconds"
args = (req.get_selector(), resp.status, resp.reason, conn, elapsed)
debug(msg % args)
return resp