def wrapper(self, *args, **kwargs):
try:
return meth(self, *args, **kwargs)
except IOError as (errno, strerror):
if errno == ENOSPC:
msg = 'No space left on device'
raise w3afMustStopByKnownReasonExc(msg)
def _increment_global_error_count(self, error, parsed_traceback=[]):
'''
Increment the error count, and if we got a lot of failures raise a
"w3afMustStopException" subtype.
:param error: Exception object.
:param parsed_traceback: A list with the following format:
[('trace_test.py', '9', 'one'), ('trace_test.py', '17', 'two'),
('trace_test.py', '5', 'abc')]
Where ('filename', 'line-number', 'function-name')
'''
if self._ignore_errors_conf:
return
last_errors = self._last_errors
if self._last_request_failed:
last_errors.append((str(error), parsed_traceback))
else:
self._last_request_failed = True
errtotal = len(last_errors)
om.out.debug('Incrementing global error count. GEC: %s' % errtotal)
with self._count_lock:
if errtotal >= 10 and not self._error_stopped:
# Stop using ExtendedUrllib instance
self.stop()
# Known reason errors. See errno module for more info on these
# errors.
EUNKNSERV = -2 # Name or service not known error
EINVHOSTNAME = -5 # No address associated with hostname
known_errors = (EUNKNSERV, ECONNREFUSED, EHOSTUNREACH,
ECONNRESET, ENETDOWN, ENETUNREACH,
EINVHOSTNAME, ETIMEDOUT, ENOSPC)
msg = ('w3af found too many consecutive errors while performing'
' HTTP requests. Either the web server is not reachable'
' anymore or there is an internal error. The last error'
' message is "%s".')
if parsed_traceback:
tback_str = ''
for path, line, call in parsed_traceback[-3:]:
tback_str += ' %s:%s at %s\n' % (path, line, call)
msg += ' The last calls in the traceback are: \n%s' % tback_str
if type(error) is urllib2.URLError:
# URLError exceptions may wrap either httplib.HTTPException
# or socket.error exception instances. We're interested on
# treat'em in a special way.
reason_err = error.reason
reason_msg = None
if isinstance(reason_err, socket.error):
if isinstance(reason_err, socket.sslerror):
reason_msg = 'SSL Error: %s' % error.reason
elif reason_err[0] in known_errors:
reason_msg = str(reason_err)
elif isinstance(reason_err, httplib.HTTPException):
#
# Here we catch:
#
# BadStatusLine, ResponseNotReady, CannotSendHeader,
# CannotSendRequest, ImproperConnectionState,
# IncompleteRead, UnimplementedFileMode, UnknownTransferEncoding,
# UnknownProtocol, InvalidURL, NotConnected.
#
# TODO: Maybe we're being TOO generic in this isinstance?
#
reason_msg = '%s: %s' % (error.__class__.__name__,
error.args)
if reason_msg is not None:
raise w3afMustStopByKnownReasonExc(
msg % error, reason=reason_err)
errors = [] if parsed_traceback else last_errors
raise w3afMustStopByUnknownReasonExc(msg % error, errs=errors)
def do_open(self, req):
'''
Called by handler's url_open method.
'''
host = req.get_host()
if not host:
raise urllib2.URLError('no host given')
try:
resp_statuses = self._hostresp.setdefault(host,
self._get_tail_filter())
# Check if all our last 'resp_statuses' were timeouts and raise
# a w3afMustStopException if this is the case.
if len(resp_statuses) == self._curr_check_failures and \
all(st == RESP_TIMEOUT for st in resp_statuses):
msg = ('w3af found too many consecutive timeouts. The remote '
'webserver seems to be unresponsive; please verify manually.')
reason = 'Timeout while trying to reach target.'
raise w3afMustStopByKnownReasonExc(msg, reason=reason)
conn_factory = self._get_connection
conn = self._cm.get_available_connection(host, conn_factory)
if conn.is_fresh:
# First of all, call the request method. This is needed for
# HTTPS Proxy
if isinstance(conn, ProxyHTTPConnection):
conn.proxy_setup(req.get_full_url())
conn.is_fresh = False
self._start_transaction(conn, req)
resp = conn.getresponse()
else:
# We'll try to use a previously created connection
resp = self._reuse_connection(conn, req, host)
# If the resp is None it means that connection is bad. It was
# possibly closed by the server. Replace it with a new one.
if resp is None:
conn.close()
conn = self._cm.replace_connection(conn, host,
conn_factory)
# First of all, call the request method. This is needed for
# HTTPS Proxy
if isinstance(conn, ProxyHTTPConnection):
conn.proxy_setup(req.get_full_url())
# Try again with the fresh one
conn.is_fresh = False
self._start_transaction(conn, req)
resp = conn.getresponse()
except (socket.error, httplib.HTTPException), err:
# We better discard this connection
self._cm.remove_connection(conn, host)
if isinstance(err, socket.timeout):
resp_statuses.append(RESP_TIMEOUT)
_err = URLTimeoutError()
else:
resp_statuses.append(RESP_BAD)
if isinstance(err, httplib.HTTPException):
err = repr(err)
_err = urllib2.URLError(err)
raise _err
def _handle_error_on_increment(self, error, parsed_traceback, last_errors):
'''
Handle the error
'''
# Stop using ExtendedUrllib instance
self._error_stopped = True
#
# Create a detailed exception message
#
msg = (
'w3af found too many consecutive errors while performing'
' HTTP requests. In most cases this means that the remote web'
' server is not reachable anymore, the network is down, or'
' a WAF is blocking our tests. The last error message was "%s".')
if parsed_traceback:
tback_str = ''
for path, line, call in parsed_traceback[-3:]:
tback_str += ' %s:%s at %s\n' % (path, line, call)
msg += ' The last calls in the traceback are: \n%s' % tback_str
reason_msg = None
if isinstance(error, URLTimeoutError):
# New exception type raised by keepalive handler
reason_msg = error.message
reason_err = error.message
# Exceptions may be of type httplib.HTTPException or socket.error
# We're interested on handling them in different ways
elif isinstance(error, urllib2.URLError):
reason_err = error.reason
# Known reason errors. See errno module for more info on these
# errors.
EUNKNSERV = -2 # Name or service not known error
EINVHOSTNAME = -5 # No address associated with hostname
known_errors = (EUNKNSERV, ECONNREFUSED, EHOSTUNREACH, ECONNRESET,
ENETDOWN, ENETUNREACH, EINVHOSTNAME, ETIMEDOUT,
ENOSPC)
if isinstance(reason_err, socket.error):
if isinstance(reason_err, socket.sslerror):
reason_msg = 'SSL Error: %s' % error.reason
elif reason_err[0] in known_errors:
reason_msg = str(reason_err)
elif isinstance(error, httplib.HTTPException):
#
# Here we catch:
#
# BadStatusLine, ResponseNotReady, CannotSendHeader,
# CannotSendRequest, ImproperConnectionState,
# IncompleteRead, UnimplementedFileMode, UnknownTransferEncoding,
# UnknownProtocol, InvalidURL, NotConnected.
#
# TODO: Maybe we're being TOO generic in this isinstance?
#
reason_msg = '%s: %s' % (error.__class__.__name__, error.args)
reason_err = error.message
# If I got a reason, it means that it is a known exception.
if reason_msg is not None:
raise w3afMustStopByKnownReasonExc(msg % error, reason=reason_err)
else:
errors = [] if parsed_traceback else last_errors
raise w3afMustStopByUnknownReasonExc(msg % error, errs=errors)
def _handle_error_on_increment(self, error, parsed_traceback, last_errors):
'''
Handle the error
'''
# Stop using ExtendedUrllib instance
self._error_stopped = True
#
# Create a detailed exception message
#
msg = ('w3af found too many consecutive errors while performing'
' HTTP requests. In most cases this means that the remote web'
' server is not reachable anymore, the network is down, or'
' a WAF is blocking our tests. The last error message was "%s".')
if parsed_traceback:
tback_str = ''
for path, line, call in parsed_traceback[-3:]:
tback_str += ' %s:%s at %s\n' % (path, line, call)
msg += ' The last calls in the traceback are: \n%s' % tback_str
reason_msg = None
if isinstance(error, URLTimeoutError):
# New exception type raised by keepalive handler
reason_msg = error.message
reason_err = error.message
# Exceptions may be of type httplib.HTTPException or socket.error
# We're interested on handling them in different ways
elif isinstance(error, urllib2.URLError):
reason_err = error.reason
# Known reason errors. See errno module for more info on these
# errors.
EUNKNSERV = -2 # Name or service not known error
EINVHOSTNAME = -5 # No address associated with hostname
known_errors = (EUNKNSERV, ECONNREFUSED, EHOSTUNREACH,
ECONNRESET, ENETDOWN, ENETUNREACH,
EINVHOSTNAME, ETIMEDOUT, ENOSPC)
if isinstance(reason_err, socket.error):
if isinstance(reason_err, socket.sslerror):
reason_msg = 'SSL Error: %s' % error.reason
elif reason_err[0] in known_errors:
reason_msg = str(reason_err)
elif isinstance(error, httplib.HTTPException):
#
# Here we catch:
#
# BadStatusLine, ResponseNotReady, CannotSendHeader,
# CannotSendRequest, ImproperConnectionState,
# IncompleteRead, UnimplementedFileMode, UnknownTransferEncoding,
# UnknownProtocol, InvalidURL, NotConnected.
#
# TODO: Maybe we're being TOO generic in this isinstance?
#
reason_msg = '%s: %s' % (error.__class__.__name__,
error.args)
reason_err = error.message
# If I got a reason, it means that it is a known exception.
if reason_msg is not None:
raise w3afMustStopByKnownReasonExc(msg % error,
reason=reason_err)
else:
errors = [] if parsed_traceback else last_errors
raise w3afMustStopByUnknownReasonExc(msg % error, errs=errors)
def do_open(self, req):
'''
Called by handler's url_open method.
'''
host = req.get_host()
if not host:
raise urllib2.URLError('no host given')
try:
resp_statuses = self._hostresp.setdefault(host,
self._get_tail_filter())
# Check if all our last 'resp_statuses' were timeouts and raise
# a w3afMustStopException if this is the case.
if len(resp_statuses) == self._curr_check_failures and \
all(st == RESP_TIMEOUT for st in resp_statuses):
msg = (
'w3af found too many consecutive timeouts. The remote '
'webserver seems to be unresponsive; please verify manually.'
)
reason = 'Timeout while trying to reach target.'
raise w3afMustStopByKnownReasonExc(msg, reason=reason)
conn_factory = self._get_connection
conn = self._cm.get_available_connection(host, conn_factory)
if conn.is_fresh:
# First of all, call the request method. This is needed for
# HTTPS Proxy
if isinstance(conn, ProxyHTTPConnection):
conn.proxy_setup(req.get_full_url())
conn.is_fresh = False
self._start_transaction(conn, req)
resp = conn.getresponse()
else:
# We'll try to use a previously created connection
resp = self._reuse_connection(conn, req, host)
# If the resp is None it means that connection is bad. It was
# possibly closed by the server. Replace it with a new one.
if resp is None:
conn.close()
conn = self._cm.replace_connection(conn, host,
conn_factory)
# First of all, call the request method. This is needed for
# HTTPS Proxy
if isinstance(conn, ProxyHTTPConnection):
conn.proxy_setup(req.get_full_url())
# Try again with the fresh one
conn.is_fresh = False
self._start_transaction(conn, req)
resp = conn.getresponse()
except (socket.error, httplib.HTTPException), err:
# We better discard this connection
self._cm.remove_connection(conn, host)
if isinstance(err, socket.timeout):
resp_statuses.append(RESP_TIMEOUT)
_err = URLTimeoutError()
else:
resp_statuses.append(RESP_BAD)
if isinstance(err, httplib.HTTPException):
err = repr(err)
_err = urllib2.URLError(err)
raise _err
def _increment_global_error_count(self, error, parsed_traceback=[]):
'''
Increment the error count, and if we got a lot of failures raise a
"w3afMustStopException" subtype.
:param error: Exception object.
:param parsed_traceback: A list with the following format:
[('trace_test.py', '9', 'one'), ('trace_test.py', '17', 'two'),
('trace_test.py', '5', 'abc')]
Where ('filename', 'line-number', 'function-name')
'''
if self._ignore_errors_conf:
return
last_errors = self._last_errors
if self._last_request_failed:
last_errors.append((str(error), parsed_traceback))
else:
self._last_request_failed = True
errtotal = len(last_errors)
om.out.debug('Incrementing global error count. GEC: %s' % errtotal)
with self._count_lock:
if errtotal >= 10 and not self._error_stopped:
# Stop using ExtendedUrllib instance
self.stop()
# Known reason errors. See errno module for more info on these
# errors.
EUNKNSERV = -2 # Name or service not known error
EINVHOSTNAME = -5 # No address associated with hostname
known_errors = (EUNKNSERV, ECONNREFUSED, EHOSTUNREACH,
ECONNRESET, ENETDOWN, ENETUNREACH,
EINVHOSTNAME, ETIMEDOUT, ENOSPC)
msg = ('w3af found too many consecutive errors while performing'
' HTTP requests. Either the web server is not reachable'
' anymore or there is an internal error. The last error'
' message is "%s".')
if parsed_traceback:
tback_str = ''
for path, line, call in parsed_traceback[-3:]:
tback_str += ' %s:%s at %s\n' % (path, line, call)
msg += ' The last calls in the traceback are: \n%s' % tback_str
if type(error) is urllib2.URLError:
# URLError exceptions may wrap either httplib.HTTPException
# or socket.error exception instances. We're interested on
# treat'em in a special way.
reason_err = error.reason
reason_msg = None
if isinstance(reason_err, socket.error):
if isinstance(reason_err, socket.sslerror):
reason_msg = 'SSL Error: %s' % error.reason
elif reason_err[0] in known_errors:
reason_msg = str(reason_err)
elif isinstance(reason_err, httplib.HTTPException):
#
# Here we catch:
#
# BadStatusLine, ResponseNotReady, CannotSendHeader,
# CannotSendRequest, ImproperConnectionState,
# IncompleteRead, UnimplementedFileMode, UnknownTransferEncoding,
# UnknownProtocol, InvalidURL, NotConnected.
#
# TODO: Maybe we're being TOO generic in this isinstance?
#
reason_msg = '%s: %s' % (error.__class__.__name__,
error.args)
if reason_msg is not None:
raise w3afMustStopByKnownReasonExc(
msg % error, reason=reason_err)
errors = [] if parsed_traceback else last_errors
raise w3afMustStopByUnknownReasonExc(msg % error, errs=errors)
def do_open(self, req):
'''
Called by handler's url_open method.
'''
host = req.get_host()
if not host:
raise urllib2.URLError('no host given')
try:
resp_statuses = self._hostresp.setdefault(host,
self._get_tail_filter())
# Check if all our last 'resp_statuses' were timeouts and raise
# a w3afMustStopException if this is the case.
if len(resp_statuses) == self._curr_check_failures:
# Only copy the resp_statuses when we want to iterate it to
# verify which responses are timeouts. This is done in order
# to avoid issues with "deque mutated during iteration" because
# other threads might be adding stuff to the deque while we
# run the "... for st in resp..."
resp_statuses_copy = copy.copy(resp_statuses)
if all(st == RESP_TIMEOUT for st in resp_statuses_copy):
msg = ('w3af found too many consecutive timeouts. The'
' remote webserver seems to be unresponsive; please'
' verify manually.')
reason = 'Timeout while trying to reach target.'
raise w3afMustStopByKnownReasonExc(msg, reason=reason)
conn_factory = self._get_connection
conn = self._cm.get_available_connection(host, conn_factory)
if conn.is_fresh:
# First of all, call the request method. This is needed for
# HTTPS Proxy
if isinstance(conn, ProxyHTTPConnection):
conn.proxy_setup(req.get_full_url())
conn.is_fresh = False
self._start_transaction(conn, req)
resp = conn.getresponse()
else:
# We'll try to use a previously created connection
resp = self._reuse_connection(conn, req, host)
# If the resp is None it means that connection is bad. It was
# possibly closed by the server. Replace it with a new one.
if resp is None:
conn.close()
conn = self._cm.replace_connection(conn, host,
conn_factory)
# First of all, call the request method. This is needed for
# HTTPS Proxy
if isinstance(conn, ProxyHTTPConnection):
conn.proxy_setup(req.get_full_url())
# Try again with the fresh one
conn.is_fresh = False
self._start_transaction(conn, req)
resp = conn.getresponse()
except socket.timeout:
# We better discard this connection
self._cm.remove_connection(conn, host)
resp_statuses.append(RESP_TIMEOUT)
raise URLTimeoutError()
except socket.error:
# We better discard this connection
self._cm.remove_connection(conn, host)
resp_statuses.append(RESP_BAD)
raise
except httplib.HTTPException:
# We better discard this connection
self._cm.remove_connection(conn, host)
resp_statuses.append(RESP_BAD)
raise
else:
# This response seems to be fine
resp_statuses.append(RESP_OK)
# If not a persistent connection, don't try to reuse it
if resp.will_close:
self._cm.remove_connection(conn, host)
if DEBUG:
om.out.debug("STATUS: %s, %s" % (resp.status, resp.reason))
resp._handler = self
resp._host = host
resp._url = req.get_full_url()
resp._connection = conn
resp.code = resp.status
resp.headers = resp.msg
resp.msg = resp.reason
return resp
