def test_many_parsers_timing_out(self):
"""
Received more reports of parsers timing out, and after that
w3af showing always "The parser took more than X seconds to complete
parsing of" for all calls to the parser.
Want to test how well the the parser recovers from many timeouts.
"""
mmpdp = 'w3af.core.data.parsers.mp_document_parser.%s'
kmpdp = mmpdp % 'MultiProcessingDocumentParser.%s'
modp = 'w3af.core.data.parsers.document_parser.%s'
with patch(mmpdp % 'om.out') as om_mock,\
patch(kmpdp % 'PARSER_TIMEOUT', new_callable=PropertyMock) as timeout_mock,\
patch(kmpdp % 'MAX_WORKERS', new_callable=PropertyMock) as max_workers_mock,\
patch(modp % 'DocumentParser.PARSERS', new_callable=PropertyMock) as parsers_mock:
# Prepare the HTTP responses
html_trigger_delay = '<html>DelayedParser!</html>%s'
html_ok = '<html>foo-</html>%s'
# Mocks
timeout_mock.return_value = 1
max_workers_mock.return_value = 5
parsers_mock.return_value = [DelayedParser, HTMLParser]
ITERATIONS = 25
#
# Lets timeout many sequentially
#
for i in xrange(ITERATIONS):
http_resp = _build_http_response(html_trigger_delay % i,
u'text/html')
try:
self.mpdoc.get_document_parser_for(http_resp)
except TimeoutError, toe:
self._is_timeout_exception_message(toe, om_mock, http_resp)
else:
self.assertTrue(False)
#
# Lets timeout randomly
#
for i in xrange(ITERATIONS):
html = random.choice([html_trigger_delay, html_ok])
http_resp = _build_http_response(html % i, u'text/html')
try:
parser = self.mpdoc.get_document_parser_for(http_resp)
except TimeoutError, toe:
self._is_timeout_exception_message(toe, om_mock, http_resp)
else:
self.assertIsInstance(parser._parser, HTMLParser)
def test_many_parsers_timing_out(self):
"""
Received more reports of parsers timing out, and after that
w3af showing always "The parser took more than X seconds to complete
parsing of" for all calls to the parser.
Want to test how well the the parser recovers from many timeouts.
"""
mmpdp = 'w3af.core.data.parsers.mp_document_parser.%s'
kmpdp = mmpdp % 'MultiProcessingDocumentParser.%s'
modp = 'w3af.core.data.parsers.document_parser.%s'
with patch(mmpdp % 'om.out') as om_mock,\
patch(kmpdp % 'PARSER_TIMEOUT', new_callable=PropertyMock) as timeout_mock,\
patch(kmpdp % 'MAX_WORKERS', new_callable=PropertyMock) as max_workers_mock,\
patch(modp % 'DocumentParser.PARSERS', new_callable=PropertyMock) as parsers_mock:
# Prepare the HTTP responses
html_trigger_delay = '<html>DelayedParser!</html>%s'
html_ok = '<html>foo-</html>%s'
# Mocks
timeout_mock.return_value = 1
max_workers_mock.return_value = 5
parsers_mock.return_value = [DelayedParser, HTMLParser]
ITERATIONS = 25
#
# Lets timeout many sequentially
#
for i in xrange(ITERATIONS):
http_resp = _build_http_response(html_trigger_delay % i, u'text/html')
try:
self.mpdoc.get_document_parser_for(http_resp)
except TimeoutError, toe:
self._is_timeout_exception_message(toe, om_mock, http_resp)
else:
self.assertTrue(False)
#
# Lets timeout randomly
#
for i in xrange(ITERATIONS):
html = random.choice([html_trigger_delay, html_ok])
http_resp = _build_http_response(html % i, u'text/html')
try:
parser = self.mpdoc.get_document_parser_for(http_resp)
except TimeoutError, toe:
self._is_timeout_exception_message(toe, om_mock, http_resp)
else:
self.assertIsInstance(parser._parser, HTMLParser)
def test_parser_timeout(self):
"""
Test to verify fix for https://github.com/andresriancho/w3af/issues/6723
"w3af running long time more than 24h"
"""
mmpdp = 'w3af.core.data.parsers.mp_document_parser.%s'
kmpdp = mmpdp % 'MultiProcessingDocumentParser.%s'
modp = 'w3af.core.data.parsers.document_parser.%s'
with patch(mmpdp % 'om.out') as om_mock,\
patch(kmpdp % 'PARSER_TIMEOUT', new_callable=PropertyMock) as timeout_mock,\
patch(kmpdp % 'MAX_WORKERS', new_callable=PropertyMock) as max_workers_mock,\
patch(modp % 'DocumentParser.PARSERS', new_callable=PropertyMock) as parsers_mock:
#
# Test the timeout
#
html = '<html>DelayedParser!</html>'
http_resp = _build_http_response(html, u'text/html')
timeout_mock.return_value = 1
max_workers_mock.return_value = 1
parsers_mock.return_value = [DelayedParser, HTMLParser]
try:
self.mpdoc.get_document_parser_for(http_resp)
except BaseFrameworkException:
msg = '[timeout] The parser took more than %s seconds'\
' to complete parsing of "%s", killed it!'
error = msg % (MultiProcessingDocumentParser.PARSER_TIMEOUT,
http_resp.get_url())
self.assertIn(call.debug(error), om_mock.mock_calls)
else:
self.assertTrue(False)
#
# We now want to make sure that after we kill the process the Pool
# creates a new process for handling our tasks
#
# https://github.com/andresriancho/w3af/issues/9713
#
html = '<html>foo-</html>'
http_resp = _build_http_response(html, u'text/html')
doc_parser = self.mpdoc.get_document_parser_for(http_resp)
self.assertIsInstance(doc_parser._parser, HTMLParser)
def test_parser_timeout(self):
"""
Test to verify fix for https://github.com/andresriancho/w3af/issues/6723
"w3af running long time more than 24h"
"""
mmpdp = 'w3af.core.data.parsers.mp_document_parser.%s'
kmpdp = mmpdp % 'MultiProcessingDocumentParser.%s'
modp = 'w3af.core.data.parsers.document_parser.%s'
with patch(mmpdp % 'om.out') as om_mock,\
patch(kmpdp % 'PARSER_TIMEOUT', new_callable=PropertyMock) as timeout_mock,\
patch(kmpdp % 'MAX_WORKERS', new_callable=PropertyMock) as max_workers_mock,\
patch(modp % 'DocumentParser.PARSERS', new_callable=PropertyMock) as parsers_mock:
#
# Test the timeout
#
html = '<html>DelayedParser!</html>'
http_resp = _build_http_response(html, u'text/html')
timeout_mock.return_value = 1
max_workers_mock.return_value = 1
parsers_mock.return_value = [DelayedParser, HTMLParser]
try:
self.mpdoc.get_document_parser_for(http_resp)
except BaseFrameworkException:
msg = '[timeout] The parser took more than %s seconds'\
' to complete parsing of "%s", killed it!'
error = msg % (MultiProcessingDocumentParser.PARSER_TIMEOUT,
http_resp.get_url())
self.assertIn(call.debug(error), om_mock.mock_calls)
else:
self.assertTrue(False)
#
# We now want to make sure that after we kill the process the Pool
# creates a new process for handling our tasks
#
# https://github.com/andresriancho/w3af/issues/9713
#
html = '<html>foo-</html>'
http_resp = _build_http_response(html, u'text/html')
doc_parser = self.mpdoc.get_document_parser_for(http_resp)
self.assertIsInstance(doc_parser._parser, HTMLParser)
def test_parser_timeout(self):
"""
Test to verify fix for https://github.com/andresriancho/w3af/issues/6723
"w3af running long time more than 24h"
"""
mmpdp = 'w3af.core.data.parsers.mp_document_parser.%s'
kmpdp = mmpdp % 'MultiProcessingDocumentParser.%s'
modp = 'w3af.core.data.parsers.document_parser.%s'
with patch(mmpdp % 'om.out') as om_mock,\
patch(kmpdp % 'PARSER_TIMEOUT', new_callable=PropertyMock) as timeout_mock,\
patch(kmpdp % 'MAX_WORKERS', new_callable=PropertyMock) as max_workers_mock,\
patch(modp % 'DocumentParser.PARSERS', new_callable=PropertyMock) as parsers_mock:
#
# Test the timeout
#
html = '<html>DelayedParser!</html>'
http_resp = _build_http_response(html, u'text/html')
timeout_mock.return_value = 1
max_workers_mock.return_value = 1
parsers_mock.return_value = [DelayedParser, HTMLParser]
try:
self.mpdoc.get_document_parser_for(http_resp)
except TimeoutError, toe:
self._is_timeout_exception_message(toe, om_mock, http_resp)
else:
def test_parser_memory_usage_exceeded(self):
"""
This makes sure that we stop parsing a document that exceeds our memory
usage limits.
"""
mmpdp = 'w3af.core.data.parsers.mp_document_parser.%s'
kmpdp = mmpdp % 'MultiProcessingDocumentParser.%s'
modp = 'w3af.core.data.parsers.document_parser.%s'
with patch(mmpdp % 'om.out') as om_mock,\
patch(kmpdp % 'MEMORY_LIMIT', new_callable=PropertyMock) as memory_mock,\
patch(kmpdp % 'MAX_WORKERS', new_callable=PropertyMock) as max_workers_mock,\
patch(modp % 'DocumentParser.PARSERS', new_callable=PropertyMock) as parsers_mock:
#
# Test the memory usage
#
html = '<html>UseMemoryParser!</html>'
http_resp = _build_http_response(html, u'text/html')
memory_mock.return_value = 150000
max_workers_mock.return_value = 1
parsers_mock.return_value = [UseMemoryParser, HTMLParser]
try:
self.mpdoc.get_document_parser_for(http_resp)
except MemoryError, me:
self.assertIn('OOM issues', str(me))
else:
def test_parser_timeout(self):
"""
Test to verify fix for https://github.com/andresriancho/w3af/issues/6723
"w3af running long time more than 24h"
"""
modc = 'w3af.core.data.parsers.parser_cache.%s'
modp = 'w3af.core.data.parsers.document_parser.%s'
with patch(modc % 'om.out') as om_mock,\
patch(modc % 'ParserCache.PARSER_TIMEOUT', new_callable=PropertyMock) as timeout_mock,\
patch(modp % 'DocumentParser.PARSERS', new_callable=PropertyMock) as parsers_mock:
timeout_mock.return_value = 1
parsers_mock.return_value = [DelayedParser]
html = '<html>foo!</html>'
http_resp = _build_http_response(html, u'text/html')
try:
self.dpc.get_document_parser_for(http_resp)
except BaseFrameworkException:
msg = '[timeout] The parser took more than %s seconds'\
' to complete parsing of "%s", killed it!'
error = msg % (ParserCache.PARSER_TIMEOUT,
http_resp.get_url())
self.assertIn(call.debug(error), om_mock.mock_calls)
else:
self.assertTrue(False)
def test_parser_timeout(self):
"""
Test to verify fix for https://github.com/andresriancho/w3af/issues/6723
"w3af running long time more than 24h"
"""
mmpdp = 'w3af.core.data.parsers.mp_document_parser.%s'
kmpdp = mmpdp % 'MultiProcessingDocumentParser.%s'
modp = 'w3af.core.data.parsers.document_parser.%s'
with patch(mmpdp % 'om.out') as om_mock,\
patch(kmpdp % 'PARSER_TIMEOUT', new_callable=PropertyMock) as timeout_mock,\
patch(kmpdp % 'MAX_WORKERS', new_callable=PropertyMock) as max_workers_mock,\
patch(modp % 'DocumentParser.PARSERS', new_callable=PropertyMock) as parsers_mock:
#
# Test the timeout
#
html = '<html>DelayedParser!</html>'
http_resp = _build_http_response(html, u'text/html')
timeout_mock.return_value = 1
max_workers_mock.return_value = 1
parsers_mock.return_value = [DelayedParser, HTMLParser]
try:
self.mpdoc.get_document_parser_for(http_resp)
except TimeoutError, toe:
self._is_timeout_exception_message(toe, om_mock, http_resp)
else:
def test_parser_memory_usage_exceeded(self):
"""
This makes sure that we stop parsing a document that exceeds our memory
usage limits.
"""
mmpdp = 'w3af.core.data.parsers.mp_document_parser.%s'
kmpdp = mmpdp % 'MultiProcessingDocumentParser.%s'
modp = 'w3af.core.data.parsers.document_parser.%s'
with patch(mmpdp % 'om.out') as om_mock,\
patch(kmpdp % 'MEMORY_LIMIT', new_callable=PropertyMock) as memory_mock,\
patch(kmpdp % 'MAX_WORKERS', new_callable=PropertyMock) as max_workers_mock,\
patch(modp % 'DocumentParser.PARSERS', new_callable=PropertyMock) as parsers_mock:
#
# Test the memory usage
#
html = '<html>UseMemoryParser!</html>'
http_resp = _build_http_response(html, u'text/html')
memory_mock.return_value = 150000
max_workers_mock.return_value = 1
parsers_mock.return_value = [UseMemoryParser, HTMLParser]
try:
self.mpdoc.get_document_parser_for(http_resp)
except MemoryError, me:
self.assertIn('OOM issues', str(me))
else:
def test_cache_blacklist_after_timeout(self):
#
# If the cache tries to parse an HTTP response, that process fails, then we blacklist
# the HTTP response so it never gets parsed again.
#
mmpdp = 'w3af.core.data.parsers.mp_document_parser.%s'
kmpdp = mmpdp % 'MultiProcessingDocumentParser.%s'
modp = 'w3af.core.data.parsers.document_parser.%s'
with patch(kmpdp % 'PARSER_TIMEOUT', new_callable=PropertyMock) as timeout_mock, \
patch(kmpdp % 'MAX_WORKERS', new_callable=PropertyMock) as max_workers_mock, \
patch(modp % 'DocumentParser.PARSERS', new_callable=PropertyMock) as parsers_mock:
#
# Trigger the timeout
#
html = '<html>DelayedParser!</html>'
http_resp = _build_http_response(html, u'text/html')
timeout_mock.return_value = 1
max_workers_mock.return_value = 1
parsers_mock.return_value = [DelayedParser, HTMLParser]
try:
self.dpc.get_document_parser_for(http_resp)
except BaseFrameworkException, bfe:
self._is_timeout_exception_message(bfe, http_resp)
else:
def test_cache_blacklist_after_timeout(self):
#
# If the cache tries to parse an HTTP response, that process fails, then we blacklist
# the HTTP response so it never gets parsed again.
#
mmpdp = 'w3af.core.data.parsers.mp_document_parser.%s'
kmpdp = mmpdp % 'MultiProcessingDocumentParser.%s'
modp = 'w3af.core.data.parsers.document_parser.%s'
with patch(kmpdp % 'PARSER_TIMEOUT', new_callable=PropertyMock) as timeout_mock, \
patch(kmpdp % 'MAX_WORKERS', new_callable=PropertyMock) as max_workers_mock, \
patch(modp % 'DocumentParser.PARSERS', new_callable=PropertyMock) as parsers_mock:
#
# Trigger the timeout
#
html = '<html>DelayedParser!</html>'
http_resp = _build_http_response(html, u'text/html')
timeout_mock.return_value = 1
max_workers_mock.return_value = 1
parsers_mock.return_value = [DelayedParser, HTMLParser]
try:
self.dpc.get_document_parser_for(http_resp)
except BaseFrameworkException, bfe:
self._is_timeout_exception_message(bfe, http_resp)
else:
def test_parser_with_large_attr_killed_when_sending_to_queue(self):
"""
https://docs.python.org/2/library/multiprocessing.html
Warning If a process is killed using Process.terminate()
or os.kill() while it is trying to use a Queue, then the
data in the queue is likely to become corrupted. This may
cause any other process to get an exception when it tries
to use the queue later on.
Try to kill the process while it is sending data to the queue
"""
raise SkipTest('This test breaks the build because it uses A LOT'
' of memory, for more information take a look at'
' https://circleci.com/gh/andresriancho/w3af/2819 .'
' Note that there is no memory leak here, just a'
' test which is designed to use a lot of memory'
' to force a specific state.')
mmpdp = 'w3af.core.data.parsers.mp_document_parser.%s'
kmpdp = mmpdp % 'MultiProcessingDocumentParser.%s'
modp = 'w3af.core.data.parsers.document_parser.%s'
with patch(mmpdp % 'om.out') as om_mock,\
patch(kmpdp % 'PARSER_TIMEOUT', new_callable=PropertyMock) as timeout_mock,\
patch(kmpdp % 'MAX_WORKERS', new_callable=PropertyMock) as max_workers_mock,\
patch(modp % 'DocumentParser.PARSERS', new_callable=PropertyMock) as parsers_mock:
# Prepare the HTTP responses
html_trigger_delay = '<html>HugeClassAttrValueParser!</html>%s'
html_ok = '<html>foo-</html>%s'
# Mocks
timeout_mock.return_value = 1
max_workers_mock.return_value = 5
parsers_mock.return_value = [HugeClassAttrValueParser, HTMLParser]
ITERATIONS = 10
#
# Lets timeout many sequentially
#
for i in xrange(ITERATIONS):
http_resp = _build_http_response(html_trigger_delay % i, u'text/html')
try:
self.mpdoc.get_document_parser_for(http_resp)
except TimeoutError, toe:
self._is_timeout_exception_message(toe, om_mock, http_resp)
else:
self.assertTrue(False)
#
# Lets timeout randomly
#
for i in xrange(ITERATIONS):
html = random.choice([html_trigger_delay, html_ok])
http_resp = _build_http_response(html % i, u'text/html')
try:
parser = self.mpdoc.get_document_parser_for(http_resp)
except TimeoutError, toe:
self._is_timeout_exception_message(toe, om_mock, http_resp)
else:
self.assertIsInstance(parser._parser, HTMLParser)
try:
self.mpdoc.get_document_parser_for(http_resp)
except TimeoutError, toe:
self._is_timeout_exception_message(toe, om_mock, http_resp)
else:
self.assertTrue(False)
#
# We now want to make sure that after we kill the process the Pool
# creates a new process for handling our tasks
#
# https://github.com/andresriancho/w3af/issues/9713
#
html = '<html>foo-</html>'
http_resp = _build_http_response(html, u'text/html')
doc_parser = self.mpdoc.get_document_parser_for(http_resp)
self.assertIsInstance(doc_parser._parser, HTMLParser)
def test_many_parsers_timing_out(self):
"""
Received more reports of parsers timing out, and after that
w3af showing always "The parser took more than X seconds to complete
parsing of" for all calls to the parser.
Want to test how well the the parser recovers from many timeouts.
"""
mmpdp = 'w3af.core.data.parsers.mp_document_parser.%s'
kmpdp = mmpdp % 'MultiProcessingDocumentParser.%s'
modp = 'w3af.core.data.parsers.document_parser.%s'
def test_parser_with_large_attr_killed_when_sending_to_queue(self):
"""
https://docs.python.org/2/library/multiprocessing.html
Warning If a process is killed using Process.terminate()
or os.kill() while it is trying to use a Queue, then the
data in the queue is likely to become corrupted. This may
cause any other process to get an exception when it tries
to use the queue later on.
Try to kill the process while it is sending data to the queue
"""
raise SkipTest('This test breaks the build because it uses A LOT'
' of memory, for more information take a look at'
' https://circleci.com/gh/andresriancho/w3af/2819 .'
' Note that there is no memory leak here, just a'
' test which is designed to use a lot of memory'
' to force a specific state.')
mmpdp = 'w3af.core.data.parsers.mp_document_parser.%s'
kmpdp = mmpdp % 'MultiProcessingDocumentParser.%s'
modp = 'w3af.core.data.parsers.document_parser.%s'
with patch(mmpdp % 'om.out') as om_mock,\
patch(kmpdp % 'PARSER_TIMEOUT', new_callable=PropertyMock) as timeout_mock,\
patch(kmpdp % 'MAX_WORKERS', new_callable=PropertyMock) as max_workers_mock,\
patch(modp % 'DocumentParser.PARSERS', new_callable=PropertyMock) as parsers_mock:
# Prepare the HTTP responses
html_trigger_delay = '<html>HugeClassAttrValueParser!</html>%s'
html_ok = '<html>foo-</html>%s'
# Mocks
timeout_mock.return_value = 1
max_workers_mock.return_value = 5
parsers_mock.return_value = [HugeClassAttrValueParser, HTMLParser]
ITERATIONS = 10
#
# Lets timeout many sequentially
#
for i in xrange(ITERATIONS):
http_resp = _build_http_response(html_trigger_delay % i, u'text/html')
try:
self.mpdoc.get_document_parser_for(http_resp)
except TimeoutError, toe:
self._is_timeout_exception_message(toe, om_mock, http_resp)
else:
self.assertTrue(False)
#
# Lets timeout randomly
#
for i in xrange(ITERATIONS):
html = random.choice([html_trigger_delay, html_ok])
http_resp = _build_http_response(html % i, u'text/html')
try:
parser = self.mpdoc.get_document_parser_for(http_resp)
except TimeoutError, toe:
self._is_timeout_exception_message(toe, om_mock, http_resp)
else:
self.assertIsInstance(parser._parser, HTMLParser)
try:
self.mpdoc.get_document_parser_for(http_resp)
except TimeoutError, toe:
self._is_timeout_exception_message(toe, om_mock, http_resp)
else:
self.assertTrue(False)
#
# We now want to make sure that after we kill the process the Pool
# creates a new process for handling our tasks
#
# https://github.com/andresriancho/w3af/issues/9713
#
html = '<html>foo-</html>'
http_resp = _build_http_response(html, u'text/html')
doc_parser = self.mpdoc.get_document_parser_for(http_resp)
self.assertIsInstance(doc_parser._parser, HTMLParser)
def test_many_parsers_timing_out(self):
"""
Received more reports of parsers timing out, and after that
w3af showing always "The parser took more than X seconds to complete
parsing of" for all calls to the parser.
Want to test how well the the parser recovers from many timeouts.
"""
mmpdp = 'w3af.core.data.parsers.mp_document_parser.%s'
kmpdp = mmpdp % 'MultiProcessingDocumentParser.%s'
modp = 'w3af.core.data.parsers.document_parser.%s'
def test_parser_with_large_attr_killed_when_sending_to_queue(self):
"""
https://docs.python.org/2/library/multiprocessing.html
Warning If a process is killed using Process.terminate()
or os.kill() while it is trying to use a Queue, then the
data in the queue is likely to become corrupted. This may
cause any other process to get an exception when it tries
to use the queue later on.
Try to kill the process while it is sending data to the queue
"""
raise SkipTest('https://github.com/andresriancho/w3af/issues/9713')
mmpdp = 'w3af.core.data.parsers.mp_document_parser.%s'
kmpdp = mmpdp % 'MultiProcessingDocumentParser.%s'
modp = 'w3af.core.data.parsers.document_parser.%s'
with patch(mmpdp % 'om.out') as om_mock,\
patch(kmpdp % 'PARSER_TIMEOUT', new_callable=PropertyMock) as timeout_mock,\
patch(kmpdp % 'MAX_WORKERS', new_callable=PropertyMock) as max_workers_mock,\
patch(modp % 'DocumentParser.PARSERS', new_callable=PropertyMock) as parsers_mock:
# Prepare the HTTP responses
html_trigger_delay = '<html>HugeClassAttrValueParser!</html>%s'
html_ok = '<html>foo-</html>%s'
# Mocks
timeout_mock.return_value = 1
max_workers_mock.return_value = 5
parsers_mock.return_value = [HugeClassAttrValueParser, HTMLParser]
ITERATIONS = 25
#
# Lets timeout many sequentially
#
for i in xrange(ITERATIONS):
http_resp = _build_http_response(html_trigger_delay % i, u'text/html')
try:
self.mpdoc.get_document_parser_for(http_resp)
except BaseFrameworkException:
self._is_timeout_exception_message(om_mock, http_resp)
else:
self.assertTrue(False)
#
# Lets timeout randomly
#
for i in xrange(ITERATIONS):
html = random.choice([html_trigger_delay, html_ok])
http_resp = _build_http_response(html % i, u'text/html')
try:
parser = self.mpdoc.get_document_parser_for(http_resp)
except BaseFrameworkException:
self._is_timeout_exception_message(om_mock, http_resp)
else:
self.assertIsInstance(parser._parser, HTMLParser)
#
# Lets parse things we know should work
#
for i in xrange(ITERATIONS):
http_resp = _build_http_response(html_ok % i, u'text/html')
parser = self.mpdoc.get_document_parser_for(http_resp)
self.assertIsInstance(parser._parser, HTMLParser)
