def get_tags_by_filter(self, http_response, tags, yield_text=False):
"""
Return Tag instances for the tags which match the `tags` filter,
parsing and all lxml stuff is done in another process and the Tag
instances are sent to the main process (the one calling this method)
through a pipe
Some things to note:
* Not all responses can be parsed, so I need to call DocumentParser
and handle exceptions
* The parser selected by DocumentParser might not have tags, and
it might not have get_tags_by_filter. In this case just return an
empty list
* Just like get_document_parser_for we have a timeout in place,
when we hit the timeout just return an empty list, this is not
the best thing to do, but makes the plugin code easier to write
(plugins would ignore this anyways)
:param tags: The filter
:param yield_text: Should we yield the tag text?
:return: A list of Tag instances as defined in sgml.py
:see: SGMLParser.get_tags_by_filter
"""
# Start the worker processes if needed
self.start_workers()
hash_string = get_request_unique_id(http_response, prepend='tags')
apply_args = (process_get_tags_by_filter,
http_response,
tags,
yield_text,
self._processes,
hash_string,
self.DEBUG)
# Push the task to the workers
result = self._pool.apply_async(apply_with_return_error, (apply_args,))
try:
filtered_tags = result.get(timeout=self.PARSER_TIMEOUT)
except multiprocessing.TimeoutError:
self._kill_parser_process(hash_string, http_response)
# We hit a timeout, return an empty list
return []
else:
# There was an exception in the parser, maybe the HTML was really
# broken, or it wasn't an HTML at all.
if isinstance(filtered_tags, Error):
return []
finally:
# Just remove it so it doesn't use memory
self._processes.pop(hash_string, None)
return filtered_tags
def get_document_parser_for(self, http_response, cache=True):
"""
Get a document parser for http_response using the cache if required
:param http_response: The http response instance
:return: An instance of DocumentParser
"""
hash_string = get_request_unique_id(http_response)
parser_finished = self._parser_finished_events.get(hash_string, None)
if parser_finished is not None:
# There is one subprocess already processing this http response
# body, the best thing to do here is to make this thread wait
# until that process has finished
try:
parser_finished.wait(timeout=mp_doc_parser.PARSER_TIMEOUT)
except:
# Act just like when there is no parser
msg = 'There is no parser for "%s". Waited more than %s sec.'
args = (http_response.get_url(), mp_doc_parser.PARSER_TIMEOUT)
raise BaseFrameworkException(msg % args)
# metric increase
self.inc_query_count()
parser = self._cache.get(hash_string, None)
if parser is not None:
self._handle_cache_hit(hash_string)
return parser
else:
# Not in cache, have to work.
self._handle_cache_miss(hash_string)
# Create a new instance of DocumentParser, add it to the cache
event = threading.Event()
self._parser_finished_events[hash_string] = event
try:
parser = mp_doc_parser.get_document_parser_for(http_response)
except:
# Act just like when there is no parser
msg = 'There is no parser for "%s".' % http_response.get_url()
raise BaseFrameworkException(msg)
else:
save_to_cache = self.should_cache(http_response) and cache
if save_to_cache:
self._cache[hash_string] = parser
else:
self._handle_no_cache(hash_string)
finally:
event.set()
self._parser_finished_events.pop(hash_string, None)
return parser
def get_document_parser_for(self, http_response):
"""
Get a document parser for http_response
This parses the http_response in a pool worker. This method has two
features:
* We can kill the worker if the parser is taking too long
* We can have different parsers
:param http_response: The http response instance
:return: An instance of DocumentParser
"""
# Start the worker processes if needed
self.start_workers()
hash_string = get_request_unique_id(http_response, prepend='parser')
apply_args = (process_document_parser,
http_response,
self._processes,
hash_string,
self.DEBUG)
# Push the task to the workers
result = self._pool.apply_async(apply_with_return_error, (apply_args,))
try:
parser_output = result.get(timeout=self.PARSER_TIMEOUT)
except multiprocessing.TimeoutError:
self._kill_parser_process(hash_string, http_response)
# Act just like when there is no parser
msg = 'There is no parser for "%s".' % http_response.get_url()
raise BaseFrameworkException(msg)
else:
if isinstance(parser_output, Error):
parser_output.reraise()
finally:
# Just remove it so it doesn't use memory
self._processes.pop(hash_string, None)
return parser_output
def get_document_parser_for(self, http_response, cache=True):
"""
Get a document parser for http_response using the cache if required
:param http_response: The http response instance
:param cache: If the DocumentParser is in the cache, return that one.
:return: An instance of DocumentParser
"""
#
# Before doing anything too complex like caching, sending the HTTP
# response to a different process for parsing, checking events, etc.
# check if we can parse this HTTP response.
#
# This is a performance improvement that works *only if* the
# DocumentParser.can_parse call is *fast*, which means that the
# `can_parse` implementations of each parser needs to be fast
#
# It doesn't matter if we say "yes" here and then parsing exceptions
# appear later, that should be a 1 / 10000 calls and we would still
# be gaining a lot of performance
#
if not self.can_parse(http_response):
msg = 'There is no parser for "%s".'
raise BaseFrameworkException(msg % http_response.get_url())
hash_string = get_request_unique_id(http_response)
if hash_string in self._parser_blacklist:
msg = 'Exceeded timeout while parsing "%s" in the past. Not trying again.'
raise BaseFrameworkException(msg % http_response.get_url())
#
# We know that we can parse this document, lets work!
#
parser_finished = self._parser_finished_events.get(hash_string, None)
if parser_finished is not None:
# There is one subprocess already processing this http response
# body, the best thing to do here is to make this thread wait
# until that process has finished
try:
parser_finished.wait(timeout=mp_doc_parser.PARSER_TIMEOUT)
except:
# Act just like when there is no parser
msg = 'There is no parser for "%s". Waited more than %s sec.'
args = (http_response.get_url(), mp_doc_parser.PARSER_TIMEOUT)
raise BaseFrameworkException(msg % args)
# metric increase
self.inc_query_count()
parser = self._cache.get(hash_string, None)
if parser is not None:
self._handle_cache_hit(hash_string)
return parser
else:
# Not in cache, have to work.
self._handle_cache_miss(hash_string)
# Create a new instance of DocumentParser, add it to the cache
event = threading.Event()
self._parser_finished_events[hash_string] = event
try:
parser = mp_doc_parser.get_document_parser_for(http_response)
except TimeoutError:
# We failed to get a parser for this HTTP response, we better
# ban this HTTP response so we don't waste more CPU cycles trying
# to parse it over and over.
self.add_to_blacklist(hash_string)
# Act just like when there is no parser
msg = 'Reached timeout parsing "%s".' % http_response.get_url()
raise BaseFrameworkException(msg)
except MemoryError:
# We failed to get a parser for this HTTP response, we better
# ban this HTTP response so we don't waste more CPU cycles or
# memory trying to parse it over and over.
self.add_to_blacklist(hash_string)
# Act just like when there is no parser
msg = 'Reached memory usage limit parsing "%s".' % http_response.get_url(
)
raise BaseFrameworkException(msg)
except:
# Act just like when there is no parser
msg = 'There is no parser for "%s".' % http_response.get_url()
raise BaseFrameworkException(msg)
else:
save_to_cache = self.should_cache(http_response) and cache
if save_to_cache:
self._cache[hash_string] = parser
else:
self._handle_no_cache(hash_string)
finally:
event.set()
self._parser_finished_events.pop(hash_string, None)
return parser
http_resp = _build_http_response(html, u'text/html')
timeout_mock.return_value = 1
max_workers_mock.return_value = 1
parsers_mock.return_value = [DelayedParser, HTMLParser]
try:
self.dpc.get_document_parser_for(http_resp)
except BaseFrameworkException, bfe:
self._is_timeout_exception_message(bfe, http_resp)
else:
self.assertTrue(False)
#
# Make sure it is in the blacklist
#
hash_string = get_request_unique_id(http_resp)
self.assertIn(hash_string, self.dpc._parser_blacklist)
#
# Make sure the blacklist is used
#
try:
self.dpc.get_document_parser_for(http_resp)
except BaseFrameworkException, bfe:
self.assertIn('Exceeded timeout while parsing', str(bfe))
def _is_timeout_exception_message(self, toe, http_resp):
msg = 'Reached timeout parsing "http://w3af.com/".'
self.assertEquals(str(toe), msg)
def get_document_parser_for(self, http_response, cache=True):
"""
Get a document parser for http_response using the cache if required
:param http_response: The http response instance
:return: An instance of DocumentParser
"""
if http_response.is_image():
# Act just like when there is no parser
msg = 'There is no parser for image("%s")' % (http_response.get_url())
raise BaseFrameworkException(msg)
hash_string = get_request_unique_id(http_response)
parser_finished = self._parser_finished_events.get(hash_string, None)
if parser_finished is not None:
# There is one subprocess already processing this http response
# body, the best thing to do here is to make this thread wait
# until that process has finished
try:
parser_finished.wait(timeout=mp_doc_parser.PARSER_TIMEOUT)
except:
# Act just like when there is no parser
msg = 'There is no parser for "%s". Waited more than %s sec.'
args = (http_response.get_url(), mp_doc_parser.PARSER_TIMEOUT)
raise BaseFrameworkException(msg % args)
# metric increase
self.inc_query_count()
parser = self._cache.get(hash_string, None)
if parser:
self._handle_cache_hit(hash_string)
# om.out.debug('[parser cache][memory] Hit for %s(%s)' % (http_response.get_uri().url_string, hash_string))
else:
# om.out.debug('[parser cache][memory] Miss for %s(%s)' % (http_response.get_uri().url_string, hash_string))
# Create a new instance of DocumentParser, add it to the cache
event = threading.Event()
self._parser_finished_events[hash_string] = event
# Not in cache, have to work.
self._handle_cache_miss(hash_string)
try:
if hash_string in self.disk_cache['key_set']:
parser = self.disk_cache['disk_cache'][hash_string]
# om.out.debug('[parser cache][disk] Hit for %s(%s)' % (http_response.get_uri().url_string, hash_string))
else:
# om.out.debug('[parser cache][disk] Miss for %s(%s)' % (http_response.get_uri().url_string, hash_string))
try:
parser = mp_doc_parser.get_document_parser_for(http_response)
except Exception as e:
# Act just like when there is no parser
msg = 'There is no parser for "%s".e=%s' % (http_response.get_url(), e)
raise BaseFrameworkException(msg)
else:
self.disk_cache['disk_cache'][hash_string] = parser
self.disk_cache['key_set'].add(hash_string)
save_to_cache = self.should_cache(http_response) and cache
if save_to_cache:
self._cache[hash_string] = parser
else:
self._handle_no_cache(hash_string)
finally:
self._parser_finished_events.pop(hash_string, None)
event.set()
return parser
def get_document_parser_for(self, http_response, cache=True):
"""
Get a document parser for http_response using the cache if required
:param http_response: The http response instance
:param cache: If the DocumentParser is in the cache, return that one.
:return: An instance of DocumentParser
"""
#
# Before doing anything too complex like caching, sending the HTTP
# response to a different process for parsing, checking events, etc.
# check if we can parse this HTTP response.
#
# This is a performance improvement that works *only if* the
# DocumentParser.can_parse call is *fast*, which means that the
# `can_parse` implementations of each parser needs to be fast
#
# It doesn't matter if we say "yes" here and then parsing exceptions
# appear later, that should be a 1 / 10000 calls and we would still
# be gaining a lot of performance
#
if not self.can_parse(http_response):
msg = 'There is no parser for "%s".' % http_response.get_url()
raise BaseFrameworkException(msg)
#
# We know that we can parse this document, lets work!
#
hash_string = get_request_unique_id(http_response)
parser_finished = self._parser_finished_events.get(hash_string, None)
if parser_finished is not None:
# There is one subprocess already processing this http response
# body, the best thing to do here is to make this thread wait
# until that process has finished
try:
parser_finished.wait(timeout=mp_doc_parser.PARSER_TIMEOUT)
except:
# Act just like when there is no parser
msg = 'There is no parser for "%s". Waited more than %s sec.'
args = (http_response.get_url(), mp_doc_parser.PARSER_TIMEOUT)
raise BaseFrameworkException(msg % args)
# metric increase
self.inc_query_count()
parser = self._cache.get(hash_string, None)
if parser is not None:
self._handle_cache_hit(hash_string)
return parser
else:
# Not in cache, have to work.
self._handle_cache_miss(hash_string)
# Create a new instance of DocumentParser, add it to the cache
event = threading.Event()
self._parser_finished_events[hash_string] = event
try:
parser = mp_doc_parser.get_document_parser_for(http_response)
except:
# Act just like when there is no parser
msg = 'There is no parser for "%s".' % http_response.get_url()
raise BaseFrameworkException(msg)
else:
save_to_cache = self.should_cache(http_response) and cache
if save_to_cache:
self._cache[hash_string] = parser
else:
self._handle_no_cache(hash_string)
finally:
event.set()
self._parser_finished_events.pop(hash_string, None)
return parser
