def get_document_parser_for(self, http_response, cache=True): """ Get a document parser for http_response using the cache if required :param http_response: The http response instance :return: An instance of DocumentParser """ hash_string = get_request_unique_id(http_response) parser_finished = self._parser_finished_events.get(hash_string, None) if parser_finished is not None: # There is one subprocess already processing this http response # body, the best thing to do here is to make this thread wait # until that process has finished try: parser_finished.wait(timeout=mp_doc_parser.PARSER_TIMEOUT) except: # Act just like when there is no parser msg = 'There is no parser for "%s". Waited more than %s sec.' args = (http_response.get_url(), mp_doc_parser.PARSER_TIMEOUT) raise BaseFrameworkException(msg % args) # metric increase self.inc_query_count() parser = self._cache.get(hash_string, None) if parser is not None: self._handle_cache_hit(hash_string) return parser else: # Not in cache, have to work. self._handle_cache_miss(hash_string) # Create a new instance of DocumentParser, add it to the cache event = threading.Event() self._parser_finished_events[hash_string] = event try: parser = mp_doc_parser.get_document_parser_for(http_response) except: # Act just like when there is no parser msg = 'There is no parser for "%s".' % http_response.get_url() raise BaseFrameworkException(msg) else: save_to_cache = self.should_cache(http_response) and cache if save_to_cache: self._cache[hash_string] = parser else: self._handle_no_cache(hash_string) finally: event.set() self._parser_finished_events.pop(hash_string, None) return parser
def get_document_parser_for(self, http_response, cache=True): """ Get a document parser for http_response using the cache if possible :param http_response: The http response instance :param cache: True if the document parser should be saved to the cache :return: An instance of DocumentParser """ # # Before doing anything too complex like caching, sending the HTTP # response to a different process for parsing, checking events, etc. # check if we can parse this HTTP response. # # This is a performance improvement that works *only if* the # DocumentParser.can_parse call is *fast*, which means that the # `can_parse` implementations of each parser needs to be fast # # It doesn't matter if we say "yes" here and then parsing exceptions # appear later, that should be a 1 / 10000 calls and we would still # be gaining a lot of performance # if not self.can_parse(http_response): msg = 'There is no parser for "%s".' raise BaseFrameworkException(msg % http_response.get_url()) hash_string = get_response_unique_id(http_response) if hash_string in self._parser_blacklist: msg = 'Exceeded timeout while parsing "%s" in the past. Not trying again.' raise BaseFrameworkException(msg % http_response.get_url()) # # We know that we can parse this document, lets work! # parser_finished = self._parser_finished_events.get(hash_string, None) if parser_finished is not None: # There is one subprocess already processing this http response # body, the best thing to do here is to make this thread wait # until that process has finished wait_result = parser_finished.wait(timeout=mp_doc_parser.PARSER_TIMEOUT) if not wait_result: # Act just like when there is no parser msg = 'There is no parser for "%s". Waited more than %s sec.' args = (http_response.get_url(), mp_doc_parser.PARSER_TIMEOUT) raise BaseFrameworkException(msg % args) # metric increase self.inc_query_count() parser = self._cache.get(hash_string, None) if parser is not None: self._handle_cache_hit(hash_string) return parser else: # Not in cache, have to work. self._handle_cache_miss(hash_string) # Create a new instance of DocumentParser, add it to the cache event = threading.Event() self._parser_finished_events[hash_string] = event try: parser = mp_doc_parser.get_document_parser_for(http_response) except TimeoutError: # We failed to get a parser for this HTTP response, we better # ban this HTTP response so we don't waste more CPU cycles trying # to parse it over and over. self.add_to_blacklist(hash_string) # Act just like when there is no parser msg = 'Reached timeout parsing "%s".' % http_response.get_url() raise BaseFrameworkException(msg) except MemoryError: # We failed to get a parser for this HTTP response, we better # ban this HTTP response so we don't waste more CPU cycles or # memory trying to parse it over and over. self.add_to_blacklist(hash_string) # Act just like when there is no parser msg = 'Reached memory usage limit parsing "%s".' % http_response.get_url() raise BaseFrameworkException(msg) except ScanMustStopException, e: msg = 'The document parser is in an invalid state! %s' raise ScanMustStopException(msg % e) except:
def get_document_parser_for(self, http_response, cache=True): """ Get a document parser for http_response using the cache if possible :param http_response: The http response instance :param cache: True if the document parser should be saved to the cache :return: An instance of DocumentParser """ # # Before doing anything too complex like caching, sending the HTTP # response to a different process for parsing, checking events, etc. # check if we can parse this HTTP response. # # This is a performance improvement that works *only if* the # DocumentParser.can_parse call is *fast*, which means that the # `can_parse` implementations of each parser needs to be fast # # It doesn't matter if we say "yes" here and then parsing exceptions # appear later, that should be a 1 / 10000 calls and we would still # be gaining a lot of performance # if not self.can_parse(http_response): msg = 'There is no parser for "%s".' raise BaseFrameworkException(msg % http_response.get_url()) hash_string = get_response_unique_id(http_response) if hash_string in self._parser_blacklist: msg = 'Exceeded timeout while parsing "%s" in the past. Not trying again.' raise BaseFrameworkException(msg % http_response.get_url()) # # We know that we can parse this document, lets work! # parser_finished = self._parser_finished_events.get(hash_string, None) if parser_finished is not None: # There is one subprocess already processing this http response # body, the best thing to do here is to make this thread wait # until that process has finished try: parser_finished.wait(timeout=mp_doc_parser.PARSER_TIMEOUT) except: # Act just like when there is no parser msg = 'There is no parser for "%s". Waited more than %s sec.' args = (http_response.get_url(), mp_doc_parser.PARSER_TIMEOUT) raise BaseFrameworkException(msg % args) # metric increase self.inc_query_count() parser = self._cache.get(hash_string, None) if parser is not None: self._handle_cache_hit(hash_string) return parser else: # Not in cache, have to work. self._handle_cache_miss(hash_string) # Create a new instance of DocumentParser, add it to the cache event = threading.Event() self._parser_finished_events[hash_string] = event try: parser = mp_doc_parser.get_document_parser_for(http_response) except TimeoutError: # We failed to get a parser for this HTTP response, we better # ban this HTTP response so we don't waste more CPU cycles trying # to parse it over and over. self.add_to_blacklist(hash_string) # Act just like when there is no parser msg = 'Reached timeout parsing "%s".' % http_response.get_url() raise BaseFrameworkException(msg) except MemoryError: # We failed to get a parser for this HTTP response, we better # ban this HTTP response so we don't waste more CPU cycles or # memory trying to parse it over and over. self.add_to_blacklist(hash_string) # Act just like when there is no parser msg = 'Reached memory usage limit parsing "%s".' % http_response.get_url() raise BaseFrameworkException(msg) except ScanMustStopException, e: msg = 'The document parser is in an invalid state! %s' raise ScanMustStopException(msg % e) except:
def get_document_parser_for(self, http_response, cache=True): """ Get a document parser for http_response using the cache if required :param http_response: The http response instance :return: An instance of DocumentParser """ if http_response.is_image(): # Act just like when there is no parser msg = 'There is no parser for image("%s")' % (http_response.get_url()) raise BaseFrameworkException(msg) hash_string = get_request_unique_id(http_response) parser_finished = self._parser_finished_events.get(hash_string, None) if parser_finished is not None: # There is one subprocess already processing this http response # body, the best thing to do here is to make this thread wait # until that process has finished try: parser_finished.wait(timeout=mp_doc_parser.PARSER_TIMEOUT) except: # Act just like when there is no parser msg = 'There is no parser for "%s". Waited more than %s sec.' args = (http_response.get_url(), mp_doc_parser.PARSER_TIMEOUT) raise BaseFrameworkException(msg % args) # metric increase self.inc_query_count() parser = self._cache.get(hash_string, None) if parser: self._handle_cache_hit(hash_string) # om.out.debug('[parser cache][memory] Hit for %s(%s)' % (http_response.get_uri().url_string, hash_string)) else: # om.out.debug('[parser cache][memory] Miss for %s(%s)' % (http_response.get_uri().url_string, hash_string)) # Create a new instance of DocumentParser, add it to the cache event = threading.Event() self._parser_finished_events[hash_string] = event # Not in cache, have to work. self._handle_cache_miss(hash_string) try: if hash_string in self.disk_cache['key_set']: parser = self.disk_cache['disk_cache'][hash_string] # om.out.debug('[parser cache][disk] Hit for %s(%s)' % (http_response.get_uri().url_string, hash_string)) else: # om.out.debug('[parser cache][disk] Miss for %s(%s)' % (http_response.get_uri().url_string, hash_string)) try: parser = mp_doc_parser.get_document_parser_for(http_response) except Exception as e: # Act just like when there is no parser msg = 'There is no parser for "%s".e=%s' % (http_response.get_url(), e) raise BaseFrameworkException(msg) else: self.disk_cache['disk_cache'][hash_string] = parser self.disk_cache['key_set'].add(hash_string) save_to_cache = self.should_cache(http_response) and cache if save_to_cache: self._cache[hash_string] = parser else: self._handle_no_cache(hash_string) finally: self._parser_finished_events.pop(hash_string, None) event.set() return parser
def get_document_parser_for(self, http_response, cache=True): """ Get a document parser for http_response using the cache if required :param http_response: The http response instance :param cache: If the DocumentParser is in the cache, return that one. :return: An instance of DocumentParser """ # # Before doing anything too complex like caching, sending the HTTP # response to a different process for parsing, checking events, etc. # check if we can parse this HTTP response. # # This is a performance improvement that works *only if* the # DocumentParser.can_parse call is *fast*, which means that the # `can_parse` implementations of each parser needs to be fast # # It doesn't matter if we say "yes" here and then parsing exceptions # appear later, that should be a 1 / 10000 calls and we would still # be gaining a lot of performance # if not self.can_parse(http_response): msg = 'There is no parser for "%s".' % http_response.get_url() raise BaseFrameworkException(msg) # # We know that we can parse this document, lets work! # hash_string = get_request_unique_id(http_response) parser_finished = self._parser_finished_events.get(hash_string, None) if parser_finished is not None: # There is one subprocess already processing this http response # body, the best thing to do here is to make this thread wait # until that process has finished try: parser_finished.wait(timeout=mp_doc_parser.PARSER_TIMEOUT) except: # Act just like when there is no parser msg = 'There is no parser for "%s". Waited more than %s sec.' args = (http_response.get_url(), mp_doc_parser.PARSER_TIMEOUT) raise BaseFrameworkException(msg % args) # metric increase self.inc_query_count() parser = self._cache.get(hash_string, None) if parser is not None: self._handle_cache_hit(hash_string) return parser else: # Not in cache, have to work. self._handle_cache_miss(hash_string) # Create a new instance of DocumentParser, add it to the cache event = threading.Event() self._parser_finished_events[hash_string] = event try: parser = mp_doc_parser.get_document_parser_for(http_response) except: # Act just like when there is no parser msg = 'There is no parser for "%s".' % http_response.get_url() raise BaseFrameworkException(msg) else: save_to_cache = self.should_cache(http_response) and cache if save_to_cache: self._cache[hash_string] = parser else: self._handle_no_cache(hash_string) finally: event.set() self._parser_finished_events.pop(hash_string, None) return parser
def get_document_parser_for(self, http_response, cache=True): """ Get a document parser for http_response using the cache if required :param http_response: The http response instance :param cache: If the DocumentParser is in the cache, return that one. :return: An instance of DocumentParser """ # # Before doing anything too complex like caching, sending the HTTP # response to a different process for parsing, checking events, etc. # check if we can parse this HTTP response. # # This is a performance improvement that works *only if* the # DocumentParser.can_parse call is *fast*, which means that the # `can_parse` implementations of each parser needs to be fast # # It doesn't matter if we say "yes" here and then parsing exceptions # appear later, that should be a 1 / 10000 calls and we would still # be gaining a lot of performance # if not self.can_parse(http_response): msg = 'There is no parser for "%s".' % http_response.get_url() raise BaseFrameworkException(msg) # # We know that we can parse this document, lets work! # hash_string = get_request_unique_id(http_response) parser_finished = self._parser_finished_events.get(hash_string, None) if parser_finished is not None: # There is one subprocess already processing this http response # body, the best thing to do here is to make this thread wait # until that process has finished try: parser_finished.wait(timeout=mp_doc_parser.PARSER_TIMEOUT) except: # Act just like when there is no parser msg = 'There is no parser for "%s". Waited more than %s sec.' args = (http_response.get_url(), mp_doc_parser.PARSER_TIMEOUT) raise BaseFrameworkException(msg % args) # metric increase self.inc_query_count() parser = self._cache.get(hash_string, None) if parser is not None: self._handle_cache_hit(hash_string) return parser else: # Not in cache, have to work. self._handle_cache_miss(hash_string) # Create a new instance of DocumentParser, add it to the cache event = threading.Event() self._parser_finished_events[hash_string] = event try: parser = mp_doc_parser.get_document_parser_for(http_response) except: # Act just like when there is no parser msg = 'There is no parser for "%s".' % http_response.get_url() raise BaseFrameworkException(msg) else: save_to_cache = self.should_cache(http_response) and cache if save_to_cache: self._cache[hash_string] = parser else: self._handle_no_cache(hash_string) finally: event.set() self._parser_finished_events.pop(hash_string, None) return parser