def process_get_tags_by_filter(filename, tags, yield_text, debug): """ Simple wrapper to get the current process id and store it in a shared object so we can kill the process if needed. """ http_resp = load_http_response_from_temp_file(filename) document_parser = DocumentParser(http_resp) parser = document_parser.get_parser() # Not all parsers have tags if not hasattr(parser, 'get_tags_by_filter'): return write_tags_to_temp_file([]) filtered_tags = [] for tag in parser.get_tags_by_filter(tags, yield_text=yield_text): filtered_tags.append(tag) msg = ('Returned %s Tag instances at get_tags_by_filter() for URL %s' ' and tags filter %r') args = (len(filtered_tags), http_resp.get_uri(), tags) om.out.debug(msg % args) result_filename = write_tags_to_temp_file(filtered_tags) return result_filename
def process_get_tags_by_filter(http_resp, tags, yield_text, debug): """ Simple wrapper to get the current process id and store it in a shared object so we can kill the process if needed. """ document_parser = DocumentParser(http_resp) # Not all parsers have tags if not hasattr(document_parser, 'get_tags_by_filter'): return [] filtered_tags = [] for tag in document_parser.get_tags_by_filter(tags, yield_text=yield_text): filtered_tags.append(tag) return filtered_tags
def can_parse(self, http_response): """ Check if we can parse an HTTP response :param http_response: The HTTP response to verify :return: True if we can parse this HTTP response """ cached_can_parse = self._can_parse_cache.get(http_response.get_id(), default=None) if cached_can_parse is not None: return cached_can_parse # # We need to verify if we can parse this HTTP response # try: can_parse = DocumentParser.can_parse(http_response) except: # We catch all the exceptions here and just return False because # the real parsing procedure will (most likely) fail to parse # this response too. can_parse = False self._can_parse_cache[can_parse] = can_parse return can_parse
def process_get_tags_by_filter(http_resp, tags, yield_text, processes, hash_string, debug): """ Simple wrapper to get the current process id and store it in a shared object so we can kill the process if needed. """ # Save this for tracking pid = multiprocessing.current_process().pid processes[hash_string] = pid document_parser = DocumentParser(http_resp, False) # Not all parsers have tags if not hasattr(document_parser, 'get_tags_by_filter'): return [] filtered_tags = [] for tag in document_parser.get_tags_by_filter(tags, yield_text=yield_text): filtered_tags.append(tag) return filtered_tags
def process_get_tags_by_filter(http_resp, tags, yield_text, debug): """ Simple wrapper to get the current process id and store it in a shared object so we can kill the process if needed. """ document_parser = DocumentParser(http_resp) parser = document_parser.get_parser() # Not all parsers have tags if not hasattr(parser, 'get_tags_by_filter'): return [] filtered_tags = [] for tag in parser.get_tags_by_filter(tags, yield_text=yield_text): filtered_tags.append(tag) msg = ('Returned %s Tag instances at get_tags_by_filter() for URL %s' ' and tags filter %r') args = (len(filtered_tags), http_resp.get_uri(), tags) om.out.debug(msg % args) return filtered_tags
def process_document_parser(http_resp, debug): """ Simple wrapper to get the current process id and store it in a shared object so we can kill the process if needed. """ pid = multiprocessing.current_process().pid if debug: msg = '[mp_document_parser] PID %s is starting to parse %s' args = (pid, http_resp.get_url()) om.out.debug(msg % args) try: # Parse document_parser = DocumentParser(http_resp) except Exception, e: if debug: msg = ('[mp_document_parser] PID %s finished parsing %s with' ' exception: "%s"') args = (pid, http_resp.get_url(), e) om.out.debug(msg % args) raise
def _test_parse_http_response(self, http_response, *args): """ Left here for testing! """ return DocumentParser(http_response)