def extract_article_html_from_page_html(content: str,
config: Optional[CommonConfig] = None
) -> Dict[str, str]:
"""
Using full page HTML as a parameter, extract part of HTML that contains the news article.
:param content: Full page HTML.
:param config: Optional CommonConfig object, useful for testing.
:return: Dictionary with HTML that contains the news article content ("extracted_html" key) and extractor version
tag ("extractor_version" key).
"""
content = decode_object_from_bytes_if_needed(content)
if not config:
config = CommonConfig()
ua = UserAgent()
api_url = config.extractor_api_url()
# Wait up to a minute for extraction to finish
ua.set_timeout(EXTRACT_TIMEOUT)
# Wait for the extractor's HTTP port to become open as the service might be still starting up somewhere
api_uri = furl(api_url)
api_url_hostname = str(api_uri.host)
api_url_port = int(api_uri.port)
assert api_url_hostname, f"API URL hostname is not set for URL {api_url}"
assert api_url_port, f"API URL port is not set for URL {api_url}"
if not wait_for_tcp_port_to_open(
port=api_url_port,
hostname=api_url_hostname,
retries=EXTRACTOR_SERVICE_TIMEOUT,
):
# Instead of throwing an exception, just crash the whole application
# because there's no point in continuing on running it whatsoever:
#
# 1) If the extractor service didn't come up in a given time, it won't
# suddenly show up
# 2) If it's a test that's doing the extraction, it can't do its job
# and should fail one way or another; exit(1) is just one of the
# ways how it can fail
# 3) If it's some production code that needs something to get
# extracted, and if we were to throw an exception instead of doing
# exit(1), the caller might treat this exception as a failure to
# extract this one specific input HTML file, and so it might
# mis-extract a bunch of stories that way (making it hard for us to
# spot the problem and time-consuming to fix it later (e.g. there
# would be a need to manually re-extract a million of stories))
#
# A better solution instead of exit(1) might be to throw different
# kinds of exceptions and handle them appropriately in the caller, but
# with the Perl-Python codebase that's a bit hard to do.
fatal_error(
"Extractor service at {url} didn't come up in {timeout} seconds, exiting..."
.format(
url=api_url,
timeout=EXTRACTOR_SERVICE_TIMEOUT,
))
request_json = encode_json({'html': content})
http_request = Request(method='POST', url=api_url)
http_request.set_content_type('application/json; charset=utf-8')
http_request.set_content(request_json)
# Try extracting multiple times
#
# UserAgent's set_timing() would only retry on retryable HTTP status codes and doesn't retry on connection errors by
# default as such retries might have side effects, e.g. an API getting called multiple times. So, we retry
# extracting the content a couple of times manually.
http_response = None
extraction_succeeded = False
for retry in range(EXTRACT_RETRIES):
if retry > 0:
log.warning(f"Retrying #{retry + 1}...")
http_response = ua.request(http_request)
if http_response.is_success():
extraction_succeeded = True
break
else:
log.error(
f"Extraction attempt {retry + 1} failed: {http_response.decoded_content()}"
)
if not extraction_succeeded:
raise McExtractArticleFromPageException(
f"Extraction of {len(content)} characters; failed; last error: {http_response.decoded_content()}"
)
response_json = http_response.decoded_content()
response = decode_json(response_json)
assert 'extracted_html' in response, "Response is expected to have 'extracted_html' key."
assert 'extractor_version' in response, "Response is expected to have 'extractor_version' key."
return response
def __annotate_text(self, text: str) -> Union[dict, list]:
"""Takes text, fetches JSON annotation, decodes it into dictionary / list."""
text = decode_object_from_bytes_if_needed(text)
if text is None:
fatal_error("Text is None.")
if len(text) == 0:
# Annotators accept empty strings, but that might happen with some stories so we're just die()ing here
raise McTagsFromJSONAnnotationException("Text is empty.")
log.info(f"Annotating {len(text)} characters of text...")
# Trim the text because that's what the annotator will do, and if the text is empty, we want to fail early
# without making a request to the annotator at all
text = text.strip()
if self.__TEXT_LENGTH_LIMIT > 0:
text_length = len(text)
if text_length > self.__TEXT_LENGTH_LIMIT:
log.warning(
f"Text length ({text_length}) has exceeded the request text length limit"
f"({self.__TEXT_LENGTH_LIMIT}) so I will truncate it.")
text = text[:self.__TEXT_LENGTH_LIMIT]
# Make a request
ua = UserAgent()
ua.set_timing([1, 2, 4, 8])
ua.set_timeout(self.__HTTP_TIMEOUT)
ua.set_max_size(None)
request = None
try:
request = self._request_for_text(text=text)
if request is None:
raise McTagsFromJSONAnnotationException(
"Returned request is None.")
except Exception as ex:
# Assume that this is some sort of a programming error too
fatal_error(
f"Unable to create annotator request for text '{text}': {ex}")
# Wait for the service's HTTP port to become open as the service might be
# still starting up somewhere
uri = furl(request.url())
hostname = str(uri.host)
port = int(uri.port)
assert hostname, f"URL hostname is not set for URL {request.url()}"
assert port, f"API URL port is not set for URL {request.url()}"
if not wait_for_tcp_port_to_open(
port=port,
hostname=hostname,
retries=self.__ANNOTATOR_SERVICE_TIMEOUT,
):
# Instead of throwing an exception, just crash the whole application
# because there's no point in continuing on running it whatsoever.
fatal_error(
f"Annotator service at {request.url()} didn't come up in {self.__ANNOTATOR_SERVICE_TIMEOUT} seconds, "
f"exiting...")
log.debug(f"Sending request to {request.url()}...")
# Try requesting a few times because sometimes it throws a connection error, e.g.:
#
# WARNING mediawords.util.web.user_agent: Client-side error while processing request <PreparedRequest [POST]>:
# ('Connection aborted.', ConnectionResetError(104, 'Connection reset by peer'))
# WARNING mediawords.tag_from_annotation.fetch_and_tag: Request failed: ('Connection aborted.', ConnectionResetError(104,
# 'Connection reset by peer'))
# ERROR mediawords.util.process: User agent error: 400 Client-side error: ('Connection aborted.',
# ConnectionResetError(104, 'Connection reset by peer'))
response = None
retries = 60
sleep_between_retries = 1
for retry in range(1, retries + 1):
if retry > 1:
log.warning(f"Retrying ({retry} / {retries})...")
response = ua.request(request)
if response.is_success():
break
else:
if response.error_is_client_side():
log.error(
f"Request failed on the client side: {response.decoded_content()}"
)
time.sleep(sleep_between_retries)
else:
break
log.debug("Response received.")
# Force UTF-8 encoding on the response because the server might not always
# return correct "Content-Type"
results_string = response.decoded_utf8_content()
if not response.is_success():
# Error; determine whether we should be blamed for making a malformed
# request, or is it an extraction error
log.warning(f"Request failed: {response.decoded_content()}")
if response.code() == HTTPStatus.REQUEST_TIMEOUT.value:
# Raise on request timeouts without retrying anything because those usually mean that we posted
# something funky to the annotator service and it got stuck
raise McTagsFromJSONAnnotationException(
f"The request timed out, giving up; text length: {len(text)}; text: {text}"
)
if response.error_is_client_side():
# Error was generated by the user agent client code; likely didn't reach server at all (timeout,
# unresponsive host, etc.)
fatal_error(
f"User agent error: {response.status_line()}: {results_string}"
)
else:
# Error was generated by server
http_status_code = response.code()
if http_status_code == HTTPStatus.METHOD_NOT_ALLOWED.value \
or http_status_code == HTTPStatus.BAD_REQUEST.value:
# Not POST, empty POST
fatal_error(f'{response.status_line()}: {results_string}')
elif http_status_code == HTTPStatus.INTERNAL_SERVER_ERROR.value:
# Processing error -- raise so that the error gets caught and logged into a database
raise McTagsFromJSONAnnotationException(
f'Annotator service was unable to process the download: {results_string}'
)
else:
# Shutdown the extractor on unconfigured responses
fatal_error(
f'Unknown HTTP response: {response.status_line()}: {results_string}'
)
if results_string is None or len(results_string) == 0:
raise McTagsFromJSONAnnotationException(
f"Annotator returned nothing for text: {text}")
log.debug("Parsing response's JSON...")
results = None
try:
results = decode_json(results_string)
if results is None:
raise McTagsFromJSONAnnotationException(
"Returned JSON is None.")
except Exception as ex:
# If the JSON is invalid, it's probably something broken with the remote service, so that's why whe do
# fatal_error() here
fatal_error(
f"Unable to parse JSON response: {ex}\nJSON string: {results_string}"
)
log.debug("Done parsing response's JSON.")
response_is_valid = False
try:
response_is_valid = self._fetched_annotation_is_valid(results)
except Exception as ex:
fatal_error(
f"Unable to determine whether response is valid: {ex}\nJSON string: {results_string}"
)
if not response_is_valid:
fatal_error(
f"Annotator response is invalid for JSON string: {results_string}"
)
log.info(f"Done annotating {len(text)} characters of text.")
return results
def solr_request(path: str,
params: SolrParams = None,
content: Union[str, SolrParams] = None,
content_type: Optional[str] = None,
config: Optional[CommonConfig] = None) -> str:
"""
Send a request to Solr.
:param path: Solr path to call, e.g. 'select'.
:param params: Query parameters to add to the path.
:param content: String or dictionary content to send via POST request.
:param content_type: Content-Type for the POST content.
:param config: (testing) Configuration object
:return: Raw response content on success, raise exception on error.
"""
path = decode_object_from_bytes_if_needed(path)
params = decode_object_from_bytes_if_needed(params)
content = decode_object_from_bytes_if_needed(content)
content_type = decode_object_from_bytes_if_needed(content_type)
if not path:
raise McSolrRequestInvalidParamsException("Path is unset.")
if params:
if not isinstance(params, dict):
raise McSolrRequestInvalidParamsException(
f"Params is not a dictionary: {params}")
if content:
if not (isinstance(content, str) or isinstance(content, dict)):
raise McSolrRequestInvalidParamsException(
f"Content is not a string not a dictionary: {content}")
if not config:
config = CommonConfig()
solr_url = config.solr_url()
if not params:
params = {}
abs_uri = furl(f"{solr_url}/mediacloud/{path}")
abs_uri = abs_uri.set(params)
abs_url = str(abs_uri)
ua = UserAgent()
ua.set_timeout(__QUERY_HTTP_TIMEOUT)
ua.set_max_size(None)
# Remediate CVE-2017-12629
q_param = str(params.get('q', ''))
if 'xmlparser' in q_param.lower():
raise McSolrRequestQueryErrorException(
"XML queries are not supported.")
# Solr might still be starting up so wait for it to expose the collections list
__wait_for_solr_to_start(config=config)
if content:
if not content_type:
fallback_content_type = 'text/plain; charset=utf-8'
log.warning(
f"Content-Type is not set; falling back to '{fallback_content_type}'"
)
content_type = fallback_content_type
if isinstance(content, dict):
content = urlencode(content, doseq=True)
content_encoded = content.encode('utf-8', errors='replace')
request = Request(method='POST', url=abs_url)
request.set_header(name='Content-Type', value=content_type)
request.set_header(name='Content-Length',
value=str(len(content_encoded)))
request.set_content(content_encoded)
else:
request = Request(method='GET', url=abs_url)
log.debug(f"Sending Solr request: {request}")
response = ua.request(request)
if not response.is_success():
error_message = __solr_error_message_from_response(response=response)
raise McSolrRequestQueryErrorException(
f"Error fetching Solr response: {error_message}")
return response.decoded_content()
def __annotate_text(self, text: str) -> Union[dict, list]:
"""Fetch JSON annotation for text, decode it into dictionary / list."""
text = decode_object_from_bytes_if_needed(text)
if text is None:
fatal_error("Text is None.")
if len(text) == 0:
# Annotators accept empty strings, but that might happen with some stories so we're just die()ing here
raise McJSONAnnotatorException("Text is empty.")
log.info("Annotating %d characters of text..." % len(text))
# Trim the text because that's what the annotator will do, and if the text is empty, we want to fail early
# without making a request to the annotator at all
text = text.strip()
if self.__TEXT_LENGTH_LIMIT > 0:
text_length = len(text)
if text_length > self.__TEXT_LENGTH_LIMIT:
log.warning(
"Text length (%d) has exceeded the request text length limit (%d) so I will truncate it."
% (
text_length,
self.__TEXT_LENGTH_LIMIT,
))
text = text[:self.__TEXT_LENGTH_LIMIT]
# Make a request
ua = UserAgent()
ua.set_timing([1, 2, 4, 8])
ua.set_timeout(self.__HTTP_TIMEOUT)
ua.set_max_size(None)
request = None
try:
request = self._request_for_text(text=text)
if request is None:
raise McJSONAnnotatorException("Returned request is None.")
except Exception as ex:
# Assume that this is some sort of a programming error too
fatal_error(
"Unable to create annotator request for text '%s': %s" % (
text,
str(ex),
))
log.debug("Sending request to %s..." % request.url())
response = ua.request(request)
log.debug("Response received.")
# Force UTF-8 encoding on the response because the server might not always
# return correct "Content-Type"
results_string = response.decoded_utf8_content()
if not response.is_success():
# Error; determine whether we should be blamed for making a malformed
# request, or is it an extraction error
log.warning("Request failed: %s" % response.decoded_content())
if response.code() == HTTPStatus.REQUEST_TIMEOUT.value:
# Raise on request timeouts without retrying anything because those usually mean that we posted
# something funky to the annotator service and it got stuck
raise McJSONAnnotatorException(
"The request timed out, giving up; text length: %d; text: %s"
% (
len(text),
text,
))
if response.error_is_client_side():
# Error was generated by the user agent client code; likely didn't reach server at all (timeout,
# unresponsive host, etc.)
fatal_error("User agent error: %s: %s" % (
response.status_line(),
results_string,
))
else:
# Error was generated by server
http_status_code = response.code()
if http_status_code == HTTPStatus.METHOD_NOT_ALLOWED.value \
or http_status_code == HTTPStatus.BAD_REQUEST.value:
# Not POST, empty POST
fatal_error('%s: %s' % (
response.status_line(),
results_string,
))
elif http_status_code == HTTPStatus.INTERNAL_SERVER_ERROR.value:
# Processing error -- raise so that the error gets caught and logged into a database
raise McJSONAnnotatorException(
'Annotator service was unable to process the download: %s'
% results_string)
else:
# Shutdown the extractor on unconfigured responses
fatal_error('Unknown HTTP response: %s: %s' % (
response.status_line(),
results_string,
))
if results_string is None or len(results_string) == 0:
raise McJSONAnnotatorException(
"Annotator returned nothing for text: %s" % text)
log.debug("Parsing response's JSON...")
results = None
try:
results = decode_json(results_string)
if results is None:
raise McJSONAnnotatorException("Returned JSON is None.")
except Exception as ex:
# If the JSON is invalid, it's probably something broken with the remote service, so that's why whe do
# fatal_error() here
fatal_error("Unable to parse JSON response: %s\nJSON string: %s" %
(
str(ex),
results_string,
))
log.debug("Done parsing response's JSON.")
response_is_valid = False
try:
response_is_valid = self._fetched_annotation_is_valid(results)
except Exception as ex:
fatal_error(
"Unable to determine whether response is valid: %s\nJSON string: %s"
% (str(ex), results_string))
if not response_is_valid:
fatal_error("Annotator response is invalid for JSON string: %s" %
results_string)
try:
results = self._postprocess_fetched_annotation(results)
if results is None:
raise McJSONAnnotatorException(
"Annotation is None after postprocessing.")
except Exception as ex:
fatal_error(
"Unable to postprocess fetched response: %s\nJSON string: %s" %
(str(ex), results_string))
log.info("Done annotating %d characters of text." % len(text))
return results