def _fetch_url(db: DatabaseHandler,
url: str,
network_down_host: str = DEFAULT_NETWORK_DOWN_HOST,
network_down_port: int = DEFAULT_NETWORK_DOWN_PORT,
network_down_timeout: int = DEFAULT_NETWORK_DOWN_TIMEOUT,
domain_timeout: Optional[int] = None) -> FetchLinkResponse:
"""Fetch a url and return the content.
If fetching the url results in a 400 error, check whether the network_down_host is accessible. If so,
return the errored response. Otherwise, wait network_down_timeout seconds and try again.
This function catches McGetException and returns a dummy 400 Response object.
Arguments:
db - db handle
url - url to fetch
network_down_host - host to check if network is down on error
network_down_port - port to check if network is down on error
network_down_timeout - seconds to wait if the network is down
domain_timeout - value to pass to ThrottledUserAgent()
Returns:
Response object
"""
if url_has_binary_extension(url):
return _make_dummy_bypassed_response(url)
while True:
ua = ThrottledUserAgent(db, domain_timeout=domain_timeout)
if is_http_url(url):
ua_response = ua.get_follow_http_html_redirects(url)
response = FetchLinkResponse.from_useragent_response(
url, ua_response)
else:
log.warning(f"URL is not HTTP(s), returning dummy response: {url}")
response = FetchLinkResponse(
url=url,
is_success=False,
code=HTTPStatus.BAD_REQUEST.value,
message=HTTPStatus.BAD_REQUEST.phrase,
content='bad url',
last_requested_url=None,
)
if response.is_success:
return response
if response.code == HTTPStatus.BAD_REQUEST.value and not tcp_port_is_open(
port=network_down_port, hostname=network_down_host):
log.warning(
"Response failed with %s and network is down. Waiting to retry ..."
% (url, ))
time.sleep(network_down_timeout)
else:
return response
def _fetch_url(
db: DatabaseHandler,
url: str,
network_down_host: str = DEFAULT_NETWORK_DOWN_HOST,
network_down_port: int = DEFAULT_NETWORK_DOWN_PORT,
network_down_timeout: int = DEFAULT_NETWORK_DOWN_TIMEOUT,
domain_timeout: typing.Optional[int] = None) -> FetchLinkResponse:
"""Fetch a url and return the content.
If fetching the url results in a 400 error, check whether the network_down_host is accessible. If so,
return the errored response. Otherwise, wait network_down_timeout seconds and try again.
This function catches McGetException and returns a dummy 400 Response object.
Arguments:
db - db handle
url - url to fetch
network_down_host - host to check if network is down on error
network_down_port - port to check if network is down on error
network_down_timeout - seconds to wait if the network is down
domain_timeout - value to pass to ThrottledUserAgent()
Returns:
Response object
"""
if mediawords.tm.stories.url_has_binary_extension(url):
return _make_dummy_bypassed_response(url)
while True:
ua = ThrottledUserAgent(db, domain_timeout=domain_timeout)
if mediawords.util.url.is_http_url(url):
ua_response = ua.get_follow_http_html_redirects(url)
response = FetchLinkResponse.from_useragent_response(url, ua_response)
else:
response = FetchLinkResponse(
url=url,
is_success=False,
code=HTTPStatus.BAD_REQUEST.value,
message=HTTPStatus.BAD_REQUEST.phrase,
content='bad url',
last_requested_url=None,
)
if response.is_success:
return response
if response.code == HTTPStatus.BAD_REQUEST.value and not tcp_port_is_open(port=network_down_port,
hostname=network_down_host):
log.warning("Response failed with %s and network is down. Waiting to retry ..." % (url,))
time.sleep(network_down_timeout)
else:
return response
def fetch_url(
db: DatabaseHandler,
url: str,
network_down_host: str = DEFAULT_NETWORK_DOWN_HOST,
network_down_port: str = DEFAULT_NETWORK_DOWN_PORT,
network_down_timeout: int = DEFAULT_NETWORK_DOWN_TIMEOUT,
domain_timeout: typing.Optional[int] = None
) -> typing.Optional[Request]:
"""Fetch a url and return the content.
If fetching the url results in a 400 error, check whether the network_down_host is accessible. If so,
return the errored response. Otherwise, wait network_down_timeout seconds and try again.
This function catches McGetException and returns a dummy 400 Response object.
Arguments:
db - db handle
url - url to fetch
network_down_host - host to check if network is down on error
network_down_port - port to check if network is down on error
network_down_timeout - seconds to wait if the network is down
domain_timeout - value to pass to ThrottledUserAgent()
Returns:
Response object
"""
while True:
ua = ThrottledUserAgent(db, domain_timeout=domain_timeout)
try:
response = ua.get_follow_http_html_redirects(url)
except mediawords.util.web.user_agent.McGetFollowHTTPHTMLRedirectsException:
response = Response(400, 'bad url', {}, 'not a http url')
if response.is_success():
return response
if response.code() == 400 and _network_is_down(network_down_host,
network_down_port):
log.warning(
"Response failed with %s and network is down. Waiting to retry ..."
% (url, ))
time.sleep(network_down_timeout)
else:
return response
def test_request(self) -> None:
"""Test requests with throttling."""
pages = {'/test': 'Hello!', }
port = 8888
hs = HashServer(port=port, pages=pages)
hs.start()
ua = ThrottledUserAgent(self.db(), domain_timeout=2)
test_url = hs.page_url('/test')
# first request should work
response = ua.get(test_url)
assert response.decoded_content() == 'Hello!'
# fail because we're in the timeout
ua = ThrottledUserAgent(self.db(), domain_timeout=2)
self.assertRaises(McThrottledDomainException, ua.get, test_url)
# succeed because it's a different domain
ua = ThrottledUserAgent(self.db(), domain_timeout=2)
response = ua.get('http://127.0.0.1:8888/test')
assert response.decoded_content() == 'Hello!'
# still fail within the timeout
ua = ThrottledUserAgent(self.db(), domain_timeout=2)
self.assertRaises(McThrottledDomainException, ua.get, test_url)
time.sleep(2)
# now we're outside the timeout, so it should work
ua = ThrottledUserAgent(self.db(), domain_timeout=2)
response = ua.get(test_url)
assert response.decoded_content() == 'Hello!'
# and follow up request on the same ua object should work
response = ua.get(test_url)
assert response.decoded_content() == 'Hello!'
# but then fail within the new timeout period with a new object
ua = ThrottledUserAgent(self.db(), domain_timeout=2)
self.assertRaises(McThrottledDomainException, ua.get, test_url)
hs.stop()
# test domain_timeout assignment logic
ua = ThrottledUserAgent(self.db(), domain_timeout=100)
assert ua.domain_timeout == 100
config = mediawords.util.config.get_config()
config['mediawords']['throttled_user_agent_domain_timeout'] = 200
ua = ThrottledUserAgent(self.db())
assert ua.domain_timeout == 200
del config['mediawords']['throttled_user_agent_domain_timeout']
ua = ThrottledUserAgent(self.db())
assert ua.domain_timeout == mediawords.util.web.user_agent.throttled._DEFAULT_DOMAIN_TIMEOUT
def test_request(self) -> None:
"""Test requests with throttling."""
pages = {'/test': 'Hello!', }
port = 8888
hs = HashServer(port=port, pages=pages)
hs.start()
ua = ThrottledUserAgent(self.db(), domain_timeout=2)
test_url = hs.page_url('/test')
# first request should work
response = ua.get(test_url)
assert response.decoded_content() == 'Hello!'
# fail because we're in the timeout
ua = ThrottledUserAgent(self.db(), domain_timeout=2)
self.assertRaises(McThrottledDomainException, ua.get, test_url)
# succeed because it's a different domain
ua = ThrottledUserAgent(self.db(), domain_timeout=2)
response = ua.get('http://127.0.0.1:8888/test')
assert response.decoded_content() == 'Hello!'
# still fail within the timeout
ua = ThrottledUserAgent(self.db(), domain_timeout=2)
self.assertRaises(McThrottledDomainException, ua.get, test_url)
time.sleep(2)
# now we're outside the timeout, so it should work
ua = ThrottledUserAgent(self.db(), domain_timeout=2)
response = ua.get(test_url)
assert response.decoded_content() == 'Hello!'
# and follow up request on the same ua object should work
response = ua.get(test_url)
assert response.decoded_content() == 'Hello!'
# but then fail within the new timeout period with a new object
ua = ThrottledUserAgent(self.db(), domain_timeout=2)
self.assertRaises(McThrottledDomainException, ua.get, test_url)
hs.stop()
# test domain_timeout assignment logic
ua = ThrottledUserAgent(self.db(), domain_timeout=100)
assert ua.domain_timeout == 100
config = mediawords.util.config.get_config()
config['mediawords']['throttled_user_agent_domain_timeout'] = 200
ua = ThrottledUserAgent(self.db())
assert ua.domain_timeout == 200
del config['mediawords']['throttled_user_agent_domain_timeout']
ua = ThrottledUserAgent(self.db())
assert ua.domain_timeout == mediawords.util.web.user_agent.throttled._DEFAULT_DOMAIN_TIMEOUT