def target_request_from_meta_refresh_url(
content: str, archive_site_url: str) -> Union[Request, None]:
"""Given a URL and content from website with META refresh, return a request for the original URL."""
content = decode_object_from_bytes_if_needed(content)
archive_site_url = decode_object_from_bytes_if_needed(archive_site_url)
if content is None:
return None
target_url = meta_refresh_url_from_html(html=content,
base_url=archive_site_url)
if target_url is None:
return None
return Request(method='GET', url=target_url)
def test_meta_refresh_url_from_html():
# No <meta http-equiv="refresh" />
assert meta_refresh_url_from_html(html="""
<html>
<head>
<title>This is a test</title>
<meta http-equiv="content-type" content="text/html; charset=UTF-8" />
</head>
<body>
<p>This is a test.</p>
</body>
</html>
""",
base_url='http://example.com/') is None
# Basic HTML <meta http-equiv="refresh">
assert meta_refresh_url_from_html(
html="""
<HTML>
<HEAD>
<TITLE>This is a test</TITLE>
<META HTTP-EQUIV="content-type" CONTENT="text/html; charset=UTF-8">
<META HTTP-EQUIV="refresh" CONTENT="0; URL=http://example.com/">
</HEAD>
<BODY>
<P>This is a test.</P>
</BODY>
</HTML>
""",
base_url='http://example.com/') == 'http://example.com/'
# Basic XHTML <meta http-equiv="refresh" />
assert meta_refresh_url_from_html(
html="""
<html>
<head>
<title>This is a test</title>
<meta http-equiv="content-type" content="text/html; charset=UTF-8" />
<meta http-equiv="refresh" content="0; url=http://example.com/" />
</head>
<body>
<p>This is a test.</p>
</body>
</html>
""",
base_url='http://example.com/') == 'http://example.com/'
# Basic XHTML sans the seconds part
assert meta_refresh_url_from_html(
html="""
<html>
<head>
<title>This is a test</title>
<meta http-equiv="content-type" content="text/html; charset=UTF-8" />
<meta http-equiv="refresh" content="url=http://example.com/" />
</head>
<body>
<p>This is a test.</p>
</body>
</html>
""",
base_url='http://example.com/') == 'http://example.com/'
# Basic XHTML with quoted url
assert meta_refresh_url_from_html(
html="""
<html>
<head>
<title>This is a test</title>
<meta http-equiv="content-type" content="text/html; charset=UTF-8" />
<meta http-equiv="refresh" content="url='http://example.com/'" />
</head>
<body>
<p>This is a test.</p>
</body>
</html>
""",
base_url='http://example.com/') == 'http://example.com/'
# Basic XHTML with reverse quoted url
assert meta_refresh_url_from_html(
html="""
<html>
<head>
<title>This is a test</title>
<meta http-equiv="content-type" content="text/html; charset=UTF-8" />
<meta http-equiv="refresh" content='url="http://example.com/"' />
</head>
<body>
<p>This is a test.</p>
</body>
</html>
""",
base_url='http://example.com/') == 'http://example.com/'
# Relative path (base URL with trailing slash)
assert meta_refresh_url_from_html(
html="""
<meta http-equiv="refresh" content="0; url=second/third/" />
""",
base_url='http://example.com/first/'
) == 'http://example.com/first/second/third/'
# Relative path (base URL without trailing slash)
assert meta_refresh_url_from_html(html="""
<meta http-equiv="refresh" content="0; url=second/third/" />
""",
base_url='http://example.com/first'
) == 'http://example.com/second/third/'
# Absolute path
assert meta_refresh_url_from_html(
html="""
<meta http-equiv="refresh" content="0; url=/first/second/third/" />
""",
base_url='http://example.com/fourth/fifth/'
) == 'http://example.com/first/second/third/'
# Invalid URL without base URL
assert meta_refresh_url_from_html("""
<meta http-equiv="refresh" content="0; url=/first/second/third/" />
""") is None
# No url
no_url_results = meta_refresh_url_from_html(
'<meta http-equiv="refresh" content="2700"/>',
base_url='htt://foo.com')
assert no_url_results is None