def lint_urls(root, lint_ctx):
"""Find referenced URLs and verify they are valid."""
urls, docs = find_urls_for_xml(root)
# This is from Google Chome 53.0.2785.143, current at time of writing:
BROWSER_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36"
def validate_url(url, lint_ctx, user_agent=None):
is_valid = True
if user_agent:
req = Request(url, headers={"User-Agent": user_agent})
else:
req = url
try:
handle = urlopen(req)
handle.read(100)
except HTTPError as e:
if e.code == 429:
# too many requests
pass
else:
is_valid = False
lint_ctx.error("HTTP Error %s accessing %s" % (e.code, url))
except URLError as e:
is_valid = False
lint_ctx.error("URL Error %s accessing %s" % (str(e), url))
if is_valid:
lint_ctx.info("URL OK %s" % url)
for url in urls:
validate_url(url, lint_ctx)
for url in docs:
validate_url(url, lint_ctx, BROWSER_USER_AGENT)
def lint_urls(root, lint_ctx):
"""Find referenced URLs and verify they are valid."""
urls, docs = find_urls_for_xml(root)
# This is from Google Chome 53.0.2785.143, current at time of writing:
BROWSER_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36"
def validate_url(url, lint_ctx, user_agent=None):
is_valid = True
if user_agent:
req = Request(url, headers={"User-Agent": user_agent})
else:
req = url
try:
handle = urlopen(req)
handle.read(100)
except HTTPError as e:
if e.code == 429:
# too many requests
pass
else:
is_valid = False
lint_ctx.error("HTTP Error %s accessing %s" % (e.code, url))
except URLError as e:
is_valid = False
lint_ctx.error("URL Error %s accessing %s" % (str(e), url))
if is_valid:
lint_ctx.info("URL OK %s" % url)
for url in urls:
validate_url(url, lint_ctx)
for url in docs:
validate_url(url, lint_ctx, BROWSER_USER_AGENT)
def lint_urls(root, lint_ctx):
urls = find_urls_for_xml(root)
def validate_url(url, lint_ctx):
try:
handle = urllib2.urlopen(url)
handle.read(100)
lint_ctx.info("URL OK %s" % url)
except urllib2.HTTPError as e:
lint_ctx.error("HTTP Error %s accessing %s" % (e.code, url))
except urllib2.URLError as e:
lint_ctx.error("URL Error %s accessing %s" % (str(e), url))
for url in urls:
validate_url(url, lint_ctx)
def lint_urls(root, lint_ctx):
"""Find referenced URLs and verify they are valid."""
urls, docs = find_urls_for_xml(root)
# This is from Google Chome on macOS, current at time of writing:
BROWSER_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Safari/537.36"
def validate_url(url, lint_ctx, user_agent=None):
is_valid = True
if url.startswith('http://') or url.startswith('https://'):
if user_agent:
headers = {"User-Agent": user_agent, 'Accept': '*/*'}
else:
headers = None
r = None
try:
r = requests.get(url, headers=headers, stream=True)
r.raise_for_status()
next(r.iter_content(1000))
except Exception as e:
if r is not None and r.status_code == 429:
# too many requests
pass
if r is not None and r.status_code == 403 and 'cloudflare' in r.text:
# CloudFlare protection block
pass
else:
is_valid = False
lint_ctx.error("Error '%s' accessing %s" % (e, url))
else:
try:
with urlopen(url) as handle:
handle.read(100)
except Exception as e:
is_valid = False
lint_ctx.error("Error '%s' accessing %s" % (e, url))
if is_valid:
lint_ctx.info("URL OK %s" % url)
for url in urls:
validate_url(url, lint_ctx)
for url in docs:
validate_url(url, lint_ctx, BROWSER_USER_AGENT)
def lint_urls(root, lint_ctx):
"""Find referenced URLs and verify they are valid."""
urls, docs = find_urls_for_xml(root)
# This is from Google Chome 53.0.2785.143, current at time of writing:
BROWSER_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36"
def validate_url(url, lint_ctx, user_agent=None):
is_valid = True
if url.startswith('http://') or url.startswith('https://'):
if user_agent:
headers = {"User-Agent": user_agent, 'Accept': '*/*'}
else:
headers = None
r = None
try:
r = requests.get(url, headers=headers, stream=True)
r.raise_for_status()
next(r.iter_content(1000))
except Exception as e:
if r and r.status_code == 429:
# too many requests
pass
else:
is_valid = False
lint_ctx.error("Error '%s' accessing %s" % (e, url))
else:
try:
handle = urlopen(url)
handle.read(100)
except Exception as e:
is_valid = False
lint_ctx.error("Error '%s' accessing %s" % (e, url))
if is_valid:
lint_ctx.info("URL OK %s" % url)
for url in urls:
validate_url(url, lint_ctx)
for url in docs:
validate_url(url, lint_ctx, BROWSER_USER_AGENT)