def do_prompt(d, key, text, default=None, validator=nonempty):
while True:
if default:
prompt = purple(PROMPT_PREFIX + '%s [%s]: ' % (text, default))
else:
prompt = purple(PROMPT_PREFIX + text + ': ')
x = term_input(prompt)
if default and not x:
x = default
if not isinstance(x, unicode):
# for Python 2.x, try to get a Unicode string out of it
if x.decode('ascii', 'replace').encode('ascii', 'replace') != x:
if TERM_ENCODING:
x = x.decode(TERM_ENCODING)
else:
print turquoise('* Note: non-ASCII characters entered '
'and terminal encoding unknown -- assuming '
'UTF-8 or Latin-1.')
try:
x = x.decode('utf-8')
except UnicodeDecodeError:
x = x.decode('latin1')
try:
x = validator(x)
except ValidationError, err:
print red('* ' + str(err))
continue
break
def do_prompt(d, key, text, default=None, validator=nonempty):
while True:
if default:
prompt = purple(PROMPT_PREFIX + '%s [%s]: ' % (text, default))
else:
prompt = purple(PROMPT_PREFIX + text + ': ')
x = term_input(prompt).strip()
if default and not x:
x = default
if not isinstance(x, unicode):
# for Python 2.x, try to get a Unicode string out of it
if x.decode('ascii', 'replace').encode('ascii', 'replace') != x:
if TERM_ENCODING:
x = x.decode(TERM_ENCODING)
else:
print turquoise(
'* Note: non-ASCII characters entered '
'and terminal encoding unknown -- assuming '
'UTF-8 or Latin-1.')
try:
x = x.decode('utf-8')
except UnicodeDecodeError:
x = x.decode('latin1')
try:
x = validator(x)
except ValidationError, err:
print red('* ' + str(err))
continue
break
def do_prompt(d, key, text, default=None, validator=nonempty):
while True:
if default:
prompt = purple(PROMPT_PREFIX + "%s [%s]: " % (text, default))
else:
prompt = purple(PROMPT_PREFIX + text + ": ")
x = term_input(prompt).strip()
if default and not x:
x = default
if not isinstance(x, unicode):
# for Python 2.x, try to get a Unicode string out of it
if x.decode("ascii", "replace").encode("ascii", "replace") != x:
if TERM_ENCODING:
x = x.decode(TERM_ENCODING)
else:
print turquoise(
"* Note: non-ASCII characters entered "
"and terminal encoding unknown -- assuming "
"UTF-8 or Latin-1."
)
try:
x = x.decode("utf-8")
except UnicodeDecodeError:
x = x.decode("latin1")
try:
x = validator(x)
except ValidationError, err:
print red("* " + str(err))
continue
break
def do_prompt(d, key, text, default=None, validator=nonempty):
while True:
if default:
prompt = purple(PROMPT_PREFIX + '%s [%s]: ' % (text, default))
else:
prompt = purple(PROMPT_PREFIX + text + ': ')
x = raw_input(prompt)
if default and not x:
x = default
if x.decode('ascii', 'replace').encode('ascii', 'replace') != x:
if TERM_ENCODING:
x = x.decode(TERM_ENCODING)
else:
print turquoise(
'* Note: non-ASCII characters entered and terminal '
'encoding unknown -- assuming UTF-8 or Latin-1.')
try:
x = x.decode('utf-8')
except UnicodeDecodeError:
x = x.decode('latin1')
if validator and not validator(x):
print red('* ' + validator.__doc__)
continue
break
d[key] = x
def do_prompt(d, key, text, default=None, validator=nonempty):
while True:
if default:
prompt = purple('> %s [%s]: ' % (text, default))
else:
prompt = purple('> ' + text + ': ')
x = raw_input(prompt)
if default and not x:
x = default
if validator and not validator(x):
print red(" * " + validator.__doc__)
continue
break
d[key] = x
def do_prompt(d, key, text, default=None, validator=nonempty):
while True:
if default:
prompt = purple(PROMPT_PREFIX + '%s [%s]: ' % (text, default))
else:
prompt = purple(PROMPT_PREFIX + text + ': ')
x = raw_input(prompt)
if default and not x:
x = default
if validator and not validator(x):
print red(" * " + validator.__doc__)
continue
break
d[key] = x
def process_result(self, result):
uri, docname, lineno, status, info = result
if status == 'unchecked':
return
if status == 'working' and info != 'new':
return
if lineno:
self.info('(line %3d) ' % lineno, nonl=1)
if status == 'ignored':
self.info(uri + ' - ' + darkgray('ignored'))
elif status == 'local':
self.info(uri + ' - ' + darkgray('local'))
self.write_entry('local', docname, lineno, uri)
elif status == 'working':
self.info(uri + ' - ' + darkgreen('working'))
elif status == 'broken':
self.info(uri + ' - ' + red('broken: ') + info)
self.write_entry('broken', docname, lineno, uri + ': ' + info)
if self.app.quiet:
self.warn('broken link: %s' % uri,
'%s:%s' % (self.env.doc2path(docname), lineno))
elif status == 'redirected':
self.info(uri + ' - ' + purple('redirected') + ' to ' + info)
self.write_entry('redirected', docname, lineno,
uri + ' to ' + info)
def do_prompt(text, default=None, validator=nonempty):
# type: (unicode, unicode, Callable[[unicode], Any]) -> Union[unicode, bool]
while True:
if default is not None:
prompt = PROMPT_PREFIX + '%s [%s]: ' % (text, default) # type: unicode
else:
prompt = PROMPT_PREFIX + text + ': '
if PY2:
# for Python 2.x, try to get a Unicode string out of it
if prompt.encode('ascii', 'replace').decode('ascii', 'replace') \
!= prompt:
if TERM_ENCODING:
prompt = prompt.encode(TERM_ENCODING)
else:
print(turquoise(__('* Note: non-ASCII default value provided '
'and terminal encoding unknown -- assuming '
'UTF-8 or Latin-1.')))
try:
prompt = prompt.encode('utf-8')
except UnicodeEncodeError:
prompt = prompt.encode('latin1')
prompt = purple(prompt)
x = term_input(prompt).strip()
if default and not x:
x = default
x = term_decode(x)
try:
x = validator(x)
except ValidationError as err:
print(red('* ' + str(err)))
continue
break
return x
def do_prompt(d, key, text, default=None, validator=nonempty):
while True:
if default is not None:
prompt = PROMPT_PREFIX + "%s [%s]: " % (text, default)
else:
prompt = PROMPT_PREFIX + text + ": "
if PY2:
# for Python 2.x, try to get a Unicode string out of it
if prompt.encode("ascii", "replace").decode("ascii", "replace") != prompt:
if TERM_ENCODING:
prompt = prompt.encode(TERM_ENCODING)
else:
print(
turquoise(
"* Note: non-ASCII default value provided "
"and terminal encoding unknown -- assuming "
"UTF-8 or Latin-1."
)
)
try:
prompt = prompt.encode("utf-8")
except UnicodeEncodeError:
prompt = prompt.encode("latin1")
prompt = purple(prompt)
x = term_input(prompt).strip()
if default and not x:
x = default
x = term_decode(x)
try:
x = validator(x)
except ValidationError as err:
print(red("* " + str(err)))
continue
break
d[key] = x
def do_prompt(d, key, text, default=None, validator=nonempty):
# type: (Dict, unicode, unicode, unicode, Callable[[unicode], Any]) -> None
while True:
if default is not None:
prompt = PROMPT_PREFIX + '%s [%s]: ' % (text, default) # type: unicode
else:
prompt = PROMPT_PREFIX + text + ': '
if PY2:
# for Python 2.x, try to get a Unicode string out of it
if prompt.encode('ascii', 'replace').decode('ascii', 'replace') \
!= prompt:
if TERM_ENCODING:
prompt = prompt.encode(TERM_ENCODING)
else:
print(turquoise('* Note: non-ASCII default value provided '
'and terminal encoding unknown -- assuming '
'UTF-8 or Latin-1.'))
try:
prompt = prompt.encode('utf-8')
except UnicodeEncodeError:
prompt = prompt.encode('latin1')
prompt = purple(prompt)
x = term_input(prompt).strip()
if default and not x:
x = default
x = term_decode(x)
try:
x = validator(x)
except ValidationError as err:
print(red('* ' + str(err)))
continue
break
d[key] = x
def check(self, node, docname):
uri = node['refuri']
if '#' in uri:
uri = uri.split('#')[0]
if uri in self.good:
return
lineno = None
while lineno is None:
node = node.parent
if node is None:
break
lineno = node.line
if len(uri) == 0 or uri[0:7] == 'mailto:' or uri[0:4] == 'ftp:':
return
if lineno:
self.info('(line %3d) ' % lineno, nonl=1)
for rex in self.to_ignore:
if rex.match(uri):
self.info(uri + ' - ' + darkgray('ignored'))
return
if uri[0:5] == 'http:' or uri[0:6] == 'https:':
self.info(uri, nonl=1)
if uri in self.broken:
(r, s) = self.broken[uri]
elif uri in self.redirected:
(r, s) = self.redirected[uri]
else:
(r, s) = self.resolve(uri)
if r == 0:
self.info(' - ' + darkgreen('working'))
self.good.add(uri)
elif r == 2:
self.info(' - ' + red('broken: ') + s)
self.write_entry('broken', docname, lineno, uri + ': ' + s)
self.broken[uri] = (r, s)
if self.app.quiet:
self.warn('broken link: %s' % uri,
'%s:%s' % (self.env.doc2path(docname), lineno))
else:
self.info(' - ' + purple('redirected') + ' to ' + s)
self.write_entry('redirected', docname, lineno,
uri + ' to ' + s)
self.redirected[uri] = (r, s)
else:
self.info(uri + ' - ' + darkgray('local'))
self.write_entry('local', docname, lineno, uri)
if self.broken:
self.app.statuscode = 1
def check(self, node, docname):
uri = node['refuri']
if '#' in uri:
uri = uri.split('#')[0]
if uri in self.good:
return
lineno = None
while lineno is None:
node = node.parent
if node is None:
break
lineno = node.line
if len(uri) == 0 or uri[0:7] == 'mailto:' or uri[0:4] == 'ftp:':
return
if lineno:
self.info('(line %3d) ' % lineno, nonl=1)
for rex in self.to_ignore:
if rex.match(uri):
self.info(uri + ' - ' + darkgray('ignored'))
return
if uri[0:5] == 'http:' or uri[0:6] == 'https:':
self.info(uri, nonl=1)
if uri in self.broken:
(r, s) = self.broken[uri]
elif uri in self.redirected:
(r, s) = self.redirected[uri]
else:
(r, s) = self.resolve(uri)
if r == 0:
self.info(' - ' + darkgreen('working'))
self.good.add(uri)
elif r == 2:
self.info(' - ' + red('broken: ') + s)
self.write_entry('broken', docname, lineno, uri + ': ' + s)
self.broken[uri] = (r, s)
if self.app.quiet:
self.warn('broken link: %s' % uri,
'%s:%s' % (self.env.doc2path(docname), lineno))
else:
self.info(' - ' + purple('redirected') + ' to ' + s)
self.write_entry('redirected', docname,
lineno, uri + ' to ' + s)
self.redirected[uri] = (r, s)
else:
self.info(uri + ' - ' + darkgray('local'))
self.write_entry('local', docname, lineno, uri)
if self.broken:
self.app.statuscode = 1
def logger_func(string):
if string.startswith("preserved"):
return
elif string.startswith("rewrote"):
string = purple(string)
elif string.startswith("pruned"):
string = darkred(string)
elif string.startswith("wrote"):
string = darkgreen(string)
logger.info("{} {}".format(bold("[uqbar-api]"), string))
def check(self, node, docname):
uri = node['refuri']
if '#' in uri:
uri = uri.split('#')[0]
if uri in self.good:
return
lineno = None
while lineno is None and node:
node = node.parent
lineno = node.line
if uri[0:5] == 'http:' or uri[0:6] == 'https:':
self.info(uri, nonl=1)
if uri in self.broken:
(r, s) = self.broken[uri]
elif uri in self.redirected:
(r, s) = self.redirected[uri]
else:
(r, s) = self.resolve(uri)
if r == 0:
self.info(' - ' + darkgreen('working'))
self.good.add(uri)
elif r == 2:
self.info(' - ' + red('broken: ') + s)
self.write_entry('broken', docname, lineno, uri + ': ' + s)
self.broken[uri] = (r, s)
if self.app.quiet:
self.warn('%s:%s: broken link: %s' %
(docname, lineno, uri))
else:
self.info(' - ' + purple('redirected') + ' to ' + s)
self.write_entry('redirected', docname, lineno,
uri + ' to ' + s)
self.redirected[uri] = (r, s)
elif len(uri) == 0 or uri[0:7] == 'mailto:' or uri[0:4] == 'ftp:':
return
else:
self.warn(uri + ' - ' + red('malformed!'))
self.write_entry('malformed', docname, lineno, uri)
if self.app.quiet:
self.warn('%s:%s: malformed link: %s' % (docname, lineno, uri))
self.app.statuscode = 1
if self.broken:
self.app.statuscode = 1
def check(self, node, docname):
uri = node["refuri"]
if "#" in uri:
uri = uri.split("#")[0]
if uri in self.good:
return
lineno = None
while lineno is None:
node = node.parent
if node is None:
break
lineno = node.line
if len(uri) == 0 or uri[0:7] == "mailto:" or uri[0:4] == "ftp:":
return
if lineno:
self.info("(line %3d) " % lineno, nonl=1)
if uri[0:5] == "http:" or uri[0:6] == "https:":
self.info(uri, nonl=1)
if uri in self.broken:
(r, s) = self.broken[uri]
elif uri in self.redirected:
(r, s) = self.redirected[uri]
else:
(r, s) = self.resolve(uri)
if r == 0:
self.info(" - " + darkgreen("working"))
self.good.add(uri)
elif r == 2:
self.info(" - " + red("broken: ") + s)
self.write_entry("broken", docname, lineno, uri + ": " + s)
self.broken[uri] = (r, s)
if self.app.quiet:
self.warn("broken link: %s" % uri, "%s:%s" % (self.env.doc2path(docname), lineno))
else:
self.info(" - " + purple("redirected") + " to " + s)
self.write_entry("redirected", docname, lineno, uri + " to " + s)
self.redirected[uri] = (r, s)
else:
self.info(uri + " - " + darkgray("local"))
self.write_entry("local", docname, lineno, uri)
if self.broken:
self.app.statuscode = 1
def do_prompt(d, key, text, default=None, validator=nonempty):
while True:
if default:
prompt = purple(PROMPT_PREFIX + '%s [%s]: ' % (text, default))
else:
prompt = purple(PROMPT_PREFIX + text + ': ')
x = raw_input(prompt)
if default and not x:
x = default
if x.decode('ascii', 'replace').encode('ascii', 'replace') != x:
if TERM_ENCODING:
x = x.decode(TERM_ENCODING)
else:
print turquoise('* Note: non-ASCII characters entered and terminal '
'encoding unknown -- assuming UTF-8 or Latin-1.')
try:
x = x.decode('utf-8')
except UnicodeDecodeError:
x = x.decode('latin1')
if validator and not validator(x):
print red('* ' + validator.__doc__)
continue
break
d[key] = x
def check(self, node, docname):
uri = node['refuri']
if '#' in uri:
uri = uri.split('#')[0]
if uri in self.good:
return
lineno = None
while lineno is None and node:
node = node.parent
lineno = node.line
if uri[0:5] == 'http:' or uri[0:6] == 'https:':
self.info(uri, nonl=1)
if uri in self.broken:
(r, s) = self.broken[uri]
elif uri in self.redirected:
(r, s) = self.redirected[uri]
else:
(r, s) = self.resolve(uri)
if r == 0:
self.info(' - ' + darkgreen('working'))
self.good.add(uri)
elif r == 2:
self.info(' - ' + red('broken: ') + s)
self.write_entry('broken', docname, lineno, uri + ': ' + s)
self.broken[uri] = (r, s)
if self.app.quiet:
self.warn('%s:%s: broken link: %s' % (docname, lineno, uri))
else:
self.info(' - ' + purple('redirected') + ' to ' + s)
self.write_entry('redirected', docname, lineno, uri + ' to ' + s)
self.redirected[uri] = (r, s)
elif len(uri) == 0 or uri[0:7] == 'mailto:' or uri[0:4] == 'ftp:':
return
else:
self.warn(uri + ' - ' + red('malformed!'))
self.write_entry('malformed', docname, lineno, uri)
if self.app.quiet:
self.warn('%s:%s: malformed link: %s' % (docname, lineno, uri))
self.app.statuscode = 1
if self.broken:
self.app.statuscode = 1
def do_prompt(d, key, text, default=None, validator=nonempty):
while True:
if default:
prompt = PROMPT_PREFIX + '%s [%s]: ' % (text, default)
else:
prompt = PROMPT_PREFIX + text + ': '
if sys.version_info < (3, 0):
# for Python 2.x, try to get a Unicode string out of it
if prompt.encode('ascii', 'replace').decode('ascii', 'replace') \
!= prompt:
if TERM_ENCODING:
prompt = prompt.encode(TERM_ENCODING)
else:
print(turquoise('* Note: non-ASCII default value provided '
'and terminal encoding unknown -- assuming '
'UTF-8 or Latin-1.'))
try:
prompt = prompt.encode('utf-8')
except UnicodeEncodeError:
prompt = prompt.encode('latin1')
prompt = purple(prompt)
x = term_input(prompt).strip()
if default and not x:
x = default
if not isinstance(x, str):
# for Python 2.x, try to get a Unicode string out of it
if x.decode('ascii', 'replace').encode('ascii', 'replace') != x:
if TERM_ENCODING:
x = x.decode(TERM_ENCODING)
else:
print(turquoise('* Note: non-ASCII characters entered '
'and terminal encoding unknown -- assuming '
'UTF-8 or Latin-1.'))
try:
x = x.decode('utf-8')
except UnicodeDecodeError:
x = x.decode('latin1')
try:
x = validator(x)
except ValidationError as err:
print(red('* ' + str(err)))
continue
break
d[key] = x
def process_result(self, result):
uri, docname, lineno, status, info = result
if status == 'unchecked':
return
if status == 'working' and info != 'new':
return
if lineno:
self.info('(line %3d) ' % lineno, nonl=1)
if status == 'ignored':
self.info(uri + ' - ' + darkgray('ignored'))
elif status == 'local':
self.info(uri + ' - ' + darkgray('local'))
self.write_entry('local', docname, lineno, uri)
elif status == 'working':
self.info(uri + ' - ' + darkgreen('working'))
elif status == 'broken':
self.info(uri + ' - ' + red('broken: ') + info)
self.write_entry('broken', docname, lineno, uri + ': ' + info)
if self.app.quiet:
self.warn('broken link: %s' % uri,
'%s:%s' % (self.env.doc2path(docname), lineno))
elif status == 'redirected':
self.info(uri + ' - ' + purple('redirected') + ' to ' + info)
self.write_entry('redirected', docname, lineno, uri + ' to ' + info)
def main():
# check that the linkcheck file exists
linkcheck_file = os.path.join('build', 'linkcheck', 'output.txt')
if not os.path.exists(linkcheck_file):
error_exit('no linkcheck output file; run make linkcheck')
# check that it hasn't been more than a day since the last linkcheck
last_linkcheck = datetime.fromtimestamp(os.path.getmtime(linkcheck_file))
if datetime.now() - last_linkcheck > timedelta(days=1):
error_exit('linkcheck output outdated; run make linkcheck')
# parse each line of the linkcheck output.txt file
with open(linkcheck_file) as fp:
lines = fp.readlines()
local = {}
broken = {}
perm_redirects = {}
temp_redirects = {}
for line in lines:
m = LINE_RE.match(line)
if m is None:
error_exit('could not parse: {!r}'.format(line))
continue
filename, lineno, status, url, more = m.groups()
# ignore links with certain status messages
if '429' in more and 'Too Many Requests' in more.title():
continue
# gather data for broken urls
elif status == 'broken':
url = url.rstrip(':')
m = REASON_RE.match(more)
more = m.group(1) if m else more.strip()
broken.setdefault(filename, {})[url] = more
# gather local links
elif status == 'local':
local.setdefault(filename, set()).add(url)
# gather data for permanent redirects
elif status == 'redirected permanently':
dst = more.split(' to ', 1)[-1].strip()
perm_redirects.setdefault(filename, {})[url] = dst
# gather data for ...other ...temporary? redirects
elif status.startswith('redirected'):
dst = more.split(' to ', 1)[-1].strip()
temp_redirects.setdefault(filename, {})[url] = dst
n = update_redirects(perm_redirects)
print()
urls = {x for urls in perm_redirects.values() for x in urls.items()}
print(white('Found {} links returning 301s [{} replacements made]'.format(len(urls), n)))
for src, dst in sorted(urls):
print(src + darkgreen(' -> ' + dst))
print()
urls = {x for urls in temp_redirects.values() for x in urls.items()}
print(white('Found {} links returning other 300 codes [no replacements made]'.format(len(urls))))
for src, dst in sorted(urls):
print(src + purple(' -> ' + dst))
print()
urls = {x for urls in broken.values() for x in urls.items()}
print(white('Found {} links returning error codes (excluding 429)'.format(len(urls))))
for url, reason in sorted(urls):
print(url + darkred(' - ' + reason))