def do_prompt(d, key, text, default=None, validator=nonempty): while True: if default: prompt = purple(PROMPT_PREFIX + '%s [%s]: ' % (text, default)) else: prompt = purple(PROMPT_PREFIX + text + ': ') x = term_input(prompt) if default and not x: x = default if not isinstance(x, unicode): # for Python 2.x, try to get a Unicode string out of it if x.decode('ascii', 'replace').encode('ascii', 'replace') != x: if TERM_ENCODING: x = x.decode(TERM_ENCODING) else: print turquoise('* Note: non-ASCII characters entered ' 'and terminal encoding unknown -- assuming ' 'UTF-8 or Latin-1.') try: x = x.decode('utf-8') except UnicodeDecodeError: x = x.decode('latin1') try: x = validator(x) except ValidationError, err: print red('* ' + str(err)) continue break
def do_prompt(d, key, text, default=None, validator=nonempty): while True: if default: prompt = purple(PROMPT_PREFIX + '%s [%s]: ' % (text, default)) else: prompt = purple(PROMPT_PREFIX + text + ': ') x = term_input(prompt).strip() if default and not x: x = default if not isinstance(x, unicode): # for Python 2.x, try to get a Unicode string out of it if x.decode('ascii', 'replace').encode('ascii', 'replace') != x: if TERM_ENCODING: x = x.decode(TERM_ENCODING) else: print turquoise( '* Note: non-ASCII characters entered ' 'and terminal encoding unknown -- assuming ' 'UTF-8 or Latin-1.') try: x = x.decode('utf-8') except UnicodeDecodeError: x = x.decode('latin1') try: x = validator(x) except ValidationError, err: print red('* ' + str(err)) continue break
def do_prompt(d, key, text, default=None, validator=nonempty): while True: if default: prompt = purple(PROMPT_PREFIX + "%s [%s]: " % (text, default)) else: prompt = purple(PROMPT_PREFIX + text + ": ") x = term_input(prompt).strip() if default and not x: x = default if not isinstance(x, unicode): # for Python 2.x, try to get a Unicode string out of it if x.decode("ascii", "replace").encode("ascii", "replace") != x: if TERM_ENCODING: x = x.decode(TERM_ENCODING) else: print turquoise( "* Note: non-ASCII characters entered " "and terminal encoding unknown -- assuming " "UTF-8 or Latin-1." ) try: x = x.decode("utf-8") except UnicodeDecodeError: x = x.decode("latin1") try: x = validator(x) except ValidationError, err: print red("* " + str(err)) continue break
def do_prompt(d, key, text, default=None, validator=nonempty): while True: if default: prompt = purple(PROMPT_PREFIX + '%s [%s]: ' % (text, default)) else: prompt = purple(PROMPT_PREFIX + text + ': ') x = raw_input(prompt) if default and not x: x = default if x.decode('ascii', 'replace').encode('ascii', 'replace') != x: if TERM_ENCODING: x = x.decode(TERM_ENCODING) else: print turquoise( '* Note: non-ASCII characters entered and terminal ' 'encoding unknown -- assuming UTF-8 or Latin-1.') try: x = x.decode('utf-8') except UnicodeDecodeError: x = x.decode('latin1') if validator and not validator(x): print red('* ' + validator.__doc__) continue break d[key] = x
def do_prompt(d, key, text, default=None, validator=nonempty): while True: if default: prompt = purple('> %s [%s]: ' % (text, default)) else: prompt = purple('> ' + text + ': ') x = raw_input(prompt) if default and not x: x = default if validator and not validator(x): print red(" * " + validator.__doc__) continue break d[key] = x
def do_prompt(d, key, text, default=None, validator=nonempty): while True: if default: prompt = purple(PROMPT_PREFIX + '%s [%s]: ' % (text, default)) else: prompt = purple(PROMPT_PREFIX + text + ': ') x = raw_input(prompt) if default and not x: x = default if validator and not validator(x): print red(" * " + validator.__doc__) continue break d[key] = x
def process_result(self, result): uri, docname, lineno, status, info = result if status == 'unchecked': return if status == 'working' and info != 'new': return if lineno: self.info('(line %3d) ' % lineno, nonl=1) if status == 'ignored': self.info(uri + ' - ' + darkgray('ignored')) elif status == 'local': self.info(uri + ' - ' + darkgray('local')) self.write_entry('local', docname, lineno, uri) elif status == 'working': self.info(uri + ' - ' + darkgreen('working')) elif status == 'broken': self.info(uri + ' - ' + red('broken: ') + info) self.write_entry('broken', docname, lineno, uri + ': ' + info) if self.app.quiet: self.warn('broken link: %s' % uri, '%s:%s' % (self.env.doc2path(docname), lineno)) elif status == 'redirected': self.info(uri + ' - ' + purple('redirected') + ' to ' + info) self.write_entry('redirected', docname, lineno, uri + ' to ' + info)
def do_prompt(text, default=None, validator=nonempty): # type: (unicode, unicode, Callable[[unicode], Any]) -> Union[unicode, bool] while True: if default is not None: prompt = PROMPT_PREFIX + '%s [%s]: ' % (text, default) # type: unicode else: prompt = PROMPT_PREFIX + text + ': ' if PY2: # for Python 2.x, try to get a Unicode string out of it if prompt.encode('ascii', 'replace').decode('ascii', 'replace') \ != prompt: if TERM_ENCODING: prompt = prompt.encode(TERM_ENCODING) else: print(turquoise(__('* Note: non-ASCII default value provided ' 'and terminal encoding unknown -- assuming ' 'UTF-8 or Latin-1.'))) try: prompt = prompt.encode('utf-8') except UnicodeEncodeError: prompt = prompt.encode('latin1') prompt = purple(prompt) x = term_input(prompt).strip() if default and not x: x = default x = term_decode(x) try: x = validator(x) except ValidationError as err: print(red('* ' + str(err))) continue break return x
def do_prompt(d, key, text, default=None, validator=nonempty): while True: if default is not None: prompt = PROMPT_PREFIX + "%s [%s]: " % (text, default) else: prompt = PROMPT_PREFIX + text + ": " if PY2: # for Python 2.x, try to get a Unicode string out of it if prompt.encode("ascii", "replace").decode("ascii", "replace") != prompt: if TERM_ENCODING: prompt = prompt.encode(TERM_ENCODING) else: print( turquoise( "* Note: non-ASCII default value provided " "and terminal encoding unknown -- assuming " "UTF-8 or Latin-1." ) ) try: prompt = prompt.encode("utf-8") except UnicodeEncodeError: prompt = prompt.encode("latin1") prompt = purple(prompt) x = term_input(prompt).strip() if default and not x: x = default x = term_decode(x) try: x = validator(x) except ValidationError as err: print(red("* " + str(err))) continue break d[key] = x
def do_prompt(d, key, text, default=None, validator=nonempty): # type: (Dict, unicode, unicode, unicode, Callable[[unicode], Any]) -> None while True: if default is not None: prompt = PROMPT_PREFIX + '%s [%s]: ' % (text, default) # type: unicode else: prompt = PROMPT_PREFIX + text + ': ' if PY2: # for Python 2.x, try to get a Unicode string out of it if prompt.encode('ascii', 'replace').decode('ascii', 'replace') \ != prompt: if TERM_ENCODING: prompt = prompt.encode(TERM_ENCODING) else: print(turquoise('* Note: non-ASCII default value provided ' 'and terminal encoding unknown -- assuming ' 'UTF-8 or Latin-1.')) try: prompt = prompt.encode('utf-8') except UnicodeEncodeError: prompt = prompt.encode('latin1') prompt = purple(prompt) x = term_input(prompt).strip() if default and not x: x = default x = term_decode(x) try: x = validator(x) except ValidationError as err: print(red('* ' + str(err))) continue break d[key] = x
def check(self, node, docname): uri = node['refuri'] if '#' in uri: uri = uri.split('#')[0] if uri in self.good: return lineno = None while lineno is None: node = node.parent if node is None: break lineno = node.line if len(uri) == 0 or uri[0:7] == 'mailto:' or uri[0:4] == 'ftp:': return if lineno: self.info('(line %3d) ' % lineno, nonl=1) for rex in self.to_ignore: if rex.match(uri): self.info(uri + ' - ' + darkgray('ignored')) return if uri[0:5] == 'http:' or uri[0:6] == 'https:': self.info(uri, nonl=1) if uri in self.broken: (r, s) = self.broken[uri] elif uri in self.redirected: (r, s) = self.redirected[uri] else: (r, s) = self.resolve(uri) if r == 0: self.info(' - ' + darkgreen('working')) self.good.add(uri) elif r == 2: self.info(' - ' + red('broken: ') + s) self.write_entry('broken', docname, lineno, uri + ': ' + s) self.broken[uri] = (r, s) if self.app.quiet: self.warn('broken link: %s' % uri, '%s:%s' % (self.env.doc2path(docname), lineno)) else: self.info(' - ' + purple('redirected') + ' to ' + s) self.write_entry('redirected', docname, lineno, uri + ' to ' + s) self.redirected[uri] = (r, s) else: self.info(uri + ' - ' + darkgray('local')) self.write_entry('local', docname, lineno, uri) if self.broken: self.app.statuscode = 1
def logger_func(string): if string.startswith("preserved"): return elif string.startswith("rewrote"): string = purple(string) elif string.startswith("pruned"): string = darkred(string) elif string.startswith("wrote"): string = darkgreen(string) logger.info("{} {}".format(bold("[uqbar-api]"), string))
def check(self, node, docname): uri = node['refuri'] if '#' in uri: uri = uri.split('#')[0] if uri in self.good: return lineno = None while lineno is None and node: node = node.parent lineno = node.line if uri[0:5] == 'http:' or uri[0:6] == 'https:': self.info(uri, nonl=1) if uri in self.broken: (r, s) = self.broken[uri] elif uri in self.redirected: (r, s) = self.redirected[uri] else: (r, s) = self.resolve(uri) if r == 0: self.info(' - ' + darkgreen('working')) self.good.add(uri) elif r == 2: self.info(' - ' + red('broken: ') + s) self.write_entry('broken', docname, lineno, uri + ': ' + s) self.broken[uri] = (r, s) if self.app.quiet: self.warn('%s:%s: broken link: %s' % (docname, lineno, uri)) else: self.info(' - ' + purple('redirected') + ' to ' + s) self.write_entry('redirected', docname, lineno, uri + ' to ' + s) self.redirected[uri] = (r, s) elif len(uri) == 0 or uri[0:7] == 'mailto:' or uri[0:4] == 'ftp:': return else: self.warn(uri + ' - ' + red('malformed!')) self.write_entry('malformed', docname, lineno, uri) if self.app.quiet: self.warn('%s:%s: malformed link: %s' % (docname, lineno, uri)) self.app.statuscode = 1 if self.broken: self.app.statuscode = 1
def check(self, node, docname): uri = node["refuri"] if "#" in uri: uri = uri.split("#")[0] if uri in self.good: return lineno = None while lineno is None: node = node.parent if node is None: break lineno = node.line if len(uri) == 0 or uri[0:7] == "mailto:" or uri[0:4] == "ftp:": return if lineno: self.info("(line %3d) " % lineno, nonl=1) if uri[0:5] == "http:" or uri[0:6] == "https:": self.info(uri, nonl=1) if uri in self.broken: (r, s) = self.broken[uri] elif uri in self.redirected: (r, s) = self.redirected[uri] else: (r, s) = self.resolve(uri) if r == 0: self.info(" - " + darkgreen("working")) self.good.add(uri) elif r == 2: self.info(" - " + red("broken: ") + s) self.write_entry("broken", docname, lineno, uri + ": " + s) self.broken[uri] = (r, s) if self.app.quiet: self.warn("broken link: %s" % uri, "%s:%s" % (self.env.doc2path(docname), lineno)) else: self.info(" - " + purple("redirected") + " to " + s) self.write_entry("redirected", docname, lineno, uri + " to " + s) self.redirected[uri] = (r, s) else: self.info(uri + " - " + darkgray("local")) self.write_entry("local", docname, lineno, uri) if self.broken: self.app.statuscode = 1
def do_prompt(d, key, text, default=None, validator=nonempty): while True: if default: prompt = purple(PROMPT_PREFIX + '%s [%s]: ' % (text, default)) else: prompt = purple(PROMPT_PREFIX + text + ': ') x = raw_input(prompt) if default and not x: x = default if x.decode('ascii', 'replace').encode('ascii', 'replace') != x: if TERM_ENCODING: x = x.decode(TERM_ENCODING) else: print turquoise('* Note: non-ASCII characters entered and terminal ' 'encoding unknown -- assuming UTF-8 or Latin-1.') try: x = x.decode('utf-8') except UnicodeDecodeError: x = x.decode('latin1') if validator and not validator(x): print red('* ' + validator.__doc__) continue break d[key] = x
def do_prompt(d, key, text, default=None, validator=nonempty): while True: if default: prompt = PROMPT_PREFIX + '%s [%s]: ' % (text, default) else: prompt = PROMPT_PREFIX + text + ': ' if sys.version_info < (3, 0): # for Python 2.x, try to get a Unicode string out of it if prompt.encode('ascii', 'replace').decode('ascii', 'replace') \ != prompt: if TERM_ENCODING: prompt = prompt.encode(TERM_ENCODING) else: print(turquoise('* Note: non-ASCII default value provided ' 'and terminal encoding unknown -- assuming ' 'UTF-8 or Latin-1.')) try: prompt = prompt.encode('utf-8') except UnicodeEncodeError: prompt = prompt.encode('latin1') prompt = purple(prompt) x = term_input(prompt).strip() if default and not x: x = default if not isinstance(x, str): # for Python 2.x, try to get a Unicode string out of it if x.decode('ascii', 'replace').encode('ascii', 'replace') != x: if TERM_ENCODING: x = x.decode(TERM_ENCODING) else: print(turquoise('* Note: non-ASCII characters entered ' 'and terminal encoding unknown -- assuming ' 'UTF-8 or Latin-1.')) try: x = x.decode('utf-8') except UnicodeDecodeError: x = x.decode('latin1') try: x = validator(x) except ValidationError as err: print(red('* ' + str(err))) continue break d[key] = x
def main(): # check that the linkcheck file exists linkcheck_file = os.path.join('build', 'linkcheck', 'output.txt') if not os.path.exists(linkcheck_file): error_exit('no linkcheck output file; run make linkcheck') # check that it hasn't been more than a day since the last linkcheck last_linkcheck = datetime.fromtimestamp(os.path.getmtime(linkcheck_file)) if datetime.now() - last_linkcheck > timedelta(days=1): error_exit('linkcheck output outdated; run make linkcheck') # parse each line of the linkcheck output.txt file with open(linkcheck_file) as fp: lines = fp.readlines() local = {} broken = {} perm_redirects = {} temp_redirects = {} for line in lines: m = LINE_RE.match(line) if m is None: error_exit('could not parse: {!r}'.format(line)) continue filename, lineno, status, url, more = m.groups() # ignore links with certain status messages if '429' in more and 'Too Many Requests' in more.title(): continue # gather data for broken urls elif status == 'broken': url = url.rstrip(':') m = REASON_RE.match(more) more = m.group(1) if m else more.strip() broken.setdefault(filename, {})[url] = more # gather local links elif status == 'local': local.setdefault(filename, set()).add(url) # gather data for permanent redirects elif status == 'redirected permanently': dst = more.split(' to ', 1)[-1].strip() perm_redirects.setdefault(filename, {})[url] = dst # gather data for ...other ...temporary? redirects elif status.startswith('redirected'): dst = more.split(' to ', 1)[-1].strip() temp_redirects.setdefault(filename, {})[url] = dst n = update_redirects(perm_redirects) print() urls = {x for urls in perm_redirects.values() for x in urls.items()} print(white('Found {} links returning 301s [{} replacements made]'.format(len(urls), n))) for src, dst in sorted(urls): print(src + darkgreen(' -> ' + dst)) print() urls = {x for urls in temp_redirects.values() for x in urls.items()} print(white('Found {} links returning other 300 codes [no replacements made]'.format(len(urls)))) for src, dst in sorted(urls): print(src + purple(' -> ' + dst)) print() urls = {x for urls in broken.values() for x in urls.items()} print(white('Found {} links returning error codes (excluding 429)'.format(len(urls)))) for url, reason in sorted(urls): print(url + darkred(' - ' + reason))