def check_git(): """Check for uncommited git files..""" if not os.path.isdir(".git"): print("No .git dir, ignoring") print() return False untracked = [] changed = [] gitst = subprocess.check_output(['git', 'status', '--porcelain']) gitst = gitst.decode('UTF-8').strip() for line in gitst.splitlines(): s, name = line.split(maxsplit=1) if s == '??' and name != '.venv/': untracked.append(name) elif s == 'M': changed.append(name) status = True if untracked: status = False utils.print_col("Untracked files:", 'red') print('\n'.join(untracked)) if changed: status = False utils.print_col("Uncommited changes:", 'red') print('\n'.join(changed)) print() return status
def main(colors=False): """Generate html files for the online documentation.""" utils.change_cwd() utils.use_color = colors parser = argparse.ArgumentParser() parser.add_argument('--website', help="Build website into a given " "directory.", nargs=1) parser.add_argument('--asciidoc', help="Full path to python and " "asciidoc.py. If not given, it's searched in PATH.", nargs=2, required=False, metavar=('PYTHON', 'ASCIIDOC')) parser.add_argument('--no-authors', help=argparse.SUPPRESS, action='store_true') args = parser.parse_args() try: os.mkdir('qutebrowser/html/doc') except FileExistsError: pass asciidoc = AsciiDoc(args) try: asciidoc.prepare() except FileNotFoundError: utils.print_col("Could not find asciidoc! Please install it, or use " "the --asciidoc argument to point this script to the " "correct python/asciidoc.py location!", 'red') sys.exit(1) try: asciidoc.build() finally: asciidoc.cleanup()
def call_asciidoc(src, dst): """Call asciidoc for the given files. Args: src: The source .asciidoc file. dst: The destination .html file, or None to auto-guess. """ print("Calling asciidoc for {}...".format(os.path.basename(src))) if os.name == 'nt': # FIXME this is highly specific to my machine # https://github.com/The-Compiler/qutebrowser/issues/106 args = [r'C:\Python27\python', r'J:\bin\asciidoc-8.6.9\asciidoc.py'] else: args = ['asciidoc'] if dst is not None: args += ['--out-file', dst] args.append(src) try: subprocess.check_call(args) except FileNotFoundError: utils.print_col("asciidoc needs to be installed to use this script!", 'red') sys.exit(1) except (subprocess.CalledProcessError, OSError) as e: utils.print_col(str(e), 'red') sys.exit(1)
def main(): """Regenerate all documentation.""" utils.print_col("Generating asciidoc files...", 'cyan') regenerate_manpage('doc/qutebrowser.1.asciidoc') generate_settings('doc/help/settings.asciidoc') generate_commands('doc/help/commands.asciidoc') regenerate_authors('README.asciidoc') if '--html' in sys.argv: asciidoc2html.main()
def main(colors=False): """Generate html files for the online documentation.""" utils.change_cwd() utils.use_color = colors parser = argparse.ArgumentParser() parser.add_argument('--all', help="Build all documentation into a given " "directory.", nargs=1) parser.add_argument('--asciidoc', help="Full path to python and " "asciidoc.py. If not given, it's searched in PATH.", nargs=2, required=False, metavar=('PYTHON', 'ASCIIDOC')) args = parser.parse_args() asciidoc_files = [ ('FAQ.asciidoc', 'qutebrowser/html/doc/FAQ.html'), ('CHANGELOG.asciidoc', 'qutebrowser/html/doc/CHANGELOG.html'), ('doc/quickstart.asciidoc', 'qutebrowser/html/doc/quickstart.html'), ('doc/userscripts.asciidoc', 'qutebrowser/html/doc/userscripts.html'), ] try: os.mkdir('qutebrowser/html/doc') except FileExistsError: pass try: asciidoc = _get_asciidoc_cmd(args) except FileNotFoundError: utils.print_col("Could not find asciidoc! Please install it, or use " "the --asciidoc argument to point this script to the " "correct python/asciidoc.py location!", 'red') sys.exit(1) if args.all: for root, _dirs, files in os.walk(os.getcwd()): for filename in files: if os.path.splitext(filename)[1] != '.asciidoc': continue src = os.path.join(root, filename) parts = [args.all[0]] dirname = os.path.dirname(src) if dirname: parts.append(os.path.relpath(os.path.dirname(src))) parts.append( os.extsep.join((os.path.splitext(os.path.basename(src))[0], 'html'))) dst = os.path.join(*parts) os.makedirs(os.path.dirname(dst), exist_ok=True) call_asciidoc(asciidoc, src, dst) else: for src in glob.glob('doc/help/*.asciidoc'): name, _ext = os.path.splitext(os.path.basename(src)) dst = 'qutebrowser/html/doc/{}.html'.format(name) asciidoc_files.append((src, dst)) for src, dst in asciidoc_files: call_asciidoc(asciidoc, src, dst)
def main(): """Main entry point.""" exit_status = collections.OrderedDict() exit_status_bool = {} parser = argparse.ArgumentParser(description='Run various checkers.') parser.add_argument('-s', '--setup', help="Run additional setup checks", action='store_true') parser.add_argument('checkers', help="Checkers to run (or 'all')", default='all', nargs='?') args = parser.parse_args() checkers = _get_checkers() groups = ['global'] groups += config.get('DEFAULT', 'targets').split(',') groups.append('setup') for group in groups: print() utils.print_col("==================== {} ====================".format( group), 'yellow') for name, func in checkers[group].items(): utils.print_col("------ {} ------".format(name), 'cyan') if _checker_enabled(args, group, name): status = func() key = '{}_{}'.format(group, name) exit_status[key] = status if name == 'flake8': # pyflakes uses True for errors and False for ok. exit_status_bool[key] = not status elif isinstance(status, bool): exit_status_bool[key] = status else: # sys.exit(0) means no problems -> True, anything != 0 # means problems. exit_status_bool[key] = (status == 0) else: utils.print_col("Checker disabled.", 'blue') print() utils.print_col("Exit status values:", 'yellow') for (k, v) in exit_status.items(): ok = exit_status_bool[k] color = 'green' if ok else 'red' utils.print_col( ' {} - {} ({})'.format(k, 'ok' if ok else 'FAIL', v), color) if all(exit_status_bool): return 0 else: return 1
def main(): """Main entry point.""" utils.change_cwd() read_files = config.read('.run_checks') if not read_files: raise IOError("Could not read config!") exit_status = collections.OrderedDict() exit_status_bool = {} args = _parse_args() checkers = _get_checkers() groups = ['global'] groups += config.get('DEFAULT', 'targets').split(',') groups.append('setup') for group in groups: print() utils.print_col("==================== {} ====================".format( group), 'yellow') for name, func in checkers[group].items(): utils.print_col("------ {} ------".format(name), 'cyan') if _checker_enabled(args, group, name): status = func() key = '{}_{}'.format(group, name) exit_status[key] = status if name == 'flake8': # pyflakes uses True for errors and False for ok. exit_status_bool[key] = not status elif isinstance(status, bool): exit_status_bool[key] = status else: # sys.exit(0) means no problems -> True, anything != 0 # means problems. exit_status_bool[key] = (status == 0) else: utils.print_col("Checker disabled.", 'blue') print() utils.print_col("Exit status values:", 'yellow') for (k, v) in exit_status.items(): ok = exit_status_bool[k] color = 'green' if ok else 'red' utils.print_col( ' {} - {} ({})'.format(k, 'ok' if ok else 'FAIL', v), color) if all(exit_status_bool): return 0 else: return 1
def call_asciidoc(args, src, dst): """Call asciidoc for the given files. Args: args: The asciidoc binary to use, as a list. src: The source .asciidoc file. dst: The destination .html file, or None to auto-guess. """ print("Calling asciidoc for {}...".format(os.path.basename(src))) args = args[:] if dst is not None: args += ['--out-file', dst] args.append(src) try: subprocess.check_call(args) except (subprocess.CalledProcessError, OSError) as e: utils.print_col(str(e), 'red') sys.exit(1)
def check_git(): """Check for uncommitted git files..""" if not os.path.isdir(".git"): print("No .git dir, ignoring") print() return False untracked = [] gitst = subprocess.check_output(["git", "status", "--porcelain"]) gitst = gitst.decode("UTF-8").strip() for line in gitst.splitlines(): s, name = line.split(maxsplit=1) if s == "??" and name != ".venv/": untracked.append(name) status = True if untracked: status = False utils.print_col("Untracked files:", "red") print("\n".join(untracked)) print() return status
def call(self, src, dst, *args): """Call asciidoc for the given files. Args: src: The source .asciidoc file. dst: The destination .html file, or None to auto-guess. *args: Additional arguments passed to asciidoc. """ print("Calling asciidoc for {}...".format(os.path.basename(src))) cmdline = self._cmd[:] if dst is not None: cmdline += ['--out-file', dst] cmdline += args cmdline.append(src) try: subprocess.check_call(cmdline, env={'HOME': self._homedir}) self._failed = True except (subprocess.CalledProcessError, OSError) as e: self._failed = True utils.print_col(str(e), 'red') print("Keeping modified sources in {}.".format(self._homedir)) sys.exit(1)
def main() -> None: """Install qutebrowser in a virtualenv..""" args = parse_args() venv_dir = pathlib.Path(args.venv_dir) utils.change_cwd() if args.tox_error: show_tox_error(args.pyqt_type) sys.exit(1) elif args.pyqt_type == 'link' and args.pyqt_version != 'auto': utils.print_col( 'The --pyqt-version option is not available when ' 'linking a system-wide install.', 'red') sys.exit(1) if not args.keep: utils.print_title("Creating virtual environment") delete_old_venv(venv_dir) create_venv(venv_dir, use_virtualenv=args.virtualenv) upgrade_seed_pkgs(venv_dir) if args.pyqt_type == 'binary': install_pyqt_binary(venv_dir, args.pyqt_version) elif args.pyqt_type == 'source': install_pyqt_source(venv_dir, args.pyqt_version) elif args.pyqt_type == 'link': install_pyqt_link(venv_dir) elif args.pyqt_type == 'skip': pass else: raise AssertionError install_requirements(venv_dir) install_qutebrowser(venv_dir) if args.dev: install_dev_requirements(venv_dir) regenerate_docs(venv_dir, args.asciidoc)
def call_asciidoc(src, dst): """Call asciidoc for the given files. Args: src: The source .asciidoc file. dst: The destination .html file, or None to auto-guess. """ utils.print_col("Calling asciidoc for {}...".format( os.path.basename(src)), 'cyan') if os.name == 'nt': # FIXME this is highly specific to my machine args = [r'C:\Python27\python', r'J:\bin\asciidoc-8.6.9\asciidoc.py'] else: args = ['asciidoc'] if dst is not None: args += ['--out-file', dst] args.append(src) try: subprocess.check_call(args) except subprocess.CalledProcessError as e: utils.print_col(str(e), 'red') sys.exit(1)
def check_git(_args: argparse.Namespace = None) -> bool: """Check for uncommitted git files..""" if not os.path.isdir(".git"): print("No .git dir, ignoring") print() return False untracked = [] gitst = subprocess.run(['git', 'status', '--porcelain'], check=True, stdout=subprocess.PIPE).stdout gitst = gitst.decode('UTF-8').strip() for line in gitst.splitlines(): s, name = line.split(maxsplit=1) if s == '??' and name != '.venv/': untracked.append(name) status = True if untracked: status = False utils.print_col("Untracked files:", 'red') print('\n'.join(untracked)) print() return status
def check_changelog_urls(_args: argparse.Namespace = None) -> bool: """Ensure we have changelog URLs for all requirements.""" ok = True all_requirements = set() for name in recompile_requirements.get_all_names(): outfile = recompile_requirements.get_outfile(name) missing = set() with open(outfile, 'r', encoding='utf-8') as f: for line in f: line = line.strip() if line.startswith('#') or not line: continue req, _version = recompile_requirements.parse_versioned_line( line) if req.startswith('./'): continue all_requirements.add(req) if req not in recompile_requirements.CHANGELOG_URLS: missing.add(req) if missing: ok = False req_str = ', '.join(sorted(missing)) utils.print_col( f"Missing changelog URLs in {name} requirements: {req_str}", 'red') extra = set(recompile_requirements.CHANGELOG_URLS) - all_requirements if extra: ok = False req_str = ', '.join(sorted(extra)) utils.print_col(f"Extra changelog URLs: {req_str}", 'red') if not ok: print("Hint: Changelog URLs are in scripts/dev/changelog_urls.json") return ok
def main(colors=False): """Generate html files for the online documentation.""" utils.change_cwd() utils.use_color = colors parser = argparse.ArgumentParser() parser.add_argument('--website', help="Build website into a given " "directory.", nargs=1) parser.add_argument('--asciidoc', help="Full path to python and " "asciidoc.py. If not given, it's searched in PATH.", nargs=2, required=False, metavar=('PYTHON', 'ASCIIDOC')) parser.add_argument('--no-authors', help=argparse.SUPPRESS, action='store_true') args = parser.parse_args() try: os.mkdir('qutebrowser/html/doc') except FileExistsError: pass asciidoc = AsciiDoc(args) try: asciidoc.prepare() except FileNotFoundError: utils.print_col( "Could not find asciidoc! Please install it, or use " "the --asciidoc argument to point this script to the " "correct python/asciidoc.py location!", 'red') sys.exit(1) try: asciidoc.build() finally: asciidoc.cleanup()
def install_pyqt_binary(venv_dir: pathlib.Path, version: str) -> None: """Install PyQt from a binary wheel.""" utils.print_title("Installing PyQt from binary") utils.print_col( "No proprietary codec support will be available in " "qutebrowser.", 'bold') supported_archs = { 'linux': {'x86_64'}, 'win32': {'x86', 'AMD64'}, 'darwin': {'x86_64'}, } if sys.platform not in supported_archs: utils.print_error( f"{sys.platform} is not a supported platform by PyQt5 binary " "packages, this will most likely fail.") elif platform.machine() not in supported_archs[sys.platform]: utils.print_error( f"{platform.machine()} is not a supported architecture for PyQt5 binaries " f"on {sys.platform}, this will most likely fail.") pip_install(venv_dir, '-r', pyqt_requirements_file(version), '--only-binary', 'PyQt5,PyQtWebEngine')
def init_venv(host_python, venv_dir, requirements, pre=False, pip_args=None): """Initialize a new virtualenv and install the given packages.""" with utils.gha_group('Creating virtualenv'): utils.print_col('$ python3 -m venv {}'.format(venv_dir), 'blue') subprocess.run([host_python, '-m', 'venv', venv_dir], check=True) run_pip(venv_dir, 'install', '-U', 'pip', quiet=not utils.ON_CI) run_pip(venv_dir, 'install', '-U', 'setuptools', 'wheel', quiet=not utils.ON_CI) install_command = ['install', '-r', requirements] if pre: install_command.append('--pre') if pip_args: install_command += pip_args with utils.gha_group('Installing requirements'): run_pip(venv_dir, *install_command) run_pip(venv_dir, 'check')
def _get_files(*, verbose: bool, ignored: List[pathlib.Path] = None) -> Iterator[pathlib.Path]: """Iterate over all files and yield filenames.""" filenames = subprocess.run( [ 'git', 'ls-files', '--cached', '--others', '--exclude-standard', '-z' ], stdout=subprocess.PIPE, universal_newlines=True, check=True, ) all_ignored = ignored or [] all_ignored.append( pathlib.Path('tests', 'unit', 'scripts', 'importer_sample', 'chrome')) for filename in filenames.stdout.split('\0'): path = pathlib.Path(filename) is_ignored = any(path == p or p in path.parents for p in all_ignored) if not filename or path.suffix in BINARY_EXTS or is_ignored: continue try: with tokenize.open(str(path)): pass except SyntaxError as e: # Could not find encoding utils.print_col( "{} - maybe {} should be added to BINARY_EXTS?".format( str(e).capitalize(), path.suffix), 'yellow') continue if verbose: print(path) yield path
def print_ret(ret): """Print information about an exit status.""" if ret == 0: utils.print_col("success", 'green') elif ret == -signal.SIGSEGV: utils.print_col("segfault", 'red') else: utils.print_col("error {}".format(ret), 'yellow') print()
def test_tox(): """Test requirements via tox.""" host_python = get_host_python('tox') req_path = os.path.join(REQ_DIR, 'requirements-tox.txt') with tempfile.TemporaryDirectory() as tmpdir: venv_dir = os.path.join(tmpdir, 'venv') tox_workdir = os.path.join(tmpdir, 'tox-workdir') venv_python = get_venv_python(venv_dir) init_venv(host_python, venv_dir, req_path) list_proc = subprocess.run([venv_python, '-m', 'tox', '--listenvs'], check=True, stdout=subprocess.PIPE, universal_newlines=True) environments = list_proc.stdout.strip().split('\n') for env in environments: with utils.gha_group('tox for {}'.format(env)): utils.print_subtitle(env) utils.print_col('venv$ tox -e {} --notest'.format(env), 'blue') subprocess.run([ venv_python, '-m', 'tox', '--workdir', tox_workdir, '-e', env, '--notest' ], check=True)
def main(): parser = argparse.ArgumentParser() parser.add_argument('qt_location', help='Qt compiler directory') parser.add_argument('--wheels-dir', help='Directory to use for wheels', default='wheels') args = parser.parse_args() old_cwd = pathlib.Path.cwd() wheels_dir = pathlib.Path(args.wheels_dir).resolve() wheels_dir.mkdir(exist_ok=True) if list(wheels_dir.glob('*')): utils.print_col("Wheels directory is not empty, " "unexpected behavior might occur!", 'yellow') os.chdir(wheels_dir) utils.print_title("Downloading wheels") subprocess.run([sys.executable, '-m', 'pip', 'download', '--no-deps', '--only-binary', 'PyQt5,PyQtWebEngine', 'PyQt5', 'PyQtWebEngine'], check=True) utils.print_title("Patching wheels") input_files = wheels_dir.glob('*.whl') for wheel in input_files: utils.print_subtitle(wheel.stem.split('-')[0]) bin_path = pathlib.Path(sys.executable).parent subprocess.run([str(bin_path / 'pyqt-bundle'), '--qt-dir', args.qt_location, str(wheel)], check=True) wheel.unlink() print("Done, output files:") for wheel in wheels_dir.glob('*.whl'): print(wheel.relative_to(old_cwd))
def create_venv(venv_dir: pathlib.Path, use_virtualenv: bool = False) -> None: """Create a new virtualenv.""" if use_virtualenv: utils.print_col('$ python3 -m virtualenv {}'.format(venv_dir), 'blue') try: subprocess.run([sys.executable, '-m', 'virtualenv', venv_dir], check=True) except subprocess.CalledProcessError as e: utils.print_col("virtualenv failed, exiting", 'red') sys.exit(e.returncode) else: utils.print_col('$ python3 -m venv {}'.format(venv_dir), 'blue') venv.create(str(venv_dir), with_pip=True)
def main(): """Main entry point.""" utils.change_cwd() read_files = config.read('.run_checks') if not read_files: raise OSError("Could not read config!") exit_status = collections.OrderedDict() exit_status_bool = {} args = _parse_args() checkers = _get_checkers(args) groups = ['global'] groups += config.get('DEFAULT', 'targets').split(',') groups.append('setup') for group in groups: print() utils.print_title(group) for name, func in checkers[group].items(): if _checker_enabled(args, group, name): utils.print_subtitle(name) status = func() key = '{}_{}'.format(group, name) exit_status[key] = status if name == 'flake8': # pyflakes uses True for errors and False for ok. exit_status_bool[key] = not status elif isinstance(status, bool): exit_status_bool[key] = status else: # sys.exit(0) means no problems -> True, anything != 0 # means problems. exit_status_bool[key] = (status == 0) elif not args.quiet: utils.print_subtitle(name) utils.print_col("Checker disabled.", 'blue') print() utils.print_col("Exit status values:", 'yellow') for (k, v) in exit_status.items(): ok = exit_status_bool[k] color = 'green' if ok else 'red' utils.print_col( ' {} - {} ({})'.format(k, 'ok' if ok else 'FAIL', v), color) if all(exit_status_bool.values()): return 0 else: return 1
def cleanup_pylint_build(): """Clean up pylint_checkers build files.""" path = pathlib.Path(__file__).parent / 'pylint_checkers' / 'build' utils.print_col(f'$ rm -r {path}', 'blue') shutil.rmtree(path)
def pip_install(venv_dir: pathlib.Path, *args: str) -> None: """Run a pip install command inside the virtualenv.""" arg_str = ' '.join(str(arg) for arg in args) utils.print_col('venv$ pip install {}'.format(arg_str), 'blue') run_venv(venv_dir, 'python', '-m', 'pip', 'install', *args)
def print_command(*cmd: Union[str, pathlib.Path], venv: bool) -> None: """Print a command being run.""" prefix = 'venv$ ' if venv else '$ ' utils.print_col(prefix + ' '.join([str(e) for e in cmd]), 'blue')
def create_venv(venv_dir: pathlib.Path) -> None: """Create a new virtualenv.""" utils.print_col('$ python3 -m venv {}'.format(venv_dir), 'blue') venv.create(str(venv_dir), with_pip=True)