def print_changed_files(): """Output all changed files from this run.""" diff = git_diff() if utils.ON_CI: with utils.gha_group('Raw diff'): print('\n'.join(diff)) changed_files = _get_changed_files() files_text = '\n'.join('- ' + line for line in changed_files) changes = _get_changes(diff) changes_text = '\n'.join(str(change) for change in changes) utils.print_subtitle('Files') print(files_text) print() utils.print_subtitle('Changes') print(changes_text) if utils.ON_CI: print() print('::set-output name=changed::' + files_text.replace('\n', '%0A')) table_header = [ '| Requirement | old | new |', '|-------------|-----|-----|', ] diff_table = '%0A'.join(table_header + [change.table_str() for change in changes]) print('::set-output name=diff::' + diff_table)
def build_sdist(): """Build an sdist and list the contents.""" utils.print_title("Building sdist") _maybe_remove('dist') subprocess.check_call([sys.executable, 'setup.py', 'sdist']) dist_files = os.listdir(os.path.abspath('dist')) assert len(dist_files) == 1 dist_file = os.path.join('dist', dist_files[0]) subprocess.check_call(['gpg', '--detach-sign', '-a', dist_file]) tar = tarfile.open(dist_file) by_ext = collections.defaultdict(list) for tarinfo in tar.getmembers(): if not tarinfo.isfile(): continue name = os.sep.join(tarinfo.name.split(os.sep)[1:]) _base, ext = os.path.splitext(name) by_ext[ext].append(name) assert '.pyc' not in by_ext utils.print_title("sdist contents") for ext, files in sorted(by_ext.items()): utils.print_subtitle(ext) print('\n'.join(files))
def _package_windows_single( *, nsis_flags, outdir, desc_arch, desc_suffix, filename_arch, ): """Build the given installer/zip for windows.""" artifacts = [] utils.print_subtitle(f"Building {desc_arch} installer...") subprocess.run([ 'makensis.exe', f'/DVERSION={qutebrowser.__version__}', *nsis_flags, 'misc/nsis/qutebrowser.nsi' ], check=True) name = f'qutebrowser-{qutebrowser.__version__}-{filename_arch}.exe' artifacts.append(( os.path.join('dist', name), 'application/vnd.microsoft.portable-executable', f'Windows {desc_arch} installer{desc_suffix}', )) utils.print_subtitle(f"Zipping {desc_arch} standalone...") zip_name = ( f'qutebrowser-{qutebrowser.__version__}-windows-standalone-{filename_arch}' ) zip_path = os.path.join('dist', zip_name) shutil.make_archive(zip_path, 'zip', 'dist', os.path.basename(outdir)) artifacts.append((f'{zip_path}.zip', 'application/zip', f'Windows {desc_arch} standalone{desc_suffix}')) return artifacts
def print_changed_files(): """Output all changed files from this run.""" changed_files = _get_changed_files() files_text = '\n'.join('- ' + line for line in changed_files) changes = _get_changes() diff_text = '\n'.join(str(change) for change in changes) utils.print_title('Changed') utils.print_subtitle('Files') print(files_text) print() utils.print_subtitle('Diff') print(diff_text) if 'CI' in os.environ: print() print('::set-output name=changed::' + files_text.replace('\n', '%0A')) table_header = [ '| Requirement | old | new |', '|-------------|-----|-----|', ] diff_table = '%0A'.join(table_header + [change.table_str() for change in changes]) print('::set-output name=diff::' + diff_table)
def build_requirements(name): """Build a requirements file.""" utils.print_subtitle("Building") filename = os.path.join(REQ_DIR, 'requirements-{}.txt-raw'.format(name)) host_python = get_host_python(name) with open(filename, 'r', encoding='utf-8') as f: comments = read_comments(f) with tempfile.TemporaryDirectory() as tmpdir: init_venv(host_python=host_python, venv_dir=tmpdir, requirements=filename, pre=comments['pre']) with utils.gha_group('Freezing requirements'): proc = run_pip(tmpdir, 'freeze', stdout=subprocess.PIPE) reqs = proc.stdout.decode('utf-8') if utils.ON_CI: print(reqs.strip()) outfile = get_outfile(name) with open(outfile, 'w', encoding='utf-8') as f: f.write("# This file is automatically generated by " "scripts/dev/recompile_requirements.py\n\n") for line in reqs.splitlines(): if line.startswith('qutebrowser=='): continue f.write(convert_line(line, comments) + '\n') for line in comments['add']: f.write(line + '\n') return outfile
def test_tox(): """Test requirements via tox.""" utils.print_title('Testing via tox') host_python = get_host_python('tox') req_path = os.path.join(REQ_DIR, 'requirements-tox.txt') with tempfile.TemporaryDirectory() as tmpdir: venv_dir = os.path.join(tmpdir, 'venv') tox_workdir = os.path.join(tmpdir, 'tox-workdir') venv_python = os.path.join(venv_dir, 'bin', 'python') init_venv(host_python, venv_dir, req_path) list_proc = subprocess.run([venv_python, '-m', 'tox', '--listenvs'], check=True, stdout=subprocess.PIPE, universal_newlines=True) environments = list_proc.stdout.strip().split('\n') for env in environments: with utils.gha_group('tox for {}'.format(env)): utils.print_subtitle(env) utils.print_col('venv$ tox -e {} --notest'.format(env), 'blue') subprocess.run([ venv_python, '-m', 'tox', '--workdir', tox_workdir, '-e', env, '--notest' ], check=True)
def print_changed_files(): """Output all changed files from this run.""" changed_files = set() filenames = git_diff('--name-only') for filename in filenames: filename = filename.strip() filename = filename.replace('misc/requirements/requirements-', '') filename = filename.replace('.txt', '') changed_files.add(filename) files_text = '\n'.join('- ' + line for line in sorted(changed_files)) changes_dict = {} diff = git_diff() for line in diff: if not line.startswith('-') and not line.startswith('+'): continue if line.startswith('+++ ') or line.startswith('--- '): continue if '==' in line: name, version = line[1:].split('==') if ';' in version: # pip environment markers version = version.split(';')[0].strip() elif line[1:].startswith('-e'): rest, name = line.split('#egg=') version = rest.split('@')[1][:7] else: name = line[1:] version = '?' if name not in changes_dict: changes_dict[name] = Change(name) if line.startswith('-'): changes_dict[name].old = version elif line.startswith('+'): changes_dict[name].new = version changes = [change for _name, change in sorted(changes_dict.items())] diff_text = '\n'.join(str(change) for change in changes) utils.print_title('Changed') utils.print_subtitle('Files') print(files_text) print() utils.print_subtitle('Diff') print(diff_text) if 'CI' in os.environ: print() print('::set-output name=changed::' + files_text.replace('\n', '%0A')) table_header = [ '| Requirement | old | new |', '|-------------|-----|-----|', ] diff_table = '%0A'.join(table_header + [change.table_str() for change in changes]) print('::set-output name=diff::' + diff_table)
def install_dev_packages(): """Install the packages needed for development.""" for pkg in get_dev_packages(): utils.print_subtitle("Installing {}".format(pkg)) if os.name == 'nt': venv_python('-m', 'pip', 'install', '--no-clean', pkg) else: venv_python('-m', 'pip', 'install', pkg)
def test_requirements(name, outfile): """Test a resulting requirements file.""" print() utils.print_subtitle("Testing") host_python = get_host_python(name) with tempfile.TemporaryDirectory() as tmpdir: init_venv(host_python, tmpdir, outfile)
def main(): """Re-compile the given (or all) requirement files.""" names = sys.argv[1:] if len(sys.argv) > 1 else sorted(get_all_names()) for name in names: utils.print_title(name) filename = os.path.join(REQ_DIR, 'requirements-{}.txt-raw'.format(name)) if name == 'qutebrowser': outfile = os.path.join(REPO_DIR, 'requirements.txt') else: outfile = os.path.join(REQ_DIR, 'requirements-{}.txt'.format(name)) if name in [ # Need sip v4 which doesn't work on Python 3.8 'pyqt-5.7', 'pyqt-5.9', 'pyqt-5.10', 'pyqt-5.11', 'pyqt-5.12', # Installs typed_ast on < 3.8 only 'pylint', ]: host_python = 'python3.7' else: host_python = sys.executable utils.print_subtitle("Building") with open(filename, 'r', encoding='utf-8') as f: comments = read_comments(f) with tempfile.TemporaryDirectory() as tmpdir: venv_python = init_venv(host_python=host_python, venv_dir=tmpdir, requirements=filename, pre=comments['pre']) proc = subprocess.run([venv_python, '-m', 'pip', 'freeze'], check=True, stdout=subprocess.PIPE) reqs = proc.stdout.decode('utf-8') with open(outfile, 'w', encoding='utf-8') as f: f.write("# This file is automatically generated by " "scripts/dev/recompile_requirements.py\n\n") for line in reqs.splitlines(): if line.startswith('qutebrowser=='): continue f.write(convert_line(line, comments) + '\n') for line in comments['add']: f.write(line + '\n') # Test resulting file utils.print_subtitle("Testing") with tempfile.TemporaryDirectory() as tmpdir: init_venv(host_python, tmpdir, outfile)
def _package_windows_single( *, nsis_flags: List[str], out_path: pathlib.Path, desc_arch: str, desc_suffix: str, filename_arch: str, debug: bool, ) -> List[Artifact]: """Build the given installer/zip for windows.""" artifacts = [] dist_path = pathlib.Path("dist") utils.print_subtitle(f"Building {desc_arch} installer...") subprocess.run(['makensis.exe', f'/DVERSION={qutebrowser.__version__}', *nsis_flags, 'misc/nsis/qutebrowser.nsi'], check=True) name_parts = [ 'qutebrowser', str(qutebrowser.__version__), filename_arch, ] if debug: name_parts.append('debug') name = '-'.join(name_parts) + '.exe' artifacts.append(Artifact( path=dist_path / name, mimetype='application/vnd.microsoft.portable-executable', description=f'Windows {desc_arch} installer{desc_suffix}', )) utils.print_subtitle(f"Zipping {desc_arch} standalone...") zip_name_parts = [ 'qutebrowser', str(qutebrowser.__version__), 'windows', 'standalone', filename_arch, ] if debug: zip_name_parts.append('debug') zip_name = '-'.join(zip_name_parts) + '.zip' zip_path = dist_path / zip_name shutil.make_archive(str(zip_path.with_suffix('')), 'zip', 'dist', out_path.name) artifacts.append(Artifact( path=zip_path, mimetype='application/zip', description=f'Windows {desc_arch} standalone{desc_suffix}', )) return artifacts
def test_requirements(name, outfile, *, force=False): """Test a resulting requirements file.""" print() utils.print_subtitle("Testing") if name not in _get_changed_files() and not force: print(f"Skipping test as there were no changes for {name}.") return host_python = get_host_python(name) with tempfile.TemporaryDirectory() as tmpdir: init_venv(host_python, tmpdir, outfile)
def main(): parser = argparse.ArgumentParser() parser.add_argument('qt_location', help='Qt compiler directory') parser.add_argument('--wheels-dir', help='Directory to use for wheels', default='wheels') args = parser.parse_args() old_cwd = pathlib.Path.cwd() try: pyqt_bundle = find_pyqt_bundle() except FileNotFoundError as e: utils.print_error(str(e)) sys.exit(1) qt_dir = pathlib.Path(args.qt_location) bin_dir = qt_dir / 'bin' if not bin_dir.exists(): utils.print_error("Can't find {}".format(bin_dir)) sys.exit(1) wheels_dir = pathlib.Path(args.wheels_dir).resolve() wheels_dir.mkdir(exist_ok=True) if list(wheels_dir.glob('*')): utils.print_col( "Wheels directory is not empty, " "unexpected behavior might occur!", 'yellow') os.chdir(wheels_dir) utils.print_title("Downloading wheels") subprocess.run([ sys.executable, '-m', 'pip', 'download', '--no-deps', '--only-binary', 'PyQt5,PyQtWebEngine', 'PyQt5', 'PyQtWebEngine' ], check=True) utils.print_title("Patching wheels") input_files = wheels_dir.glob('*.whl') for wheel in input_files: utils.print_subtitle(wheel.stem.split('-')[0]) subprocess.run([ str(pyqt_bundle), '--qt-dir', args.qt_location, '--ignore-missing', str(wheel) ], check=True) wheel.unlink() print("Done, output files:") for wheel in wheels_dir.glob('*.whl'): print(wheel.relative_to(old_cwd))
def main(): """Main entry point.""" utils.change_cwd() read_files = config.read('.run_checks') if not read_files: raise OSError("Could not read config!") exit_status = collections.OrderedDict() exit_status_bool = {} args = _parse_args() checkers = _get_checkers(args) groups = ['global'] groups += config.get('DEFAULT', 'targets').split(',') groups.append('setup') for group in groups: print() utils.print_title(group) for name, func in checkers[group].items(): if _checker_enabled(args, group, name): utils.print_subtitle(name) status = func() key = '{}_{}'.format(group, name) exit_status[key] = status if name == 'flake8': # pyflakes uses True for errors and False for ok. exit_status_bool[key] = not status elif isinstance(status, bool): exit_status_bool[key] = status else: # sys.exit(0) means no problems -> True, anything != 0 # means problems. exit_status_bool[key] = (status == 0) elif not args.quiet: utils.print_subtitle(name) utils.print_col("Checker disabled.", 'blue') print() utils.print_col("Exit status values:", 'yellow') for (k, v) in exit_status.items(): ok = exit_status_bool[k] color = 'green' if ok else 'red' utils.print_col( ' {} - {} ({})'.format(k, 'ok' if ok else 'FAIL', v), color) if all(exit_status_bool.values()): return 0 else: return 1
def main(): """Main entry point.""" utils.change_cwd() read_files = config.read('.run_checks') if not read_files: raise OSError("Could not read config!") exit_status = collections.OrderedDict() exit_status_bool = {} args = _parse_args() checkers = _get_checkers() groups = ['global'] groups += config.get('DEFAULT', 'targets').split(',') groups.append('setup') for group in groups: print() utils.print_title(group) for name, func in checkers[group].items(): if _checker_enabled(args, group, name): utils.print_subtitle(name) status = func() key = '{}_{}'.format(group, name) exit_status[key] = status if name == 'flake8': # pyflakes uses True for errors and False for ok. exit_status_bool[key] = not status elif isinstance(status, bool): exit_status_bool[key] = status else: # sys.exit(0) means no problems -> True, anything != 0 # means problems. exit_status_bool[key] = (status == 0) elif not args.quiet: utils.print_subtitle(name) utils.print_col("Checker disabled.", 'blue') print() utils.print_col("Exit status values:", 'yellow') for (k, v) in exit_status.items(): ok = exit_status_bool[k] color = 'green' if ok else 'red' utils.print_col( ' {} - {} ({})'.format(k, 'ok' if ok else 'FAIL', v), color) if all(exit_status_bool.values()): return 0 else: return 1
def test_requirements(name, outfile, *, force=False): """Test a resulting requirements file.""" print() utils.print_subtitle("Testing") if name not in _get_changed_files() and not force: print(f"Skipping test as there were no changes for {name}.") return in_file = os.path.join(REQ_DIR, 'requirements-{}.txt-raw'.format(name)) with open(in_file, 'r', encoding='utf-8') as f: comments = read_comments(f) host_python = get_host_python(name) with tempfile.TemporaryDirectory() as tmpdir: init_venv(host_python, tmpdir, outfile, pip_args=comments['pip_args'])
def build_sdist() -> List[Artifact]: """Build an sdist and list the contents.""" utils.print_title("Building sdist") dist_path = pathlib.Path('dist') _maybe_remove(dist_path) subprocess.run([sys.executable, '-m', 'build'], check=True) dist_files = list(dist_path.glob('*.tar.gz')) filename = f'qutebrowser-{qutebrowser.__version__}.tar.gz' assert dist_files == [dist_path / filename], dist_files dist_file = dist_files[0] subprocess.run(['gpg', '--detach-sign', '-a', str(dist_file)], check=True) by_ext = collections.defaultdict(list) with tarfile.open(dist_file) as tar: for tarinfo in tar.getmembers(): if not tarinfo.isfile(): continue path = pathlib.Path(*pathlib.Path(tarinfo.name).parts[1:]) by_ext[path.suffix].append(path) assert '.pyc' not in by_ext utils.print_title("sdist contents") for ext, paths in sorted(by_ext.items()): utils.print_subtitle(ext) print('\n'.join(str(p) for p in paths)) artifacts = [ Artifact( path=dist_file, mimetype='application/gzip', description='Source release', ), Artifact( path=dist_file.with_suffix(dist_file.suffix + '.asc'), mimetype='application/pgp-signature', description='Source release - PGP signature', ), ] return artifacts
def build_sdist(): """Build an sdist and list the contents.""" utils.print_title("Building sdist") dist_path = pathlib.Path('dist') _maybe_remove(dist_path) subprocess.run([sys.executable, '-m', 'build'], check=True) dist_files = list(dist_path.glob('*.tar.gz')) filename = 'qutebrowser-{}.tar.gz'.format(qutebrowser.__version__) assert dist_files == [dist_path / filename], dist_files dist_file = dist_files[0] subprocess.run(['gpg', '--detach-sign', '-a', str(dist_file)], check=True) by_ext = collections.defaultdict(list) with tarfile.open(dist_file) as tar: for tarinfo in tar.getmembers(): if not tarinfo.isfile(): continue name = os.sep.join(tarinfo.name.split(os.sep)[1:]) _base, ext = os.path.splitext(name) by_ext[ext].append(name) assert '.pyc' not in by_ext utils.print_title("sdist contents") for ext, files in sorted(by_ext.items()): utils.print_subtitle(ext) print('\n'.join(files)) artifacts = [ (str(dist_file), 'application/gzip', 'Source release'), ( str(dist_file.with_suffix(dist_file.suffix + '.asc')), 'application/pgp-signature', 'Source release - PGP signature', ), ] return artifacts
def build_sdist(): """Build an sdist and list the contents.""" utils.print_title("Building sdist") _maybe_remove('dist') subprocess.run([sys.executable, 'setup.py', 'sdist'], check=True) dist_files = os.listdir(os.path.abspath('dist')) if len(dist_files) != 1: raise AssertionError dist_file = os.path.join('dist', dist_files[0]) subprocess.run(['gpg', '--detach-sign', '-a', dist_file], check=True) tar = tarfile.open(dist_file) by_ext = collections.defaultdict(list) for tarinfo in tar.getmembers(): if not tarinfo.isfile(): continue name = os.sep.join(tarinfo.name.split(os.sep)[1:]) _base, ext = os.path.splitext(name) by_ext[ext].append(name) if '.pyc' in by_ext: raise AssertionError utils.print_title("sdist contents") for ext, files in sorted(by_ext.items()): utils.print_subtitle(ext) print('\n'.join(files)) filename = 'qutebrowser-{}.tar.gz'.format(qutebrowser.__version__) artifacts = [ (os.path.join('dist', filename), 'application/gzip', 'Source release'), (os.path.join('dist', filename + '.asc'), 'application/pgp-signature', 'Source release - PGP signature'), ] return artifacts
def build_sdist(): """Build an sdist and list the contents.""" utils.print_title("Building sdist") _maybe_remove('dist') subprocess.run([sys.executable, 'setup.py', 'sdist'], check=True) dist_files = os.listdir(os.path.abspath('dist')) assert len(dist_files) == 1 dist_file = os.path.join('dist', dist_files[0]) subprocess.run(['gpg', '--detach-sign', '-a', dist_file], check=True) tar = tarfile.open(dist_file) by_ext = collections.defaultdict(list) for tarinfo in tar.getmembers(): if not tarinfo.isfile(): continue name = os.sep.join(tarinfo.name.split(os.sep)[1:]) _base, ext = os.path.splitext(name) by_ext[ext].append(name) assert '.pyc' not in by_ext utils.print_title("sdist contents") for ext, files in sorted(by_ext.items()): utils.print_subtitle(ext) print('\n'.join(files)) filename = 'qutebrowser-{}.tar.gz'.format(qutebrowser.__version__) artifacts = [ (os.path.join('dist', filename), 'application/gzip', 'Source release'), (os.path.join('dist', filename + '.asc'), 'application/pgp-signature', 'Source release - PGP signature'), ] return artifacts
def main(): parser = argparse.ArgumentParser() parser.add_argument('qt_location', help='Qt compiler directory') parser.add_argument('--wheels-dir', help='Directory to use for wheels', default='wheels') args = parser.parse_args() old_cwd = pathlib.Path.cwd() wheels_dir = pathlib.Path(args.wheels_dir).resolve() wheels_dir.mkdir(exist_ok=True) if list(wheels_dir.glob('*')): utils.print_col("Wheels directory is not empty, " "unexpected behavior might occur!", 'yellow') os.chdir(wheels_dir) utils.print_title("Downloading wheels") subprocess.run([sys.executable, '-m', 'pip', 'download', '--no-deps', '--only-binary', 'PyQt5,PyQtWebEngine', 'PyQt5', 'PyQtWebEngine'], check=True) utils.print_title("Patching wheels") input_files = wheels_dir.glob('*.whl') for wheel in input_files: utils.print_subtitle(wheel.stem.split('-')[0]) bin_path = pathlib.Path(sys.executable).parent subprocess.run([str(bin_path / 'pyqt-bundle'), '--qt-dir', args.qt_location, str(wheel)], check=True) wheel.unlink() print("Done, output files:") for wheel in wheels_dir.glob('*.whl'): print(wheel.relative_to(old_cwd))
def install_dev_packages(): """Install the packages needed for development.""" for pkg in get_dev_packages(): utils.print_subtitle("Installing {}".format(pkg)) venv_python('-m', 'pip', 'install', pkg)