def print_changed_files():
"""Output all changed files from this run."""
diff = git_diff()
if utils.ON_CI:
with utils.gha_group('Raw diff'):
print('\n'.join(diff))
changed_files = _get_changed_files()
files_text = '\n'.join('- ' + line for line in changed_files)
changes = _get_changes(diff)
changes_text = '\n'.join(str(change) for change in changes)
utils.print_subtitle('Files')
print(files_text)
print()
utils.print_subtitle('Changes')
print(changes_text)
if utils.ON_CI:
print()
print('::set-output name=changed::' + files_text.replace('\n', '%0A'))
table_header = [
'| Requirement | old | new |',
'|-------------|-----|-----|',
]
diff_table = '%0A'.join(table_header +
[change.table_str() for change in changes])
print('::set-output name=diff::' + diff_table)
def build_sdist():
"""Build an sdist and list the contents."""
utils.print_title("Building sdist")
_maybe_remove('dist')
subprocess.check_call([sys.executable, 'setup.py', 'sdist'])
dist_files = os.listdir(os.path.abspath('dist'))
assert len(dist_files) == 1
dist_file = os.path.join('dist', dist_files[0])
subprocess.check_call(['gpg', '--detach-sign', '-a', dist_file])
tar = tarfile.open(dist_file)
by_ext = collections.defaultdict(list)
for tarinfo in tar.getmembers():
if not tarinfo.isfile():
continue
name = os.sep.join(tarinfo.name.split(os.sep)[1:])
_base, ext = os.path.splitext(name)
by_ext[ext].append(name)
assert '.pyc' not in by_ext
utils.print_title("sdist contents")
for ext, files in sorted(by_ext.items()):
utils.print_subtitle(ext)
print('\n'.join(files))
def _package_windows_single(
*,
nsis_flags,
outdir,
desc_arch,
desc_suffix,
filename_arch,
):
"""Build the given installer/zip for windows."""
artifacts = []
utils.print_subtitle(f"Building {desc_arch} installer...")
subprocess.run([
'makensis.exe', f'/DVERSION={qutebrowser.__version__}', *nsis_flags,
'misc/nsis/qutebrowser.nsi'
],
check=True)
name = f'qutebrowser-{qutebrowser.__version__}-{filename_arch}.exe'
artifacts.append((
os.path.join('dist', name),
'application/vnd.microsoft.portable-executable',
f'Windows {desc_arch} installer{desc_suffix}',
))
utils.print_subtitle(f"Zipping {desc_arch} standalone...")
zip_name = (
f'qutebrowser-{qutebrowser.__version__}-windows-standalone-{filename_arch}'
)
zip_path = os.path.join('dist', zip_name)
shutil.make_archive(zip_path, 'zip', 'dist', os.path.basename(outdir))
artifacts.append((f'{zip_path}.zip', 'application/zip',
f'Windows {desc_arch} standalone{desc_suffix}'))
return artifacts
def print_changed_files():
"""Output all changed files from this run."""
changed_files = _get_changed_files()
files_text = '\n'.join('- ' + line for line in changed_files)
changes = _get_changes()
diff_text = '\n'.join(str(change) for change in changes)
utils.print_title('Changed')
utils.print_subtitle('Files')
print(files_text)
print()
utils.print_subtitle('Diff')
print(diff_text)
if 'CI' in os.environ:
print()
print('::set-output name=changed::' + files_text.replace('\n', '%0A'))
table_header = [
'| Requirement | old | new |',
'|-------------|-----|-----|',
]
diff_table = '%0A'.join(table_header +
[change.table_str() for change in changes])
print('::set-output name=diff::' + diff_table)
def build_requirements(name):
"""Build a requirements file."""
utils.print_subtitle("Building")
filename = os.path.join(REQ_DIR, 'requirements-{}.txt-raw'.format(name))
host_python = get_host_python(name)
with open(filename, 'r', encoding='utf-8') as f:
comments = read_comments(f)
with tempfile.TemporaryDirectory() as tmpdir:
init_venv(host_python=host_python,
venv_dir=tmpdir,
requirements=filename,
pre=comments['pre'])
with utils.gha_group('Freezing requirements'):
proc = run_pip(tmpdir, 'freeze', stdout=subprocess.PIPE)
reqs = proc.stdout.decode('utf-8')
if utils.ON_CI:
print(reqs.strip())
outfile = get_outfile(name)
with open(outfile, 'w', encoding='utf-8') as f:
f.write("# This file is automatically generated by "
"scripts/dev/recompile_requirements.py\n\n")
for line in reqs.splitlines():
if line.startswith('qutebrowser=='):
continue
f.write(convert_line(line, comments) + '\n')
for line in comments['add']:
f.write(line + '\n')
return outfile
def build_sdist():
"""Build an sdist and list the contents."""
utils.print_title("Building sdist")
_maybe_remove('dist')
subprocess.check_call([sys.executable, 'setup.py', 'sdist'])
dist_files = os.listdir(os.path.abspath('dist'))
assert len(dist_files) == 1
dist_file = os.path.join('dist', dist_files[0])
subprocess.check_call(['gpg', '--detach-sign', '-a', dist_file])
tar = tarfile.open(dist_file)
by_ext = collections.defaultdict(list)
for tarinfo in tar.getmembers():
if not tarinfo.isfile():
continue
name = os.sep.join(tarinfo.name.split(os.sep)[1:])
_base, ext = os.path.splitext(name)
by_ext[ext].append(name)
assert '.pyc' not in by_ext
utils.print_title("sdist contents")
for ext, files in sorted(by_ext.items()):
utils.print_subtitle(ext)
print('\n'.join(files))
def test_tox():
"""Test requirements via tox."""
utils.print_title('Testing via tox')
host_python = get_host_python('tox')
req_path = os.path.join(REQ_DIR, 'requirements-tox.txt')
with tempfile.TemporaryDirectory() as tmpdir:
venv_dir = os.path.join(tmpdir, 'venv')
tox_workdir = os.path.join(tmpdir, 'tox-workdir')
venv_python = os.path.join(venv_dir, 'bin', 'python')
init_venv(host_python, venv_dir, req_path)
list_proc = subprocess.run([venv_python, '-m', 'tox', '--listenvs'],
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
environments = list_proc.stdout.strip().split('\n')
for env in environments:
with utils.gha_group('tox for {}'.format(env)):
utils.print_subtitle(env)
utils.print_col('venv$ tox -e {} --notest'.format(env), 'blue')
subprocess.run([
venv_python, '-m', 'tox', '--workdir', tox_workdir, '-e',
env, '--notest'
],
check=True)
def print_changed_files():
"""Output all changed files from this run."""
changed_files = set()
filenames = git_diff('--name-only')
for filename in filenames:
filename = filename.strip()
filename = filename.replace('misc/requirements/requirements-', '')
filename = filename.replace('.txt', '')
changed_files.add(filename)
files_text = '\n'.join('- ' + line for line in sorted(changed_files))
changes_dict = {}
diff = git_diff()
for line in diff:
if not line.startswith('-') and not line.startswith('+'):
continue
if line.startswith('+++ ') or line.startswith('--- '):
continue
if '==' in line:
name, version = line[1:].split('==')
if ';' in version: # pip environment markers
version = version.split(';')[0].strip()
elif line[1:].startswith('-e'):
rest, name = line.split('#egg=')
version = rest.split('@')[1][:7]
else:
name = line[1:]
version = '?'
if name not in changes_dict:
changes_dict[name] = Change(name)
if line.startswith('-'):
changes_dict[name].old = version
elif line.startswith('+'):
changes_dict[name].new = version
changes = [change for _name, change in sorted(changes_dict.items())]
diff_text = '\n'.join(str(change) for change in changes)
utils.print_title('Changed')
utils.print_subtitle('Files')
print(files_text)
print()
utils.print_subtitle('Diff')
print(diff_text)
if 'CI' in os.environ:
print()
print('::set-output name=changed::' +
files_text.replace('\n', '%0A'))
table_header = [
'| Requirement | old | new |',
'|-------------|-----|-----|',
]
diff_table = '%0A'.join(table_header +
[change.table_str() for change in changes])
print('::set-output name=diff::' + diff_table)
def install_dev_packages():
"""Install the packages needed for development."""
for pkg in get_dev_packages():
utils.print_subtitle("Installing {}".format(pkg))
if os.name == 'nt':
venv_python('-m', 'pip', 'install', '--no-clean', pkg)
else:
venv_python('-m', 'pip', 'install', pkg)
def install_dev_packages():
"""Install the packages needed for development."""
for pkg in get_dev_packages():
utils.print_subtitle("Installing {}".format(pkg))
if os.name == 'nt':
venv_python('-m', 'pip', 'install', '--no-clean', pkg)
else:
venv_python('-m', 'pip', 'install', pkg)
def test_requirements(name, outfile):
"""Test a resulting requirements file."""
print()
utils.print_subtitle("Testing")
host_python = get_host_python(name)
with tempfile.TemporaryDirectory() as tmpdir:
init_venv(host_python, tmpdir, outfile)
def main():
"""Re-compile the given (or all) requirement files."""
names = sys.argv[1:] if len(sys.argv) > 1 else sorted(get_all_names())
for name in names:
utils.print_title(name)
filename = os.path.join(REQ_DIR,
'requirements-{}.txt-raw'.format(name))
if name == 'qutebrowser':
outfile = os.path.join(REPO_DIR, 'requirements.txt')
else:
outfile = os.path.join(REQ_DIR, 'requirements-{}.txt'.format(name))
if name in [
# Need sip v4 which doesn't work on Python 3.8
'pyqt-5.7',
'pyqt-5.9',
'pyqt-5.10',
'pyqt-5.11',
'pyqt-5.12',
# Installs typed_ast on < 3.8 only
'pylint',
]:
host_python = 'python3.7'
else:
host_python = sys.executable
utils.print_subtitle("Building")
with open(filename, 'r', encoding='utf-8') as f:
comments = read_comments(f)
with tempfile.TemporaryDirectory() as tmpdir:
venv_python = init_venv(host_python=host_python,
venv_dir=tmpdir,
requirements=filename,
pre=comments['pre'])
proc = subprocess.run([venv_python, '-m', 'pip', 'freeze'],
check=True,
stdout=subprocess.PIPE)
reqs = proc.stdout.decode('utf-8')
with open(outfile, 'w', encoding='utf-8') as f:
f.write("# This file is automatically generated by "
"scripts/dev/recompile_requirements.py\n\n")
for line in reqs.splitlines():
if line.startswith('qutebrowser=='):
continue
f.write(convert_line(line, comments) + '\n')
for line in comments['add']:
f.write(line + '\n')
# Test resulting file
utils.print_subtitle("Testing")
with tempfile.TemporaryDirectory() as tmpdir:
init_venv(host_python, tmpdir, outfile)
def _package_windows_single(
*,
nsis_flags: List[str],
out_path: pathlib.Path,
desc_arch: str,
desc_suffix: str,
filename_arch: str,
debug: bool,
) -> List[Artifact]:
"""Build the given installer/zip for windows."""
artifacts = []
dist_path = pathlib.Path("dist")
utils.print_subtitle(f"Building {desc_arch} installer...")
subprocess.run(['makensis.exe',
f'/DVERSION={qutebrowser.__version__}', *nsis_flags,
'misc/nsis/qutebrowser.nsi'], check=True)
name_parts = [
'qutebrowser',
str(qutebrowser.__version__),
filename_arch,
]
if debug:
name_parts.append('debug')
name = '-'.join(name_parts) + '.exe'
artifacts.append(Artifact(
path=dist_path / name,
mimetype='application/vnd.microsoft.portable-executable',
description=f'Windows {desc_arch} installer{desc_suffix}',
))
utils.print_subtitle(f"Zipping {desc_arch} standalone...")
zip_name_parts = [
'qutebrowser',
str(qutebrowser.__version__),
'windows',
'standalone',
filename_arch,
]
if debug:
zip_name_parts.append('debug')
zip_name = '-'.join(zip_name_parts) + '.zip'
zip_path = dist_path / zip_name
shutil.make_archive(str(zip_path.with_suffix('')), 'zip', 'dist', out_path.name)
artifacts.append(Artifact(
path=zip_path,
mimetype='application/zip',
description=f'Windows {desc_arch} standalone{desc_suffix}',
))
return artifacts
def test_requirements(name, outfile, *, force=False):
"""Test a resulting requirements file."""
print()
utils.print_subtitle("Testing")
if name not in _get_changed_files() and not force:
print(f"Skipping test as there were no changes for {name}.")
return
host_python = get_host_python(name)
with tempfile.TemporaryDirectory() as tmpdir:
init_venv(host_python, tmpdir, outfile)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('qt_location', help='Qt compiler directory')
parser.add_argument('--wheels-dir',
help='Directory to use for wheels',
default='wheels')
args = parser.parse_args()
old_cwd = pathlib.Path.cwd()
try:
pyqt_bundle = find_pyqt_bundle()
except FileNotFoundError as e:
utils.print_error(str(e))
sys.exit(1)
qt_dir = pathlib.Path(args.qt_location)
bin_dir = qt_dir / 'bin'
if not bin_dir.exists():
utils.print_error("Can't find {}".format(bin_dir))
sys.exit(1)
wheels_dir = pathlib.Path(args.wheels_dir).resolve()
wheels_dir.mkdir(exist_ok=True)
if list(wheels_dir.glob('*')):
utils.print_col(
"Wheels directory is not empty, "
"unexpected behavior might occur!", 'yellow')
os.chdir(wheels_dir)
utils.print_title("Downloading wheels")
subprocess.run([
sys.executable, '-m', 'pip', 'download', '--no-deps', '--only-binary',
'PyQt5,PyQtWebEngine', 'PyQt5', 'PyQtWebEngine'
],
check=True)
utils.print_title("Patching wheels")
input_files = wheels_dir.glob('*.whl')
for wheel in input_files:
utils.print_subtitle(wheel.stem.split('-')[0])
subprocess.run([
str(pyqt_bundle), '--qt-dir', args.qt_location, '--ignore-missing',
str(wheel)
],
check=True)
wheel.unlink()
print("Done, output files:")
for wheel in wheels_dir.glob('*.whl'):
print(wheel.relative_to(old_cwd))
def main():
"""Main entry point."""
utils.change_cwd()
read_files = config.read('.run_checks')
if not read_files:
raise OSError("Could not read config!")
exit_status = collections.OrderedDict()
exit_status_bool = {}
args = _parse_args()
checkers = _get_checkers(args)
groups = ['global']
groups += config.get('DEFAULT', 'targets').split(',')
groups.append('setup')
for group in groups:
print()
utils.print_title(group)
for name, func in checkers[group].items():
if _checker_enabled(args, group, name):
utils.print_subtitle(name)
status = func()
key = '{}_{}'.format(group, name)
exit_status[key] = status
if name == 'flake8':
# pyflakes uses True for errors and False for ok.
exit_status_bool[key] = not status
elif isinstance(status, bool):
exit_status_bool[key] = status
else:
# sys.exit(0) means no problems -> True, anything != 0
# means problems.
exit_status_bool[key] = (status == 0)
elif not args.quiet:
utils.print_subtitle(name)
utils.print_col("Checker disabled.", 'blue')
print()
utils.print_col("Exit status values:", 'yellow')
for (k, v) in exit_status.items():
ok = exit_status_bool[k]
color = 'green' if ok else 'red'
utils.print_col(
' {} - {} ({})'.format(k, 'ok' if ok else 'FAIL', v), color)
if all(exit_status_bool.values()):
return 0
else:
return 1
def main():
"""Main entry point."""
utils.change_cwd()
read_files = config.read('.run_checks')
if not read_files:
raise OSError("Could not read config!")
exit_status = collections.OrderedDict()
exit_status_bool = {}
args = _parse_args()
checkers = _get_checkers()
groups = ['global']
groups += config.get('DEFAULT', 'targets').split(',')
groups.append('setup')
for group in groups:
print()
utils.print_title(group)
for name, func in checkers[group].items():
if _checker_enabled(args, group, name):
utils.print_subtitle(name)
status = func()
key = '{}_{}'.format(group, name)
exit_status[key] = status
if name == 'flake8':
# pyflakes uses True for errors and False for ok.
exit_status_bool[key] = not status
elif isinstance(status, bool):
exit_status_bool[key] = status
else:
# sys.exit(0) means no problems -> True, anything != 0
# means problems.
exit_status_bool[key] = (status == 0)
elif not args.quiet:
utils.print_subtitle(name)
utils.print_col("Checker disabled.", 'blue')
print()
utils.print_col("Exit status values:", 'yellow')
for (k, v) in exit_status.items():
ok = exit_status_bool[k]
color = 'green' if ok else 'red'
utils.print_col(
' {} - {} ({})'.format(k, 'ok' if ok else 'FAIL', v), color)
if all(exit_status_bool.values()):
return 0
else:
return 1
def test_requirements(name, outfile, *, force=False):
"""Test a resulting requirements file."""
print()
utils.print_subtitle("Testing")
if name not in _get_changed_files() and not force:
print(f"Skipping test as there were no changes for {name}.")
return
in_file = os.path.join(REQ_DIR, 'requirements-{}.txt-raw'.format(name))
with open(in_file, 'r', encoding='utf-8') as f:
comments = read_comments(f)
host_python = get_host_python(name)
with tempfile.TemporaryDirectory() as tmpdir:
init_venv(host_python, tmpdir, outfile, pip_args=comments['pip_args'])
def build_sdist() -> List[Artifact]:
"""Build an sdist and list the contents."""
utils.print_title("Building sdist")
dist_path = pathlib.Path('dist')
_maybe_remove(dist_path)
subprocess.run([sys.executable, '-m', 'build'], check=True)
dist_files = list(dist_path.glob('*.tar.gz'))
filename = f'qutebrowser-{qutebrowser.__version__}.tar.gz'
assert dist_files == [dist_path / filename], dist_files
dist_file = dist_files[0]
subprocess.run(['gpg', '--detach-sign', '-a', str(dist_file)], check=True)
by_ext = collections.defaultdict(list)
with tarfile.open(dist_file) as tar:
for tarinfo in tar.getmembers():
if not tarinfo.isfile():
continue
path = pathlib.Path(*pathlib.Path(tarinfo.name).parts[1:])
by_ext[path.suffix].append(path)
assert '.pyc' not in by_ext
utils.print_title("sdist contents")
for ext, paths in sorted(by_ext.items()):
utils.print_subtitle(ext)
print('\n'.join(str(p) for p in paths))
artifacts = [
Artifact(
path=dist_file,
mimetype='application/gzip',
description='Source release',
),
Artifact(
path=dist_file.with_suffix(dist_file.suffix + '.asc'),
mimetype='application/pgp-signature',
description='Source release - PGP signature',
),
]
return artifacts
def build_sdist():
"""Build an sdist and list the contents."""
utils.print_title("Building sdist")
dist_path = pathlib.Path('dist')
_maybe_remove(dist_path)
subprocess.run([sys.executable, '-m', 'build'], check=True)
dist_files = list(dist_path.glob('*.tar.gz'))
filename = 'qutebrowser-{}.tar.gz'.format(qutebrowser.__version__)
assert dist_files == [dist_path / filename], dist_files
dist_file = dist_files[0]
subprocess.run(['gpg', '--detach-sign', '-a', str(dist_file)], check=True)
by_ext = collections.defaultdict(list)
with tarfile.open(dist_file) as tar:
for tarinfo in tar.getmembers():
if not tarinfo.isfile():
continue
name = os.sep.join(tarinfo.name.split(os.sep)[1:])
_base, ext = os.path.splitext(name)
by_ext[ext].append(name)
assert '.pyc' not in by_ext
utils.print_title("sdist contents")
for ext, files in sorted(by_ext.items()):
utils.print_subtitle(ext)
print('\n'.join(files))
artifacts = [
(str(dist_file), 'application/gzip', 'Source release'),
(
str(dist_file.with_suffix(dist_file.suffix + '.asc')),
'application/pgp-signature',
'Source release - PGP signature',
),
]
return artifacts
def build_sdist():
"""Build an sdist and list the contents."""
utils.print_title("Building sdist")
_maybe_remove('dist')
subprocess.run([sys.executable, 'setup.py', 'sdist'], check=True)
dist_files = os.listdir(os.path.abspath('dist'))
if len(dist_files) != 1:
raise AssertionError
dist_file = os.path.join('dist', dist_files[0])
subprocess.run(['gpg', '--detach-sign', '-a', dist_file], check=True)
tar = tarfile.open(dist_file)
by_ext = collections.defaultdict(list)
for tarinfo in tar.getmembers():
if not tarinfo.isfile():
continue
name = os.sep.join(tarinfo.name.split(os.sep)[1:])
_base, ext = os.path.splitext(name)
by_ext[ext].append(name)
if '.pyc' in by_ext:
raise AssertionError
utils.print_title("sdist contents")
for ext, files in sorted(by_ext.items()):
utils.print_subtitle(ext)
print('\n'.join(files))
filename = 'qutebrowser-{}.tar.gz'.format(qutebrowser.__version__)
artifacts = [
(os.path.join('dist', filename), 'application/gzip', 'Source release'),
(os.path.join('dist', filename + '.asc'), 'application/pgp-signature',
'Source release - PGP signature'),
]
return artifacts
def build_sdist():
"""Build an sdist and list the contents."""
utils.print_title("Building sdist")
_maybe_remove('dist')
subprocess.run([sys.executable, 'setup.py', 'sdist'], check=True)
dist_files = os.listdir(os.path.abspath('dist'))
assert len(dist_files) == 1
dist_file = os.path.join('dist', dist_files[0])
subprocess.run(['gpg', '--detach-sign', '-a', dist_file], check=True)
tar = tarfile.open(dist_file)
by_ext = collections.defaultdict(list)
for tarinfo in tar.getmembers():
if not tarinfo.isfile():
continue
name = os.sep.join(tarinfo.name.split(os.sep)[1:])
_base, ext = os.path.splitext(name)
by_ext[ext].append(name)
assert '.pyc' not in by_ext
utils.print_title("sdist contents")
for ext, files in sorted(by_ext.items()):
utils.print_subtitle(ext)
print('\n'.join(files))
filename = 'qutebrowser-{}.tar.gz'.format(qutebrowser.__version__)
artifacts = [
(os.path.join('dist', filename), 'application/gzip', 'Source release'),
(os.path.join('dist', filename + '.asc'), 'application/pgp-signature',
'Source release - PGP signature'),
]
return artifacts
def main():
parser = argparse.ArgumentParser()
parser.add_argument('qt_location', help='Qt compiler directory')
parser.add_argument('--wheels-dir', help='Directory to use for wheels',
default='wheels')
args = parser.parse_args()
old_cwd = pathlib.Path.cwd()
wheels_dir = pathlib.Path(args.wheels_dir).resolve()
wheels_dir.mkdir(exist_ok=True)
if list(wheels_dir.glob('*')):
utils.print_col("Wheels directory is not empty, "
"unexpected behavior might occur!", 'yellow')
os.chdir(wheels_dir)
utils.print_title("Downloading wheels")
subprocess.run([sys.executable, '-m', 'pip', 'download',
'--no-deps', '--only-binary', 'PyQt5,PyQtWebEngine',
'PyQt5', 'PyQtWebEngine'], check=True)
utils.print_title("Patching wheels")
input_files = wheels_dir.glob('*.whl')
for wheel in input_files:
utils.print_subtitle(wheel.stem.split('-')[0])
bin_path = pathlib.Path(sys.executable).parent
subprocess.run([str(bin_path / 'pyqt-bundle'),
'--qt-dir', args.qt_location, str(wheel)],
check=True)
wheel.unlink()
print("Done, output files:")
for wheel in wheels_dir.glob('*.whl'):
print(wheel.relative_to(old_cwd))
def install_dev_packages():
"""Install the packages needed for development."""
for pkg in get_dev_packages():
utils.print_subtitle("Installing {}".format(pkg))
venv_python('-m', 'pip', 'install', pkg)
