def main():
args = parse_args()
if args.name is None:
utils.print_error('Please supply parameter name.')
exit(1)
if (args.action == 'list'):
params = list_params(args.name, args.region)
if args.verbose:
print_params_verbose(params)
else:
print_params_simple(params)
elif (args.action == 'get'):
print(
get_param(
args.name, args.region,
decrypt=(not args.plaintext)).get('Parameter').get('Value'))
elif (args.action == 'put'):
if args.value is None:
utils.print_error('Please supply parameter value.')
sys.exit(1)
put_param(args.name,
args.value,
args.region,
args.kms_key_alias,
overwrite=args.force,
plaintext=args.plaintext)
elif (args.action == 'delete'):
delete_param(args.name, args.region)
def call(self, src: pathlib.Path, dst: pathlib.Path, *args):
"""Call asciidoc for the given files.
Args:
src: The source .asciidoc file.
dst: The destination .html file, or None to auto-guess.
*args: Additional arguments passed to asciidoc.
"""
print("Calling asciidoc for {}...".format(src.name))
assert self._cmd is not None # for mypy
cmdline = self._cmd[:]
if dst is not None:
cmdline += ['--out-file', str(dst)]
cmdline += args
cmdline.append(str(src))
try:
env = os.environ.copy()
env['HOME'] = str(self._homedir)
subprocess.run(cmdline, check=True, env=env)
except (subprocess.CalledProcessError, OSError) as e:
self._failed = True
utils.print_error(str(e))
print("Keeping modified sources in {}.".format(self._homedir),
file=sys.stderr)
sys.exit(1)
def main():
args = parse_args()
try:
run(args)
except Error as e:
utils.print_error(str(e))
sys.exit(e.code)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--no-asciidoc',
action='store_true',
help="Don't generate docs")
parser.add_argument('--asciidoc',
help="Full path to python and "
"asciidoc.py. If not given, it's searched in PATH.",
nargs=2,
required=False,
metavar=('PYTHON', 'ASCIIDOC'))
parser.add_argument('--upload',
action='store_true',
required=False,
help="Toggle to upload the release to GitHub")
args = parser.parse_args()
utils.change_cwd()
upload_to_pypi = False
if args.upload:
# Fail early when trying to upload without github3 installed
# or without API token
import github3 # pylint: disable=unused-import
read_github_token()
if not misc_checks.check_git():
utils.print_error("Refusing to do a release with a dirty git tree")
sys.exit(1)
if args.no_asciidoc:
os.makedirs(os.path.join('qutebrowser', 'html', 'doc'), exist_ok=True)
else:
run_asciidoc2html(args)
if os.name == 'nt':
artifacts = build_windows()
elif sys.platform == 'darwin':
artifacts = build_mac()
else:
upgrade_sdist_dependencies()
test_makefile()
artifacts = build_sdist()
upload_to_pypi = True
if args.upload:
version_tag = "v" + qutebrowser.__version__
utils.print_title("Press enter to release {}...".format(version_tag))
input()
github_upload(artifacts, version_tag)
if upload_to_pypi:
pypi_upload(artifacts)
else:
print()
utils.print_title("Artifacts")
for artifact in artifacts:
print(artifact)
def apply_xcb_util_workaround(
venv_dir: pathlib.Path,
pyqt_type: str,
pyqt_version: str,
) -> None:
"""If needed (Debian Stable), symlink libxcb-util.so.0 -> .1.
WORKAROUND for https://bugreports.qt.io/browse/QTBUG-88688
"""
utils.print_title("Running xcb-util workaround")
if not sys.platform.startswith('linux'):
print("Workaround not needed: Not on Linux.")
return
if pyqt_type != 'binary':
print("Workaround not needed: Not installing from PyQt binaries.")
return
if pyqt_version not in ['auto', '5.15']:
print("Workaround not needed: Not installing Qt 5.15.")
return
libs = _find_libs()
abi_type = 'libc6,x86-64' # the only one PyQt wheels are available for
if ('libxcb-util.so.1', abi_type) in libs:
print("Workaround not needed: libxcb-util.so.1 found.")
return
try:
libxcb_util_libs = libs['libxcb-util.so.0', abi_type]
except KeyError:
utils.print_error('Workaround failed: libxcb-util.so.0 not found.')
return
if len(libxcb_util_libs) > 1:
utils.print_error(
f'Workaround failed: Multiple matching libxcb-util found: '
f'{libxcb_util_libs}')
return
libxcb_util_path = pathlib.Path(libxcb_util_libs[0])
code = [
'from PyQt5.QtCore import QLibraryInfo',
'print(QLibraryInfo.location(QLibraryInfo.LibrariesPath))',
]
proc = run_venv(venv_dir, 'python', '-c', '; '.join(code), capture_output=True)
venv_lib_path = pathlib.Path(proc.stdout.strip())
link_path = venv_lib_path / libxcb_util_path.with_suffix('.1').name
# This gives us a nicer path to print, and also conveniently makes sure we
# didn't accidentally end up with a path outside the venv.
rel_link_path = venv_dir / link_path.relative_to(venv_dir.resolve())
print_command('ln -s', libxcb_util_path, rel_link_path, venv=False)
link_path.symlink_to(libxcb_util_path)
def delete_param(name, region):
"""Remove SSM parameter."""
ssm = boto3.client('ssm', region)
try:
utils.print_info(json.dumps(ssm.delete_parameter(Name=name)))
except botocore.exceptions.ClientError as e:
if (e.response['Error']['Code'] == 'ParameterNotFound'):
utils.print_error(f'Cannot find {name}')
sys.exit(1)
raise e
def get_param(name, region, decrypt=True):
"""Retrieve parameter."""
ssm = boto3.client('ssm', region)
try:
return ssm.get_parameter(Name=name, WithDecryption=decrypt)
except botocore.exceptions.ClientError as e:
if (e.response['Error']['Code'] == 'ParameterNotFound'):
utils.print_error(f'Cannot find {name}')
sys.exit(1)
raise e
def github_upload(artifacts: List[Artifact], tag: str, gh_token: str) -> None:
"""Upload the given artifacts to GitHub.
Args:
artifacts: A list of Artifacts to upload.
tag: The name of the release tag
gh_token: The GitHub token to use
"""
import github3
import github3.exceptions
utils.print_title("Uploading to github...")
gh = github3.login(token=gh_token)
repo = gh.repository('qutebrowser', 'qutebrowser')
release = None # to satisfy pylint
for release in repo.releases():
if release.tag_name == tag:
break
else:
raise Exception(f"No release found for {tag!r}!")
for artifact in artifacts:
while True:
print(f"Uploading {artifact.path}")
assets = [asset for asset in release.assets()
if asset.name == artifact.path.name]
if assets:
print(f"Assets already exist: {assets}")
print("Press enter to continue anyways or Ctrl-C to abort.")
input()
try:
with artifact.path.open('rb') as f:
release.upload_asset(
artifact.mimetype,
artifact.path.name,
f,
artifact.description,
)
except github3.exceptions.ConnectionError as e:
utils.print_error(f'Failed to upload: {e}')
print("Press Enter to retry...", file=sys.stderr)
input()
print("Retrying!")
assets = [asset for asset in release.assets()
if asset.name == artifact.path.name]
if assets:
stray_asset = assets[0]
print(f"Deleting stray asset {stray_asset.name}")
stray_asset.delete()
else:
break
def run_venv(venv_dir: pathlib.Path, executable, *args: str) -> None:
"""Run the given command inside the virtualenv."""
subdir = 'Scripts' if os.name == 'nt' else 'bin'
try:
subprocess.run([str(venv_dir / subdir / executable)] +
[str(arg) for arg in args],
check=True)
except subprocess.CalledProcessError as e:
utils.print_error("Subprocess failed, exiting")
sys.exit(e.returncode)
def github_upload(artifacts, tag, gh_token):
"""Upload the given artifacts to GitHub.
Args:
artifacts: A list of (filename, mimetype, description) tuples
tag: The name of the release tag
gh_token: The GitHub token to use
"""
import github3
import github3.exceptions
utils.print_title("Uploading to github...")
gh = github3.login(token=gh_token)
repo = gh.repository('qutebrowser', 'qutebrowser')
release = None # to satisfy pylint
for release in repo.releases():
if release.tag_name == tag:
break
else:
raise Exception("No release found for {!r}!".format(tag))
for filename, mimetype, description in artifacts:
while True:
print("Uploading {}".format(filename))
basename = os.path.basename(filename)
assets = [
asset for asset in release.assets() if asset.name == basename
]
if assets:
print("Assets already exist: {}".format(assets))
print("Press enter to continue anyways or Ctrl-C to abort.")
input()
try:
with open(filename, 'rb') as f:
release.upload_asset(mimetype, basename, f, description)
except github3.exceptions.ConnectionError as e:
utils.print_error('Failed to upload: {}'.format(e))
print("Press Enter to retry...", file=sys.stderr)
input()
print("Retrying!")
assets = [
asset for asset in release.assets()
if asset.name == basename
]
if assets:
stray_asset = assets[0]
print("Deleting stray asset {}".format(stray_asset.name))
stray_asset.delete()
else:
break
def main():
parser = argparse.ArgumentParser()
parser.add_argument('qt_location', help='Qt compiler directory')
parser.add_argument('--wheels-dir',
help='Directory to use for wheels',
default='wheels')
args = parser.parse_args()
old_cwd = pathlib.Path.cwd()
try:
pyqt_bundle = find_pyqt_bundle()
except FileNotFoundError as e:
utils.print_error(str(e))
sys.exit(1)
qt_dir = pathlib.Path(args.qt_location)
bin_dir = qt_dir / 'bin'
if not bin_dir.exists():
utils.print_error("Can't find {}".format(bin_dir))
sys.exit(1)
wheels_dir = pathlib.Path(args.wheels_dir).resolve()
wheels_dir.mkdir(exist_ok=True)
if list(wheels_dir.glob('*')):
utils.print_col(
"Wheels directory is not empty, "
"unexpected behavior might occur!", 'yellow')
os.chdir(wheels_dir)
utils.print_title("Downloading wheels")
subprocess.run([
sys.executable, '-m', 'pip', 'download', '--no-deps', '--only-binary',
'PyQt5,PyQtWebEngine', 'PyQt5', 'PyQtWebEngine'
],
check=True)
utils.print_title("Patching wheels")
input_files = wheels_dir.glob('*.whl')
for wheel in input_files:
utils.print_subtitle(wheel.stem.split('-')[0])
subprocess.run([
str(pyqt_bundle), '--qt-dir', args.qt_location, '--ignore-missing',
str(wheel)
],
check=True)
wheel.unlink()
print("Done, output files:")
for wheel in wheels_dir.glob('*.whl'):
print(wheel.relative_to(old_cwd))
def create_venv(venv_dir: pathlib.Path, use_virtualenv: bool = False) -> None:
"""Create a new virtualenv."""
if use_virtualenv:
utils.print_col('$ python3 -m virtualenv {}'.format(venv_dir), 'blue')
try:
subprocess.run([sys.executable, '-m', 'virtualenv', venv_dir],
check=True)
except subprocess.CalledProcessError as e:
utils.print_error("virtualenv failed, exiting")
sys.exit(e.returncode)
else:
utils.print_col('$ python3 -m venv {}'.format(venv_dir), 'blue')
venv.create(str(venv_dir), with_pip=True)
def install(languages):
"""Install languages."""
for lang in languages:
try:
print('Installing {}: {}'.format(lang.code, lang.name))
install_lang(lang)
except PermissionError as e:
msg = ("\n{}\n\nWith Qt < 5.10, you will need to run this script "
"as root, as dictionaries need to be installed "
"system-wide. If your qutebrowser uses a newer Qt version "
"via a virtualenv, make sure you start this script with "
"the virtualenv's Python.".format(e))
scriptutils.print_error(msg)
sys.exit(1)
def show_tox_error(pyqt_type: str) -> None:
"""DIsplay an error when invoked from tox."""
if pyqt_type == 'link':
env = 'mkvenv'
args = ' --pyqt-type link'
elif pyqt_type == 'binary':
env = 'mkvenv-pypi'
args = ''
else:
raise AssertionError
print()
utils.print_error(
'tox -e {} is deprecated. '
'Please use "python3 scripts/mkvenv.py{}" instead.'.format(env, args))
print()
def main() -> None:
"""Install qutebrowser in a virtualenv.."""
args = parse_args()
venv_dir = pathlib.Path(args.venv_dir)
wheels_dir = pathlib.Path(args.pyqt_wheels_dir)
utils.change_cwd()
if args.tox_error:
show_tox_error(args.pyqt_type)
sys.exit(1)
elif (args.pyqt_version != 'auto'
and args.pyqt_type not in ['binary', 'source']):
utils.print_error('The --pyqt-version option is only available when '
'installing PyQt from binary or source')
sys.exit(1)
elif args.pyqt_wheels_dir != 'wheels' and args.pyqt_type != 'wheels':
utils.print_error('The --pyqt-wheels-dir option is only available '
'when installing PyQt from wheels')
sys.exit(1)
if not args.keep:
utils.print_title("Creating virtual environment")
delete_old_venv(venv_dir)
create_venv(venv_dir, use_virtualenv=args.virtualenv)
upgrade_seed_pkgs(venv_dir)
if args.pyqt_type == 'binary':
install_pyqt_binary(venv_dir, args.pyqt_version)
elif args.pyqt_type == 'source':
install_pyqt_source(venv_dir, args.pyqt_version)
elif args.pyqt_type == 'link':
install_pyqt_link(venv_dir)
elif args.pyqt_type == 'wheels':
install_pyqt_wheels(venv_dir, wheels_dir)
elif args.pyqt_type == 'skip':
pass
else:
raise AssertionError
install_requirements(venv_dir)
install_qutebrowser(venv_dir)
if args.dev:
install_dev_requirements(venv_dir)
if not args.skip_docs:
regenerate_docs(venv_dir, args.asciidoc)
def run(**kwargs) -> None:
"""Regenerate documentation."""
DOC_DIR.mkdir(exist_ok=True)
asciidoc = AsciiDoc(**kwargs)
try:
asciidoc.prepare()
except FileNotFoundError:
utils.print_error("Could not find asciidoc! Please install it, or use "
"the --asciidoc argument to point this script to "
"the correct python/asciidoc.py location!")
sys.exit(1)
try:
asciidoc.build()
finally:
asciidoc.cleanup()
def delete_old_venv(venv_dir: pathlib.Path) -> None:
"""Remove an existing virtualenv directory."""
if not venv_dir.exists():
return
markers = [
venv_dir / '.tox-config1', # tox
venv_dir / 'pyvenv.cfg', # venv
venv_dir / 'Scripts', # Windows
venv_dir / 'bin', # Linux
]
if not any(m.exists() for m in markers):
utils.print_error('{} does not look like a virtualenv, '
'cowardly refusing to remove it.'.format(venv_dir))
sys.exit(1)
utils.print_col('$ rm -r {}'.format(venv_dir), 'blue')
shutil.rmtree(str(venv_dir))
def run(**kwargs):
"""Regenerate documentation."""
try:
os.mkdir('qutebrowser/html/doc')
except FileExistsError:
pass
asciidoc = AsciiDoc(**kwargs)
try:
asciidoc.prepare()
except FileNotFoundError:
utils.print_error("Could not find asciidoc! Please install it, or use "
"the --asciidoc argument to point this script to "
"the correct python/asciidoc.py location!")
sys.exit(1)
try:
asciidoc.build()
finally:
asciidoc.cleanup()
def install_pyqt_binary(venv_dir: pathlib.Path, version: str) -> None:
"""Install PyQt from a binary wheel."""
utils.print_title("Installing PyQt from binary")
utils.print_col(
"No proprietary codec support will be available in "
"qutebrowser.", 'bold')
supported_archs = {
'linux': {'x86_64'},
'win32': {'x86', 'AMD64'},
'darwin': {'x86_64'},
}
if sys.platform not in supported_archs:
utils.print_error(
f"{sys.platform} is not a supported platform by PyQt5 binary "
"packages, this will most likely fail.")
elif platform.machine() not in supported_archs[sys.platform]:
utils.print_error(
f"{platform.machine()} is not a supported architecture for PyQt5 binaries "
f"on {sys.platform}, this will most likely fail.")
elif sys.platform == 'linux' and platform.libc_ver()[0] != 'glibc':
utils.print_error(
"Non-glibc Linux is not a supported platform for PyQt5 "
"binaries, this will most likely fail.")
pip_install(venv_dir, '-r', pyqt_requirements_file(version),
'--only-binary', 'PyQt5,PyQtWebEngine')
def put_param(name,
value,
region,
kms_key_alias=None,
overwrite=False,
plaintext=True):
"""Store the name and value"""
ssm = boto3.client('ssm', region)
try:
if kms_key_alias:
kms_key = kms.get_kms_key_id(kms_key_alias, region)
if not kms_key:
raise ParamException(
f'No key found for alias {kms_key_alias} {region}')
result = ssm.put_parameter(Name=name,
Description=name,
Value=value,
Type='SecureString',
KeyId=kms_key,
Overwrite=overwrite)
else:
utils.print_warning('Creating without encryption')
result = ssm.put_parameter(Name=name,
Description=name,
Value=value,
Type='String',
Overwrite=overwrite)
utils.print_info(json.dumps(result))
except botocore.exceptions.ClientError as e:
if (e.response['Error']['Code'] == 'ParameterAlreadyExists'):
utils.print_error(
f'setting "{name}" already exists, use -f to overwrite.')
sys.exit(1)
raise e
def main():
args = parse_args()
if args.data:
if len(args.data) < 8:
utils.print_error('Secrets should be 8 characters or greater')
sys.exit(1)
if (args.action == 'encrypt'):
if not args.alias:
utils.print_error('You must provide --alias to encrypt')
sys.exit(1)
print(
encrypt(args.data, args.alias, json.loads(args.context),
args.region))
elif (args.action == 'decrypt'):
print(decrypt(args.data, json.loads(args.context), args.region))
else:
utils.print_error('Commands are encrypt/decrypt')
sys.exit(1)
def rolling_replace_instances(ecs, ec2, cluster_name, batches, ami_id, force, drain_timeout_s):
replace_start_time = time.time()
services = get_services(ecs, cluster_name)
if not services:
raise RollingException('No services found in cluster. exiting.')
utils.print_info(
f'Checking cluster {cluster_name}, services {str(services)} are stable'
)
ecs_utils.poll_cluster_state(
ecs, cluster_name, services, polling_timeout=120
)
instances = get_container_instance_arns(ecs, cluster_name)
# batches determines the number of instances you want to replace at once.
# Choose conservatively, as this process temporarily reduces your capacity.
# But note each batch can be time consuming (up to 10m per batch)
batch_count = math.ceil(len(instances) / batches)
utils.print_info(f'You have {len(instances)} instances.')
utils.print_info(f'Terminating in batches of {batch_count}')
if len(instances) <= batch_count:
utils.print_warning(
f'Terminating {batch_count} instances will cause downtime.'
)
if not force:
raise RollingException('Quitting, use --force to over-ride.')
instance_batches = batch_instances(instances, batch_count)
for to_drain in instance_batches:
if len(to_drain) > 100:
utils.print_error('Batch size exceeded 100, try using more batches.')
raise RollingException(
f'Quitting, batch size exceeded 100: {batch_count}.'
)
response = ecs.describe_container_instances(
cluster=cluster_name, containerInstances=to_drain)
if not response.get('containerInstances'):
raise RollingException('No containerInstances found.')
# don't drain or teriminate any instances that are already up to date
# (if the user provided the --ami-id flag)
done_instances = get_already_updated_instances(response, ami_id)
if len(done_instances) == len(to_drain):
# move on if the whole batch is already up to date
continue
# drain instances in this batch
ecs.update_container_instances_state(cluster=cluster_name,
status='DRAINING',
containerInstances=to_drain)
utils.print_info(f'Wait for drain to complete with {drain_timeout_s}s timeout...')
start_time = time.time()
while len(done_instances) < len(to_drain):
if (time.time() - start_time) > drain_timeout_s:
raise RollingTimeoutException('Waiting for instance to complete draining. Giving up.')
time.sleep(SLEEP_TIME_S)
response = ecs.describe_container_instances(
cluster=cluster_name, containerInstances=to_drain)
for container_instance in response.get('containerInstances'):
instance_id = container_instance.get('ec2InstanceId')
running_tasks = container_instance.get('runningTasksCount')
if running_tasks > 0:
PRINT_PROGRESS()
continue
if instance_id not in done_instances:
utils.print_info(f'{instance_id} is drained, terminate!')
ec2.terminate_instances(InstanceIds=[instance_id])
done_instances.append(instance_id)
# new instance will take as much as 10m to go into service
# then we wait for ECS to resume a steady state before moving on
ecs_utils.poll_cluster_state(ecs, cluster_name,
services, polling_timeout=drain_timeout_s)
utils.print_success(f'EC2 instance replacement process complete! {int(time.time() - replace_start_time)}s elapsed')
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--skip-docs',
action='store_true',
help="Don't generate docs")
parser.add_argument('--asciidoc',
help="Full path to asciidoc.py. "
"If not given, it's searched in PATH.",
nargs='?')
parser.add_argument(
'--asciidoc-python',
help="Python to use for asciidoc."
"If not given, the current Python interpreter is used.",
nargs='?')
parser.add_argument('--gh-token', help="GitHub token to use.", nargs='?')
parser.add_argument('--upload',
action='store_true',
required=False,
help="Toggle to upload the release to GitHub.")
parser.add_argument('--no-confirm',
action='store_true',
required=False,
help="Skip confirmation before uploading.")
parser.add_argument('--skip-packaging',
action='store_true',
required=False,
help="Skip Windows installer/zip generation.")
parser.add_argument('--32bit',
action='store_true',
required=False,
help="Skip Windows 64 bit build.",
dest='only_32bit')
parser.add_argument('--64bit',
action='store_true',
required=False,
help="Skip Windows 32 bit build.",
dest='only_64bit')
parser.add_argument('--debug',
action='store_true',
required=False,
help="Build a debug build.")
args = parser.parse_args()
utils.change_cwd()
upload_to_pypi = False
if args.upload:
# Fail early when trying to upload without github3 installed
# or without API token
import github3 # pylint: disable=unused-import
gh_token = read_github_token(args.gh_token)
else:
gh_token = read_github_token(args.gh_token, optional=True)
if not misc_checks.check_git():
utils.print_error("Refusing to do a release with a dirty git tree")
sys.exit(1)
if args.skip_docs:
os.makedirs(os.path.join('qutebrowser', 'html', 'doc'), exist_ok=True)
else:
run_asciidoc2html(args)
if os.name == 'nt':
artifacts = build_windows(
gh_token=gh_token,
skip_packaging=args.skip_packaging,
only_32bit=args.only_32bit,
only_64bit=args.only_64bit,
debug=args.debug,
)
elif sys.platform == 'darwin':
artifacts = build_mac(gh_token=gh_token, debug=args.debug)
else:
test_makefile()
artifacts = build_sdist()
twine_check(artifacts)
upload_to_pypi = True
if args.upload:
version_tag = "v" + qutebrowser.__version__
if not args.no_confirm:
utils.print_title(
"Press enter to release {}...".format(version_tag))
input()
github_upload(artifacts, version_tag, gh_token=gh_token)
if upload_to_pypi:
pypi_upload(artifacts)
else:
print()
utils.print_title("Artifacts")
for artifact in artifacts:
print(artifact)
