def _global_repo_path(log):
temp_dir = tempfile.mkdtemp()
target_repo_path = os.path.join(temp_dir, 'pyci')
log.info('Copying source directory to {}...'.format(target_repo_path))
test_utils.copy_repo(target_repo_path)
log.info(
'Finished copying source directory to: {}'.format(target_repo_path))
try:
yield target_repo_path
finally:
try:
utils.rmf(target_repo_path)
except BaseException as e:
if utils.is_windows():
# The temp_dir was populated with files written by a different process (pip install)
# On windows, this causes a [Error 5] Access is denied error.
# Eventually I will have to fix this - until then, sorry windows users...
log.debug(
"Failed deleting temporary repo directory {}: {} - "
"You might have some leftovers because of this...".format(
target_repo_path, str(e)))
else:
raise
def _extract_project_name(wheel):
wheel = os.path.abspath(wheel)
wheel_parts = os.path.basename(wheel).split('-')
temp_dir = tempfile.mkdtemp()
try:
project_name = None
unpack.unpack(path=wheel, dest=temp_dir)
wheel_project = '{}-{}'.format(wheel_parts[0], wheel_parts[1])
metadata_file_path = os.path.join(
temp_dir, wheel_project, '{}.dist-info'.format(wheel_project),
'METADATA')
with open(metadata_file_path) as stream:
for line in stream.read().splitlines():
if line.startswith('Name: '):
project_name = line.split('Name: ')[1]
break
return project_name
finally:
utils.rmf(temp_dir)
def _temp_dir(request, log):
name = request.node.originalname or request.node.name
dir_path = tempfile.mkdtemp(suffix=name)
try:
yield dir_path
finally:
try:
utils.rmf(dir_path)
except BaseException as e:
if utils.is_windows():
# The temp_dir was populated with files written by a different process (pip install)
# On windows, this causes a [Error 5] Access is denied error.
# Eventually I will have to fix this - until then, sorry windows users...
log.debug(
"Failed cleaning up temporary test directory {}: {} - "
"You might have some leftovers because of this...".format(
dir_path, str(e)))
else:
raise
def release(ctx, repo, branch, master_branch, release_branch, pypi_test,
pypi_url, binary_entrypoint, binary_base_name, wheel_universal,
no_binary, no_wheel, pyinstaller_version, wheel_version,
changelog_base, version, no_wheel_publish, no_installer, force,
author, url, copyright_string, license_path, program_files_dir):
"""
Execute a complete release process.
1. Execute a github release on the specified branch. (see 'pyci github release --help')
2. Create and upload a platform dependent binary executable to the release.
3. Create and upload a platform (and distro) dependent installer to the release.
4. Create and upload a wheel to the release.
5. Publish the wheel on PyPI.
Much of this process is configurable via the command options. For example you can choose not to publish
the wheel to PyPI by specifying the '--no-wheel-publish` flag.
Currently only windows installers are created, if the command is executed from some other platform, the installer
creation is silently ignored. Adding support for all platforms and distros is in thw works.
"""
ci_provider = ctx.obj.ci_provider
branch = branch or (ci_provider.branch if ci_provider else None)
if not branch:
raise click.BadOptionUsage(
option_name='branch',
message='Must provide --branch when running outside CI')
# No other way unfortunately, importing it normally would cause
# an actual runtime cyclic-import problem.
# pylint: disable=cyclic-import
from pyci.shell import main
ctx.invoke(main.github, repo=repo)
github_release = ctx.invoke(github.release_,
version=version,
branch=branch,
master_branch=master_branch,
release_branch=release_branch,
changelog_base=changelog_base,
force=force)
if github_release is None:
# This is our way of knowing that github.release_
# decided this commit shouldn't be silently ignored, not released.
return
ctx.invoke(main.pack, repo=repo, sha=github_release.sha)
package_directory = tempfile.mkdtemp()
wheel_url = None
try:
binary_path = None
wheel_path = None
installer_path = None
log.echo('Creating packages', add=True)
if not no_binary:
binary_path = _pack_binary(ctx=ctx,
base_name=binary_base_name,
entrypoint=binary_entrypoint,
pyinstaller_version=pyinstaller_version)
if not no_wheel:
wheel_path = _pack_wheel(ctx=ctx,
wheel_universal=wheel_universal,
wheel_version=wheel_version)
if not no_installer:
installer_path = _pack_installer(
ctx=ctx,
binary_path=binary_path,
author=author,
version=version,
license_path=license_path,
copyright_string=copyright_string,
url=url,
program_files_dir=program_files_dir)
log.sub()
log.echo('Uploading packages', add=True)
if binary_path:
_upload_asset(ctx=ctx,
asset_path=binary_path,
github_release=github_release)
if installer_path:
_upload_asset(ctx=ctx,
asset_path=installer_path,
github_release=github_release)
if wheel_path:
_upload_asset(ctx=ctx,
asset_path=wheel_path,
github_release=github_release)
if not no_wheel_publish:
ctx.invoke(main.pypi, test=pypi_test, repository_url=pypi_url)
_upload_pypi(ctx=ctx, wheel_path=wheel_path)
log.sub()
finally:
try:
utils.rmf(package_directory)
except BaseException as e:
log.warn('Failed cleaning up packager directory ({}): {}'.format(
package_directory, str(e)))
log.echo(
'Hip Hip, Hurray! :). Your new version is released and ready to go.',
add=True)
log.echo('Github: {}'.format(github_release.url))
if wheel_url:
log.echo('PyPI: {}'.format(wheel_url))
def _create_virtualenv(self, name):
temp_dir = tempfile.mkdtemp()
virtualenv_path = os.path.join(temp_dir, name)
self._debug('Creating virtualenv {}'.format(virtualenv_path))
def _create_virtualenv_dist():
dist_directory = os.path.join(temp_dir, 'virtualenv-dist')
support_directory = os.path.join(dist_directory,
'virtualenv_support')
os.makedirs(dist_directory)
os.makedirs(support_directory)
_virtualenv_py = os.path.join(dist_directory, 'virtualenv.py')
def _write_support_wheel(_wheel):
with open(os.path.join(support_directory, _wheel), 'wb') as _w:
_w.write(
get_binary_resource(
os.path.join('virtualenv_support', _wheel)))
with open(_virtualenv_py, 'w') as venv_py:
venv_py.write(get_text_resource('virtualenv.py'))
_write_support_wheel('pip-19.1.1-py2.py3-none-any.whl')
_write_support_wheel('setuptools-41.0.1-py2.py3-none-any.whl')
return _virtualenv_py
virtualenv_py = _create_virtualenv_dist()
create_virtualenv_command = '{} {} --no-wheel {}'.format(
self._interpreter, virtualenv_py, virtualenv_path)
requirements_file = os.path.join(self._repo_dir, 'requirements.txt')
self._runner.run(create_virtualenv_command, cwd=self._repo_dir)
pip_path = utils.get_python_executable('pip',
exec_home=virtualenv_path)
install_command = None
if os.path.exists(requirements_file):
self._debug(
'Using requirements file: {}'.format(requirements_file))
install_command = '{} -r {}'.format(self._pip_install(pip_path),
requirements_file)
elif os.path.exists(self._setup_py_path):
self._debug('Using install_requires from setup.py: {}'.format(
self._setup_py_path))
requires = self._setup_py.get('install_requires')
install_command = '{} {}'.format(self._pip_install(pip_path),
' '.join(requires))
if install_command:
self._debug('Installing {} requirements...'.format(name))
self._runner.run(install_command, cwd=self._repo_dir)
self._debug(
'Successfully created virtualenv {}'.format(virtualenv_path))
try:
yield virtualenv_path
finally:
try:
utils.rmf(temp_dir)
except BaseException as e:
if utils.is_windows():
# The temp_dir was populated with files written by a different process
# (pip install) On windows, this causes a [Error 5] Access is denied error.
# Eventually I will have to fix this - until then, sorry windows users...
self._debug(
"Failed cleaning up temporary directory after creating virtualenv "
"{}: {} - You might have some leftovers because of this..."
.format(temp_dir, str(e)))
else:
raise
def nsis(self,
binary_path,
version=None,
output=None,
author=None,
url=None,
copyright_string=None,
description=None,
license_path=None,
program_files_dir=None):
"""
Create a windows installer package.
This method will produce an executable installer (.exe) that, when executed, will install
the provided binary into "Program Files". In addition, it will manipulate the system PATH
variable on the target machine so that the binary can be executed from any directory.
Under the hood, this uses the NSIS project.
For more information please visit https://nsis.sourceforge.io/Main_Page
Args:
binary_path (:str): True if the created will should be universal, False otherwise.
version (:str, optional): Version string metadata. Defaults to the 'version' argument
in your setup.py file.
output (:str, optional): Target file to create. Defaults to
{binary-path-basename}-installer.exe
author (:str, optional): Package author. Defaults to the value specified in setup.py.
url (:str, optional): URL to the package website. Defaults to the value specified in setup.py.
copyright_string (:str, optional): Copyright. Defaults to an empty string.
description (:str, optional): Package description. Defaults to the value specified in setup.py.
license_path (:str, optional): Path to a license file. Defaults to the value specified in setup.py.
program_files_dir (:str, optional): Directory name inside Program Files where the app will be installed.
Raises:
LicenseNotFoundException: License file doesn't exist.
BinaryFileDoesntExistException: The provided binary file doesn't exist.
FileExistsException: Destination file already exists.
DirectoryDoesntExistException: The destination directory does not exist.
"""
if not utils.is_windows():
raise exceptions.WrongPlatformException(expected='Windows')
if not binary_path:
raise exceptions.InvalidArgumentsException('Must pass binary_path')
try:
self._debug('Validating binary exists: {}'.format(binary_path))
utils.validate_file_exists(binary_path)
except (exceptions.FileDoesntExistException,
exceptions.FileIsADirectoryException):
raise exceptions.BinaryDoesntExistException(binary_path)
try:
version = version or self._version
self._debug('Validating version string: {}'.format(version))
utils.validate_nsis_version(version)
except exceptions.InvalidNSISVersionException as err:
tb = sys.exc_info()[2]
try:
# Auto-correction attempt for standard python versions
version = '{}.0'.format(version)
utils.validate_nsis_version(version)
except exceptions.InvalidNSISVersionException:
utils.raise_with_traceback(err, tb)
installer_base_name = os.path.basename(binary_path).replace('.exe', '')
try:
name = self._name
except BaseException as e:
self._debug(
'Unable to extract default name from setup.py: {}. Using binary base name...'
.format(str(e)))
name = installer_base_name
installer_name = '{}-installer'.format(installer_base_name)
copyright_string = copyright_string or ''
destination = os.path.abspath(
output
or '{}.exe'.format(os.path.join(self._target_dir, installer_name)))
self._debug('Validating destination file does not exist: {}'.format(
destination))
utils.validate_file_does_not_exist(destination)
target_directory = os.path.abspath(os.path.join(
destination, os.pardir))
self._debug(
'Validating target directory exists: {}'.format(target_directory))
utils.validate_directory_exists(target_directory)
try:
license_path = license_path or os.path.abspath(
os.path.join(self._repo_dir, self._license))
self._debug(
'Validating license file exists: {}'.format(license_path))
utils.validate_file_exists(license_path)
except (exceptions.FileDoesntExistException,
exceptions.FileIsADirectoryException) as e:
raise exceptions.LicenseNotFoundException(str(e))
author = author or self._author
url = url or self._url
description = description or self._description
program_files_dir = program_files_dir or name
config = {
'name': name,
'author': author,
'website': url,
'copyright': copyright_string,
'license_path': license_path,
'binary_path': binary_path,
'description': description,
'installer_name': installer_name,
'program_files_dir': program_files_dir
}
temp_dir = tempfile.mkdtemp()
try:
support = 'windows_support'
template = get_text_resource(
os.path.join(support, 'installer.nsi.jinja'))
nsis_zip_resource = get_binary_resource(
os.path.join(support, 'nsis-3.04.zip'))
path_header_resource = get_text_resource(
os.path.join(support, 'path.nsh'))
self._debug('Rendering nsi template...')
nsi = Template(template).render(**config)
installer_path = os.path.join(temp_dir, 'installer.nsi')
with open(installer_path, 'w') as f:
f.write(nsi)
self._debug(
'Finished rendering nsi template: {}'.format(installer_path))
self._debug('Writing path header file...')
path_header_path = os.path.join(temp_dir, 'path.nsh')
with open(path_header_path, 'w') as header:
header.write(path_header_resource)
self._debug('Finished writing path header file: {}'.format(
path_header_path))
self._debug('Extracting NSIS from resources...')
nsis_archive = os.path.join(temp_dir, 'nsis.zip')
with open(nsis_archive, 'wb') as _w:
_w.write(nsis_zip_resource)
utils.unzip(nsis_archive, target_dir=temp_dir)
self._debug(
'Finished extracting makensis.exe from resources: {}'.format(
nsis_archive))
makensis_path = os.path.join(temp_dir, 'nsis-3.04', 'makensis.exe')
command = '{} -DVERSION={} {}'.format(makensis_path, version,
installer_path)
# The installer expects the binary to be located in the working directory
# and be named {{ name }}.exe.
# See installer.nsi.jinja#L85
expected_binary_path = os.path.join(temp_dir,
'{}.exe'.format(name))
self._debug('Copying binary to expected location: {}'.format(
expected_binary_path))
shutil.copyfile(src=binary_path, dst=expected_binary_path)
self._debug('Creating installer...')
self._runner.run(command, cwd=temp_dir)
out_file = os.path.join(temp_dir, '{}.exe'.format(installer_name))
self._debug('Copying {} to target path...'.format(out_file))
shutil.copyfile(out_file, destination)
self._debug('Finished copying installer to target path: {}'.format(
destination))
self._debug('Packaged successfully.', package=destination)
return destination
finally:
utils.rmf(temp_dir)
def wheel(self, universal=False, wheel_version=None):
"""
Create a wheel package.
This method will create a wheel package, according the the regular python wheel standards.
Under the hood, this uses the bdist_wheel command provided by the wheel project.
For more information please visit https://pythonwheels.com/
Args:
universal (:bool, optional): True if the created will should be universal, False otherwise.
wheel_version (:str, optional): Which wheel version to use.
Raises:
WheelExistsException: Destination file already exists.
DirectoryDoesntExistException: Destination directory does not exist.
"""
temp_dir = tempfile.mkdtemp()
try:
dist_dir = os.path.join(temp_dir, 'dist')
bdist_dir = os.path.join(temp_dir, 'bdist')
if not os.path.exists(self._setup_py_path):
raise exceptions.SetupPyNotFoundException(
repo=self._repo_location)
name = self._name
with self._create_virtualenv(name) as virtualenv:
self._logger.debug('Installing wheel...')
pip_path = utils.get_python_executable('pip',
exec_home=virtualenv)
self._runner.run('{} wheel=={}'.format(
self._pip_install(pip_path), wheel_version
or DEFAULT_WHEEL_VERSION),
cwd=self._repo_dir)
command = '{} {} bdist_wheel --bdist-dir {} --dist-dir {}'.format(
utils.get_python_executable('python',
exec_home=virtualenv),
self._setup_py_path, bdist_dir, dist_dir)
if universal:
command = '{0} --universal'.format(command)
self._debug('Running bdist_wheel...', universal=universal)
self._runner.run(command, cwd=self._repo_dir)
self._debug('Finished running bdist_wheel.', universal=universal)
actual_name = utils.lsf(dist_dir)[0]
destination = os.path.join(self._target_dir, actual_name)
try:
utils.validate_file_does_not_exist(path=destination)
except exceptions.FileExistException as e:
raise exceptions.WheelExistsException(path=e.path)
shutil.copy(os.path.join(dist_dir, actual_name), destination)
self._debug('Packaged successfully.', package=destination)
return os.path.abspath(destination)
finally:
utils.rmf(temp_dir)
def binary(self,
base_name=None,
entrypoint=None,
pyinstaller_version=None):
"""
Create a binary executable.
This method will create a self-contained, platform dependent, executable file.
Under the hood, this uses the PyInstaller project.
For more information please visit https://www.pyinstaller.org/
Args:
base_name (:str, optional):
The base name of the target file. The final name will be in the
form of: <name>-<platform-machine>-<platform-system> (e.g pyci-x86_64-Darwin).
Defaults to the 'name' specified in your setup.py file.
entrypoint (:str, optional):
Path to a script file from which the executable
is built. This can either by a .py or a .spec file.
By default, the packager will look for the entry point specified in setup.py.
pyinstaller_version (:str, optional):
Which PyInstaller version to use. Defaults to 3.4.
Raises:
BinaryExistsException:
Destination file already exists.
DirectoryDoesntExistException:
Destination directory does not exist.
DefaultEntrypointNotFoundException:
A custom entrypoint is not provided and the default entry-points
pyci looks for are also missing.
EntrypointNotFoundException:
The provided custom entrypoint provided does not exist in the repository.
MultipleDefaultEntrypointsFoundException:
If setup.py contains multiple entrypoints specification.
"""
temp_dir = tempfile.mkdtemp()
try:
base_name = base_name or self._name
entrypoint = entrypoint or self._entrypoint
destination = os.path.join(
self._target_dir, '{0}-{1}-{2}'.format(base_name,
platform.machine(),
platform.system()))
if platform.system().lower() == 'windows':
destination = '{0}.exe'.format(destination)
try:
utils.validate_file_does_not_exist(path=destination)
except exceptions.FileExistException as e:
raise exceptions.BinaryExistsException(path=e.path)
dist_dir = os.path.join(temp_dir, 'dist')
build_dir = os.path.join(temp_dir, 'build')
script = os.path.join(self._repo_dir, entrypoint)
if not os.path.exists(script):
raise exceptions.EntrypointNotFoundException(
repo=self._repo_location, entrypoint=entrypoint)
with self._create_virtualenv(base_name) as virtualenv:
self._logger.debug('Installing pyinstaller...')
pip_path = utils.get_python_executable('pip',
exec_home=virtualenv)
self._runner.run('{} pyinstaller=={}'.format(
self._pip_install(pip_path), pyinstaller_version
or DEFAULT_PY_INSTALLER_VERSION),
cwd=self._repo_dir)
self._debug('Running pyinstaller...',
entrypoint=entrypoint,
destination=destination)
pyinstaller_path = utils.get_python_executable(
'pyinstaller', exec_home=virtualenv)
self._runner.run('{} '
'--onefile '
'--distpath {} '
'--workpath {} '
'--specpath {} {}'.format(
self._pyinstaller(pyinstaller_path),
dist_dir, build_dir, temp_dir, script))
self._debug('Finished running pyinstaller',
entrypoint=entrypoint,
destination=destination)
actual_name = utils.lsf(dist_dir)[0]
package_path = os.path.join(dist_dir, actual_name)
self._debug('Copying package to destination...',
src=package_path,
dst=destination)
shutil.copy(package_path, destination)
self._debug('Packaged successfully.', package=destination)
return os.path.abspath(destination)
finally:
utils.rmf(temp_dir)