def main(self):
args = self.parse_args()
version = get_version_from_source_tree(args.source)
if args.target == 'wheel-linux':
pkg_name = WHEEL_LINUX_CONFIGS[args.cuda]['name']
filename = wheel_name(pkg_name, version, args.python,
'manylinux1_x86_64')
elif args.target == 'wheel-win':
pkg_name = WHEEL_WINDOWS_CONFIGS[args.cuda]['name']
filename = wheel_name(pkg_name, version, args.python, 'win_amd64')
elif args.target == 'sdist':
pkg_name = 'cupy'
filename = sdist_name(pkg_name, version)
print('DIST_FILE_NAME="{}"'.format(filename))
print('DIST_PACKAGE_NAME="{}"'.format(pkg_name))
def main(self):
args = self.parse_args()
version = get_version_from_source_tree(args.source)
if args.target == 'wheel-linux':
print('DIST_FILE_NAME="{}"'.format(
wheel_name(
args.cuda, version, args.python, 'manylinux1_x86_64'),
))
print('DIST_PACKAGE_NAME="{}"'.format(
WHEEL_LINUX_CONFIGS[args.cuda]['name'],
))
elif args.target == 'sdist':
print('DIST_FILE_NAME="{}"'.format(
sdist_name('cupy', version)
))
print('DIST_PACKAGE_NAME="cupy"')
def build_windows(
self, target, nccl_assets, cuda_version, python_version,
source, output):
"""Build a single wheel distribution for Windows.
Note that Windows build is not isolated.
"""
# Perform additional check as Windows environment is not isoalted.
self._check_windows_environment(cuda_version, python_version)
if target != 'wheel-win':
raise ValueError('unknown target')
if nccl_assets is not None:
raise RuntimeError('NCCL not supported on Windows')
version = get_version_from_source_tree(source)
log(
'Starting wheel-win build from {} '
'(version {}, for CUDA {} + Python {})'.format(
source, version, cuda_version, python_version))
action = 'bdist_wheel'
package_name = WHEEL_WINDOWS_CONFIGS[cuda_version]['name']
long_description = WHEEL_LONG_DESCRIPTION.format(cuda=cuda_version)
asset_name = wheel_name(
package_name, version, python_version, 'win_amd64')
asset_dest_name = asset_name
agent_args = [
'--action', action,
'--source', 'cupy',
]
# Add requirements for build.
for req in WHEEL_PYTHON_VERSIONS[python_version]['requires']:
agent_args += ['--requires', req]
# Add arguments to pass to setup.py.
setup_args = [
'--cupy-package-name', package_name,
'--cupy-long-description', '../description.rst',
]
setup_args += ['--cupy-wheel-metadata', '../_wheel.json']
for lib in WHEEL_WINDOWS_CONFIGS[cuda_version]['libs']:
libpath = find_file_in_path(lib)
if libpath is None:
raise RuntimeError(
'Library {} could not be found in PATH'.format(lib))
setup_args += ['--cupy-wheel-lib', libpath]
agent_args += setup_args
# Create a working directory.
workdir = tempfile.mkdtemp(prefix='cupy-dist-')
try:
log('Using working directory: {}'.format(workdir))
# Copy source tree and NCCL to working directory.
log('Copying source tree from: {}'.format(source))
shutil.copytree(source, '{}/cupy'.format(workdir))
# Add long description file.
with open('{}/description.rst'.format(workdir), 'w') as f:
f.write(long_description)
# Get cuDNN version.
cudnn_version, cudnn_assets = get_cudnn_record(
cuda_version, 'Windows')
log('cuDNN version: {}'.format(cudnn_version))
log('cuDNN assets: {}'.format(cudnn_assets))
# Create a wheel metadata file for preload.
log('Writing wheel metadata')
wheel_metadata = {
'cuda': cuda_version,
'cudnn': {
'version': cudnn_version,
'filename': cudnn_assets['filename'],
}
}
with open('{}/_wheel.json'.format(workdir), 'w') as f:
json.dump(wheel_metadata, f)
# Build.
log('Starting build')
run_command(
sys.executable, '{}/builder/agent.py'.format(os.getcwd()),
*agent_args, cwd=workdir)
log('Finished build')
# Copy assets.
asset_path = '{}/cupy/dist/{}'.format(workdir, asset_name)
output_path = '{}/{}'.format(output, asset_dest_name)
log('Copying asset from {} to {}'.format(asset_path, output_path))
shutil.copy2(asset_path, output_path)
finally:
log('Removing working directory: {}'.format(workdir))
try:
shutil.rmtree(workdir)
except OSError as e:
# TODO(kmaehashi): On Windows, removal of `.git` directory may
# fail with PermissionError (on Python 3) or OSError (on
# Python 2). Note that PermissionError inherits OSError.
log('Failed to clean-up working directory: {}\n\n'
'Please remove the working directory manually: {}'.format(
e, workdir))
def build_linux(
self, target, nccl_assets, cuda_version, python_version,
source, output):
"""Build a single wheel distribution for Linux."""
version = get_version_from_source_tree(source)
if target == 'wheel-linux':
assert cuda_version is not None
log(
'Starting wheel-linux build from {} '
'(version {}, for CUDA {} + Python {})'.format(
source, version, cuda_version, python_version))
action = 'bdist_wheel'
image_tag = 'cupy-builder-{}'.format(cuda_version)
kind = WHEEL_LINUX_CONFIGS[cuda_version]['kind']
base_image = WHEEL_LINUX_CONFIGS[cuda_version]['image']
package_name = WHEEL_LINUX_CONFIGS[cuda_version]['name']
long_description = WHEEL_LONG_DESCRIPTION.format(cuda=cuda_version)
if kind == 'cuda':
# NCCL
nccl_config = WHEEL_LINUX_CONFIGS[cuda_version]['nccl']
# cuDNN
cudnn_version, cudnn_assets = get_cudnn_record(
cuda_version, 'Linux')
elif kind == 'rocm':
nccl_config = None
cudnn_version = None
cudnn_assets = None
else:
assert False
# Rename wheels to manylinux1.
asset_name = wheel_name(
package_name, version, python_version, 'linux_x86_64')
asset_dest_name = wheel_name(
package_name, version, python_version, 'manylinux1_x86_64')
elif target == 'sdist':
assert cuda_version is None
log('Starting sdist build from {} (version {})'.format(
source, version))
action = 'sdist'
image_tag = 'cupy-builder-sdist'
kind = 'cuda'
base_image = SDIST_CONFIG['image']
package_name = 'cupy'
long_description = SDIST_LONG_DESCRIPTION
# NCCL
nccl_config = None
# cuDNN (use pre-installed version)
cudnn_version = None
cudnn_assets = None
# Rename not needed for sdist.
asset_name = sdist_name('cupy', version)
asset_dest_name = asset_name
else:
raise RuntimeError('unknown target')
# Arguments for the agent.
agent_args = [
'--action', action,
'--source', 'cupy',
'--python', python_version,
'--chown', '{}:{}'.format(os.getuid(), os.getgid()),
]
# Add arguments to pass to setup.py.
setup_args = [
'--cupy-package-name', package_name,
'--cupy-long-description', '../description.rst',
]
if target == 'wheel-linux':
# Add requirements for build.
for req in WHEEL_PYTHON_VERSIONS[python_version]['requires']:
agent_args += ['--requires', req]
setup_args += [
'--cupy-no-rpath',
'--cupy-wheel-metadata', '../_wheel.json',
]
for lib in WHEEL_LINUX_CONFIGS[cuda_version]['libs']:
setup_args += ['--cupy-wheel-lib', lib]
for include_path, include_relpath in (
WHEEL_LINUX_CONFIGS[cuda_version]['includes']):
spec = '{}:{}'.format(include_path, include_relpath)
setup_args += ['--cupy-wheel-include', spec]
elif target == 'sdist':
setup_args += [
'--cupy-no-cuda',
]
agent_args += setup_args
# Create a working directory.
workdir = tempfile.mkdtemp(prefix='cupy-dist-')
try:
log('Using working directory: {}'.format(workdir))
# Copy source tree to working directory.
log('Copying source tree from: {}'.format(source))
shutil.copytree(source, '{}/cupy'.format(workdir))
# Add long description file.
with open('{}/description.rst'.format(workdir), 'w') as f:
f.write(long_description)
# Copy builder directory to working directory.
docker_ctx = '{}/builder'.format(workdir)
log('Copying builder directory to: '.format(docker_ctx))
shutil.copytree('builder/', docker_ctx)
# Extract NCCL archive.
log('Creating nccl directory under builder directory')
nccl_workdir = '{}/nccl'.format(docker_ctx)
os.mkdir(nccl_workdir)
if nccl_config is not None:
log('Extracting NCCL archive')
extract_nccl_archive(nccl_config, nccl_assets, nccl_workdir)
else:
log('NCCL is not installed for this build')
# Extract cuDNN archive.
cudnn_workdir = '{}/cudnn'.format(docker_ctx)
os.mkdir(cudnn_workdir)
if cudnn_version is not None:
log('cuDNN version: {}'.format(cudnn_version))
log('cuDNN assets: {}'.format(cudnn_assets))
log('Creating cudnn directory under builder directory')
download_extract_cudnn_archive(
cudnn_assets['url'], cudnn_workdir)
else:
log('cuDNN is not installed for this build')
# Create a wheel metadata file for preload.
if target == 'wheel-linux':
log('Writing wheel metadata')
wheel_metadata = {
'cuda': cuda_version,
}
if cudnn_version is not None:
wheel_metadata['cudnn'] = {
'version': cudnn_version,
'filename': cudnn_assets['filename'],
}
with open('{}/_wheel.json'.format(workdir), 'w') as f:
json.dump(wheel_metadata, f)
# Creates a Docker image to build distribution.
self._create_builder_linux(image_tag, base_image, docker_ctx)
# Build.
log('Starting build')
self._run_container(image_tag, kind, workdir, agent_args)
log('Finished build')
# Copy assets.
asset_path = '{}/cupy/dist/{}'.format(workdir, asset_name)
output_path = '{}/{}'.format(output, asset_dest_name)
log('Copying asset from {} to {}'.format(asset_path, output_path))
shutil.copy2(asset_path, output_path)
finally:
log('Removing working directory: {}'.format(workdir))
shutil.rmtree(workdir)
def build_linux(self, target, nccl_assets, cuda_version, python_version,
source, output):
"""Build a single wheel distribution for Linux."""
version = get_version_from_source_tree(source)
if nccl_assets is None:
raise RuntimeError('NCCL assets must be specified for Linux')
if target == 'wheel-linux':
log('Starting wheel-linux build from {} '
'(version {}, for CUDA {} + Python {})'.format(
source, version, cuda_version, python_version))
action = 'bdist_wheel'
image_tag = 'cupy-builder-{}'.format(cuda_version)
base_image = WHEEL_LINUX_CONFIGS[cuda_version]['image']
package_name = WHEEL_LINUX_CONFIGS[cuda_version]['name']
nccl_config = WHEEL_LINUX_CONFIGS[cuda_version]['nccl']
long_description = WHEEL_LONG_DESCRIPTION.format(cuda=cuda_version)
# Rename wheels to manylinux1.
asset_name = wheel_name(package_name, version, python_version,
'linux_x86_64')
asset_dest_name = wheel_name(package_name, version, python_version,
'manylinux1_x86_64')
elif target == 'sdist':
log('Starting sdist build from {} (version {})'.format(
source, version))
action = 'sdist'
image_tag = 'cupy-builder-sdist'
base_image = SDIST_CONFIG['image']
package_name = 'cupy'
nccl_config = SDIST_CONFIG['nccl']
long_description = SDIST_LONG_DESCRIPTION
asset_name = sdist_name('cupy', version)
asset_dest_name = asset_name
assert nccl_config is not None
else:
raise RuntimeError('unknown target')
# Arguments for the agent.
agent_args = [
'--action',
action,
'--source',
'cupy',
'--python',
python_version,
'--chown',
'{}:{}'.format(os.getuid(), os.getgid()),
]
if nccl_config:
agent_args += ['--nccl', 'nccl']
# Add arguments to pass to setup.py.
setup_args = [
'--cupy-package-name',
package_name,
'--cupy-long-description',
'../description.rst',
]
if target == 'wheel-linux':
# Add requirements for build.
for req in WHEEL_PYTHON_VERSIONS[python_version]['requires']:
agent_args += ['--requires', req]
setup_args += [
'--cupy-no-rpath',
]
for lib in WHEEL_LINUX_CONFIGS[cuda_version]['libs']:
setup_args += ['--cupy-wheel-lib', lib]
agent_args += setup_args
# Create a working directory.
workdir = tempfile.mkdtemp(prefix='cupy-dist-')
try:
log('Using working directory: {}'.format(workdir))
# Copy source tree and NCCL to working directory.
log('Copying source tree from: {}'.format(source))
shutil.copytree(source, '{}/cupy'.format(workdir))
# Extract NCCL.
if nccl_config:
nccl_workdir = '{}/nccl'.format(workdir)
os.mkdir(nccl_workdir)
extract_nccl_archive(nccl_config, nccl_assets, nccl_workdir)
else:
log('NCCL is not used for this package')
# Add long description file.
if long_description is not None:
with open('{}/description.rst'.format(workdir), 'w') as f:
f.write(long_description)
# Creates a Docker image to build distribution.
self._create_builder_linux(image_tag, base_image)
# Build.
log('Starting build')
self._run_container(image_tag, workdir, agent_args)
log('Finished build')
# Copy assets.
asset_path = '{}/cupy/dist/{}'.format(workdir, asset_name)
output_path = '{}/{}'.format(output, asset_dest_name)
log('Copying asset from {} to {}'.format(asset_path, output_path))
shutil.copy2(asset_path, output_path)
finally:
log('Removing working directory: {}'.format(workdir))
shutil.rmtree(workdir)
def build_linux(
self, target, cuda_version, python_version,
source, output):
"""Build a single wheel distribution for Linux."""
version = get_version_from_source_tree(source)
self._ensure_compatible_branch(version)
if target == 'wheel-linux':
assert cuda_version is not None
log(
'Starting wheel-linux build from {} '
'(version {}, for CUDA {} + Python {})'.format(
source, version, cuda_version, python_version))
action = 'bdist_wheel'
image_tag = 'cupy-builder-{}'.format(cuda_version)
kind = WHEEL_LINUX_CONFIGS[cuda_version]['kind']
preloads = WHEEL_LINUX_CONFIGS[cuda_version]['preloads']
platform_version = WHEEL_LINUX_CONFIGS[cuda_version].get(
'platform_version', cuda_version)
base_image = WHEEL_LINUX_CONFIGS[cuda_version]['image']
package_name = WHEEL_LINUX_CONFIGS[cuda_version]['name']
system_packages = \
WHEEL_LINUX_CONFIGS[cuda_version]['system_packages']
if kind == 'cuda':
long_description_tmpl = WHEEL_LONG_DESCRIPTION_CUDA
elif kind == 'rocm':
long_description_tmpl = WHEEL_LONG_DESCRIPTION_ROCM
else:
assert False
long_description = long_description_tmpl.format(
version=platform_version)
# Rename wheels to manylinux1.
asset_name = wheel_name(
package_name, version, python_version, 'linux_x86_64')
asset_dest_name = wheel_name(
package_name, version, python_version, 'manylinux1_x86_64')
elif target == 'sdist':
assert cuda_version is None
log('Starting sdist build from {} (version {})'.format(
source, version))
action = 'sdist'
image_tag = 'cupy-builder-sdist'
kind = 'cuda'
preloads = []
base_image = SDIST_CONFIG['image']
package_name = 'cupy'
system_packages = ''
long_description = SDIST_LONG_DESCRIPTION
# Rename not needed for sdist.
asset_name = sdist_name('cupy', version)
asset_dest_name = asset_name
else:
raise RuntimeError('unknown target')
# Arguments for the agent.
agent_args = [
'--action', action,
'--source', 'cupy',
'--python', python_version,
'--chown', '{}:{}'.format(os.getuid(), os.getgid()),
]
# Add arguments to pass to setup.py.
setup_args = [
'--cupy-package-name', package_name,
'--cupy-long-description', '../description.rst',
]
if target == 'wheel-linux':
# Add requirements for build.
for req in WHEEL_PYTHON_VERSIONS[python_version]['requires']:
agent_args += ['--requires', req]
setup_args += [
'--cupy-no-rpath',
'--cupy-wheel-metadata', '../_wheel.json',
]
for lib in WHEEL_LINUX_CONFIGS[cuda_version]['libs']:
setup_args += ['--cupy-wheel-lib', lib]
for include_path, include_relpath in (
WHEEL_LINUX_CONFIGS[cuda_version]['includes']):
spec = '{}:{}'.format(include_path, include_relpath)
setup_args += ['--cupy-wheel-include', spec]
elif target == 'sdist':
setup_args += [
'--cupy-no-cuda',
]
agent_args += setup_args
# Create a working directory.
workdir = tempfile.mkdtemp(prefix='cupy-dist-')
try:
log('Using working directory: {}'.format(workdir))
# Copy source tree to working directory.
log('Copying source tree from: {}'.format(source))
shutil.copytree(source, '{}/cupy'.format(workdir))
# Add long description file.
with open('{}/description.rst'.format(workdir), 'w') as f:
f.write(long_description)
# Copy builder directory to working directory.
docker_ctx = '{}/builder'.format(workdir)
log('Copying builder directory to: {}'.format(docker_ctx))
shutil.copytree('builder/', docker_ctx)
# Extract optional CUDA libraries.
optlib_workdir = '{}/cuda_lib'.format(docker_ctx)
log('Creating CUDA optional lib directory under '
'builder directory: {}'.format(optlib_workdir))
os.mkdir(optlib_workdir)
# Install CUDA optional libraries and generate a wheel metadata.
if target == 'wheel-linux':
wheel_metadata = {
'cuda': cuda_version,
'packaging': 'pip',
}
for p in preloads:
wheel_metadata[p] = prepare_cuda_opt_library(
p, cuda_version, optlib_workdir)
log('Writing wheel metadata')
with open('{}/_wheel.json'.format(workdir), 'w') as f:
json.dump(wheel_metadata, f)
# Creates a Docker image to build distribution.
self._create_builder_linux(
image_tag, base_image, system_packages, docker_ctx)
# Build.
log('Starting build')
self._run_container(image_tag, kind, workdir, agent_args)
log('Finished build')
# Copy assets.
asset_path = '{}/cupy/dist/{}'.format(workdir, asset_name)
output_path = '{}/{}'.format(output, asset_dest_name)
log('Copying asset from {} to {}'.format(asset_path, output_path))
shutil.copy2(asset_path, output_path)
finally:
log('Removing working directory: {}'.format(workdir))
shutil.rmtree(workdir)
def build_windows(self, target, cuda_version, python_version, source,
output):
"""Build a single wheel distribution for Windows.
Note that Windows build is not isolated.
"""
# Perform additional check as Windows environment is not isoalted.
self._check_windows_environment(cuda_version, python_version)
if target != 'wheel-win':
raise ValueError('unknown target')
version = get_version_from_source_tree(source)
self._ensure_compatible_branch(version)
log('Starting wheel-win build from {} '
'(version {}, for CUDA {} + Python {})'.format(
source, version, cuda_version, python_version))
action = 'bdist_wheel'
preloads = WHEEL_WINDOWS_CONFIGS[cuda_version]['preloads']
platform_version = WHEEL_LINUX_CONFIGS[cuda_version].get(
'platform_version', cuda_version)
package_name = WHEEL_WINDOWS_CONFIGS[cuda_version]['name']
long_description = WHEEL_LONG_DESCRIPTION_CUDA.format(
version=platform_version)
asset_name = wheel_name(package_name, version, python_version,
'win_amd64')
asset_dest_name = asset_name
agent_args = [
'--action',
action,
'--source',
'cupy',
]
# Add arguments to pass to setup.py.
setup_args = [
'--cupy-package-name',
package_name,
'--cupy-long-description',
'../description.rst',
]
setup_args += ['--cupy-wheel-metadata', '../_wheel.json']
for lib in WHEEL_WINDOWS_CONFIGS[cuda_version]['libs']:
libpath = find_file_in_path(lib)
if libpath is None:
raise RuntimeError(
'Library {} could not be found in PATH'.format(lib))
setup_args += ['--cupy-wheel-lib', libpath]
agent_args += setup_args
# Create a working directory.
workdir = tempfile.mkdtemp(prefix='cupy-dist-')
try:
log('Using working directory: {}'.format(workdir))
# Copy source tree and NCCL to working directory.
log('Copying source tree from: {}'.format(source))
shutil.copytree(source, '{}/cupy'.format(workdir))
# Add long description file.
with open('{}/description.rst'.format(workdir), 'w') as f:
f.write(long_description)
# Create a wheel metadata file for preload.
wheel_metadata = {
'cuda': cuda_version,
'packaging': 'pip',
}
# Extract optional CUDA libraries.
optlib_workdir = '{}/cuda_lib'.format(workdir)
log('Creating CUDA optional lib directory under '
'working directory: {}'.format(optlib_workdir))
os.mkdir(optlib_workdir)
for p in preloads:
wheel_metadata[p] = prepare_cuda_opt_library(
p, cuda_version, optlib_workdir)
# Create a wheel metadata file for preload.
log('Writing wheel metadata')
with open('{}/_wheel.json'.format(workdir), 'w') as f:
json.dump(wheel_metadata, f)
# Install optional CUDA libraries.
log('Installing CUDA optional libraries')
run_command(sys.executable,
'{}/builder/setup_cuda_opt_lib.py'.format(os.getcwd()),
'--src',
optlib_workdir,
'--dst',
os.environ['CUDA_PATH'],
cwd=workdir)
# Build.
log('Starting build')
run_command(sys.executable,
'{}/builder/agent.py'.format(os.getcwd()),
*agent_args,
cwd=workdir)
log('Finished build')
# Copy assets.
asset_path = '{}/cupy/dist/{}'.format(workdir, asset_name)
output_path = '{}/{}'.format(output, asset_dest_name)
log('Copying asset from {} to {}'.format(asset_path, output_path))
shutil.copy2(asset_path, output_path)
finally:
log('Removing working directory: {}'.format(workdir))
try:
shutil.rmtree(workdir)
except OSError as e:
# TODO(kmaehashi): On Windows, removal of `.git` directory may
# fail with PermissionError (on Python 3) or OSError (on
# Python 2). Note that PermissionError inherits OSError.
log('Failed to clean-up working directory: {}\n\n'
'Please remove the working directory manually: {}'.format(
e, workdir))