def configure_extension_build():
r"""Configures extension build options according to system environment and user's choice.
Returns:
The input to parameters ext_modules, cmdclass, packages, and entry_points as required in setuptools.setup.
"""
try:
cmake_cache_vars = defaultdict(lambda: False,
cmake.get_cmake_cache_variables())
except FileNotFoundError:
# CMakeCache.txt does not exist. Probably running "python setup.py clean" over a clean directory.
cmake_cache_vars = defaultdict(lambda: False)
################################################################################
# Configure compile flags
################################################################################
library_dirs = []
extra_install_requires = []
if IS_WINDOWS:
# /NODEFAULTLIB makes sure we only link to DLL runtime
# and matches the flags set for protobuf and ONNX
extra_link_args = ['/NODEFAULTLIB:LIBCMT.LIB']
# /MD links against DLL runtime
# and matches the flags set for protobuf and ONNX
# /EHsc is about standard C++ exception handling
# /DNOMINMAX removes builtin min/max functions
# /wdXXXX disables warning no. XXXX
extra_compile_args = [
'/MD', '/EHsc', '/DNOMINMAX', '/wd4267', '/wd4251', '/wd4522',
'/wd4522', '/wd4838', '/wd4305', '/wd4244', '/wd4190', '/wd4101',
'/wd4996', '/wd4275'
]
else:
extra_link_args = []
extra_compile_args = [
'-Wall',
'-Wextra',
'-Wno-strict-overflow',
'-Wno-unused-parameter',
'-Wno-missing-field-initializers',
'-Wno-write-strings',
'-Wno-unknown-pragmas',
# This is required for Python 2 declarations that are deprecated in 3.
'-Wno-deprecated-declarations',
# Python 2.6 requires -fno-strict-aliasing, see
# http://legacy.python.org/dev/peps/pep-3123/
# We also depend on it in our code (even Python 3).
'-fno-strict-aliasing',
# Clang has an unfixed bug leading to spurious missing
# braces warnings, see
# https://bugs.llvm.org/show_bug.cgi?id=21629
'-Wno-missing-braces',
]
if check_env_flag('WERROR'):
extra_compile_args.append('-Werror')
library_dirs.append(lib_path)
main_compile_args = []
main_libraries = ['torch_python']
main_link_args = []
main_sources = ["torch/csrc/stub.c"]
if cmake_cache_vars['USE_CUDA']:
library_dirs.append(os.path.dirname(cmake_cache_vars['CUDA_CUDA_LIB']))
if build_type.is_debug():
if IS_WINDOWS:
extra_compile_args.append('/Z7')
extra_link_args.append('/DEBUG:FULL')
else:
extra_compile_args += ['-O0', '-g']
extra_link_args += ['-O0', '-g']
if build_type.is_rel_with_deb_info():
if IS_WINDOWS:
extra_compile_args.append('/Z7')
extra_link_args.append('/DEBUG:FULL')
else:
extra_compile_args += ['-g']
extra_link_args += ['-g']
# Cross-compile for M1
if IS_DARWIN:
macos_target_arch = os.getenv('CMAKE_OSX_ARCHITECTURES', '')
if macos_target_arch in ['arm64', 'x86_64']:
macos_sysroot_path = os.getenv('CMAKE_OSX_SYSROOT')
if macos_sysroot_path is None:
macos_sysroot_path = subprocess.check_output(
['xcrun', '--show-sdk-path', '--sdk',
'macosx']).decode('utf-8').strip()
extra_compile_args += [
'-arch', macos_target_arch, '-isysroot', macos_sysroot_path
]
extra_link_args += ['-arch', macos_target_arch]
def make_relative_rpath_args(path):
if IS_DARWIN:
return ['-Wl,-rpath,@loader_path/' + path]
elif IS_WINDOWS:
return []
else:
return ['-Wl,-rpath,$ORIGIN/' + path]
################################################################################
# Declare extensions and package
################################################################################
extensions = []
packages = find_packages(exclude=('tools', 'tools.*'))
C = Extension("torch._C",
libraries=main_libraries,
sources=main_sources,
language='c',
extra_compile_args=main_compile_args + extra_compile_args,
include_dirs=[],
library_dirs=library_dirs,
extra_link_args=extra_link_args + main_link_args +
make_relative_rpath_args('lib'))
extensions.append(C)
if not IS_WINDOWS:
DL = Extension("torch._dl", sources=["torch/csrc/dl.c"], language='c')
extensions.append(DL)
# These extensions are built by cmake and copied manually in build_extensions()
# inside the build_ext implementation
extensions.append(
Extension(name=str('caffe2.python.caffe2_pybind11_state'),
sources=[]), )
if cmake_cache_vars['USE_CUDA']:
extensions.append(
Extension(name=str('caffe2.python.caffe2_pybind11_state_gpu'),
sources=[]), )
if cmake_cache_vars['USE_ROCM']:
extensions.append(
Extension(name=str('caffe2.python.caffe2_pybind11_state_hip'),
sources=[]), )
cmdclass = {
'bdist_wheel': wheel_concatenate,
'build_ext': build_ext,
'clean': clean,
'install': install,
'sdist': sdist,
}
entry_points = {
'console_scripts': [
'convert-caffe2-to-onnx = caffe2.python.onnx.bin.conversion:caffe2_to_onnx',
'convert-onnx-to-caffe2 = caffe2.python.onnx.bin.conversion:onnx_to_caffe2',
]
}
return extensions, cmdclass, packages, entry_points, extra_install_requires
def build_deps():
report('-- Building version ' + version)
def check_file(f):
if not os.path.exists(f):
report("Could not find {}".format(f))
report("Did you run 'git submodule update --init --recursive'?")
sys.exit(1)
check_file(os.path.join(third_party_path, "gloo", "CMakeLists.txt"))
check_file(os.path.join(third_party_path, "pybind11", "CMakeLists.txt"))
check_file(os.path.join(third_party_path, 'cpuinfo', 'CMakeLists.txt'))
check_file(os.path.join(third_party_path, 'tbb', 'Makefile'))
check_file(os.path.join(third_party_path, 'onnx', 'CMakeLists.txt'))
check_file(os.path.join(third_party_path, 'foxi', 'CMakeLists.txt'))
check_file(os.path.join(third_party_path, 'QNNPACK', 'CMakeLists.txt'))
check_file(os.path.join(third_party_path, 'fbgemm', 'CMakeLists.txt'))
check_file(os.path.join(third_party_path, 'fbgemm', 'third_party',
'asmjit', 'CMakeLists.txt'))
check_file(os.path.join(third_party_path, 'onnx', 'third_party',
'benchmark', 'CMakeLists.txt'))
check_pydep('yaml', 'pyyaml')
check_pydep('typing', 'typing')
build_caffe2(version=version,
cmake_python_library=cmake_python_library,
build_python=True,
rerun_cmake=RERUN_CMAKE,
cmake_only=CMAKE_ONLY,
cmake=cmake)
version_path = os.path.join(cwd, 'torch', 'version.py')
with open(version_path, 'w') as f:
f.write("__version__ = '{}'\n".format(version))
# NB: This is not 100% accurate, because you could have built the
# library code with DEBUG, but csrc without DEBUG (in which case
# this would claim to be a release build when it's not.)
f.write("debug = {}\n".format(repr(build_type.is_debug())))
cmake_cache_vars = defaultdict(lambda: None, cmake.get_cmake_cache_variables())
f.write("cuda = {}\n".format(repr(cmake_cache_vars['CUDA_VERSION'])))
f.write("git_version = {}\n".format(repr(sha)))
if CMAKE_ONLY:
report('Finished running cmake. Run "ccmake build" or '
'"cmake-gui build" to adjust build options and '
'"python setup.py install" to build.')
sys.exit()
# Use copies instead of symbolic files.
# Windows has very poor support for them.
sym_files = ['tools/shared/cwrap_common.py', 'tools/shared/_utils_internal.py']
orig_files = ['aten/src/ATen/common_with_cwrap.py', 'torch/_utils_internal.py']
for sym_file, orig_file in zip(sym_files, orig_files):
same = False
if os.path.exists(sym_file):
if filecmp.cmp(sym_file, orig_file):
same = True
else:
os.remove(sym_file)
if not same:
shutil.copyfile(orig_file, sym_file)
dir_util.copy_tree('third_party/pybind11/include/pybind11/',
'torch/include/pybind11')
def configure_extension_build():
r"""Configures extension build options according to system environment and user's choice.
Returns:
The input to parameters ext_modules, cmdclass, packages, and entry_points as required in setuptools.setup.
"""
try:
cmake_cache_vars = defaultdict(lambda: False,
cmake.get_cmake_cache_variables())
except FileNotFoundError:
# CMakeCache.txt does not exist. Probably running "python setup.py clean" over a clean directory.
cmake_cache_vars = defaultdict(lambda: False)
################################################################################
# Configure compile flags
################################################################################
library_dirs = []
if IS_WINDOWS:
# /NODEFAULTLIB makes sure we only link to DLL runtime
# and matches the flags set for protobuf and ONNX
extra_link_args = ['/NODEFAULTLIB:LIBCMT.LIB']
# /MD links against DLL runtime
# and matches the flags set for protobuf and ONNX
# /Z7 turns on symbolic debugging information in .obj files
# /EHa is about native C++ catch support for asynchronous
# structured exception handling (SEH)
# /DNOMINMAX removes builtin min/max functions
# /wdXXXX disables warning no. XXXX
extra_compile_args = [
'/MD', '/Z7', '/EHa', '/DNOMINMAX', '/wd4267', '/wd4251',
'/wd4522', '/wd4522', '/wd4838', '/wd4305', '/wd4244', '/wd4190',
'/wd4101', '/wd4996', '/wd4275'
]
if sys.version_info[0] == 2:
if not check_env_flag('FORCE_PY27_BUILD'):
report(
'The support for PyTorch with Python 2.7 on Windows is very experimental.'
)
report(
'Please set the flag `FORCE_PY27_BUILD` to 1 to continue build.'
)
sys.exit(1)
# /bigobj increases number of sections in .obj file, which is needed to link
# against libaries in Python 2.7 under Windows
extra_compile_args.append('/bigobj')
else:
extra_link_args = []
extra_compile_args = [
'-std=c++14',
'-Wall',
'-Wextra',
'-Wno-strict-overflow',
'-Wno-unused-parameter',
'-Wno-missing-field-initializers',
'-Wno-write-strings',
'-Wno-unknown-pragmas',
# This is required for Python 2 declarations that are deprecated in 3.
'-Wno-deprecated-declarations',
# Python 2.6 requires -fno-strict-aliasing, see
# http://legacy.python.org/dev/peps/pep-3123/
# We also depend on it in our code (even Python 3).
'-fno-strict-aliasing',
# Clang has an unfixed bug leading to spurious missing
# braces warnings, see
# https://bugs.llvm.org/show_bug.cgi?id=21629
'-Wno-missing-braces',
]
if check_env_flag('WERROR'):
extra_compile_args.append('-Werror')
library_dirs.append(lib_path)
main_compile_args = []
main_libraries = ['shm', 'torch_python']
main_link_args = []
main_sources = ["torch/csrc/stub.cpp"]
if cmake_cache_vars['USE_CUDA']:
library_dirs.append(os.path.dirname(cmake_cache_vars['CUDA_CUDA_LIB']))
if build_type.is_debug():
if IS_WINDOWS:
extra_link_args.append('/DEBUG:FULL')
else:
extra_compile_args += ['-O0', '-g']
extra_link_args += ['-O0', '-g']
if build_type.is_rel_with_deb_info():
if IS_WINDOWS:
extra_link_args.append('/DEBUG:FULL')
else:
extra_compile_args += ['-g']
extra_link_args += ['-g']
def make_relative_rpath(path):
if IS_DARWIN:
return '-Wl,-rpath,@loader_path/' + path
elif IS_WINDOWS:
return ''
else:
return '-Wl,-rpath,$ORIGIN/' + path
################################################################################
# Declare extensions and package
################################################################################
extensions = []
packages = find_packages(exclude=('tools', 'tools.*'))
C = Extension("torch._C",
libraries=main_libraries,
sources=main_sources,
language='c++',
extra_compile_args=main_compile_args + extra_compile_args,
include_dirs=[],
library_dirs=library_dirs,
extra_link_args=extra_link_args + main_link_args +
[make_relative_rpath('lib')])
extensions.append(C)
if not IS_WINDOWS:
DL = Extension("torch._dl", sources=["torch/csrc/dl.c"], language='c')
extensions.append(DL)
# These extensions are built by cmake and copied manually in build_extensions()
# inside the build_ext implementaiton
extensions.append(
Extension(name=str('caffe2.python.caffe2_pybind11_state'),
sources=[]), )
if cmake_cache_vars['USE_CUDA']:
extensions.append(
Extension(name=str('caffe2.python.caffe2_pybind11_state_gpu'),
sources=[]), )
if cmake_cache_vars['USE_ROCM']:
extensions.append(
Extension(name=str('caffe2.python.caffe2_pybind11_state_hip'),
sources=[]), )
cmdclass = {
'build_ext': build_ext,
'clean': clean,
'install': install,
}
entry_points = {
'console_scripts': [
'convert-caffe2-to-onnx = caffe2.python.onnx.bin.conversion:caffe2_to_onnx',
'convert-onnx-to-caffe2 = caffe2.python.onnx.bin.conversion:onnx_to_caffe2',
]
}
return extensions, cmdclass, packages, entry_points
