def install_step(self):
"""Custom install procedure for EggLib: first build/install C++ library, then build Python library."""
# build/install C++ library
cpp_subdir = os.path.join(self.builddir, 'egglib-cpp-%s' % self.version)
try:
os.chdir(cpp_subdir)
except OSError as err:
raise EasyBuildError("Failed to move to: %s", err)
ConfigureMake.configure_step(self)
ConfigureMake.build_step(self)
ConfigureMake.install_step(self)
# header files and libraries must be found when building Python library
for varname, subdir in [('CPATH', 'include'), ('LIBRARY_PATH', 'lib')]:
env.setvar(varname, '%s:%s' % (os.path.join(self.installdir, subdir), os.environ.get(varname, '')))
# build/install Python package
py_subdir = os.path.join(self.builddir, 'egglib-py-%s' % self.version)
try:
os.chdir(py_subdir)
except OSError as err:
raise EasyBuildError("Failed to move to: %s", err)
PythonPackage.build_step(self)
self.cfg.update('installopts', "--install-lib %s" % os.path.join(self.installdir, self.pylibdir))
self.cfg.update('installopts', "--install-scripts %s" % os.path.join(self.installdir, 'bin'))
PythonPackage.install_step(self)
def __init__(self, *args, **kwargs):
"""
Constructor: initialize via PythonPackage,
to ensure everything is set up as needed to build with Python bindings
"""
PythonPackage.__init__(self, *args, **kwargs)
self.with_python_bindings = False
def install_step(self):
"""
Install libxml2 and install python bindings
"""
ConfigureMake.install_step(self)
try:
os.chdir('python')
PythonPackage.install_step(self)
os.chdir('..')
except OSError, err:
self.log.error("Failed to install libxml2 Python bindings: %s" % err)
def configure_step(self):
"""
Configure and
Test if python module is loaded
"""
if not get_software_root('Python'):
raise EasyBuildError("Python module not loaded")
# We will do the python bindings ourselves so force them off
self.cfg.update('configopts', '--without-python')
ConfigureMake.configure_step(self)
# prepare for installing Python package
PythonPackage.prepare_python(self)
def build_step(self):
"""
Make libxml2 first, then make python bindings
"""
ConfigureMake.build_step(self)
try:
os.chdir('python')
# set cflags to point to include folder
env.setvar('CFLAGS', "-I../include")
PythonPackage.build_step(self)
os.chdir('..')
except OSError, err:
self.log.error("Failed to build libxml2 Python bindings: %s" % err)
def extra_options(extra_vars=None):
"""Easyconfig parameters specific to bundles of Python packages."""
if extra_vars is None:
extra_vars = {}
# combine custom easyconfig parameters of Bundle & PythonPackage
extra_vars = Bundle.extra_options(extra_vars)
return PythonPackage.extra_options(extra_vars)
def extra_options():
"""Define custom easyconfig parameters for TensorRT."""
# Combine extra variables from Binary and PythonPackage easyblocks
extra_vars = Binary.extra_options()
extra_vars = PythonPackage.extra_options(extra_vars)
return EasyBlock.extra_options(extra_vars)
def configure_step(self):
"""
Configure and
Test if python module is loaded
"""
if not get_software_root('Python'):
self.log.error("Python module not loaded")
ConfigureMake.configure_step(self)
try:
os.chdir('python')
PythonPackage.configure_step(self)
os.chdir('..')
except OSError, err:
self.log.error("Failed to configure libxml2 Python bindings: %s" % err)
def __init__(self, *args, **kwargs):
"""Initialize PythonBundle easyblock."""
super(PythonBundle, self).__init__(*args, **kwargs)
self.cfg['exts_defaultclass'] = 'PythonPackage'
# need to disable templating to ensure that actual value for exts_default_options is updated...
prev_enable_templating = self.cfg.enable_templating
self.cfg.enable_templating = False
# set default options for extensions according to relevant top-level easyconfig parameters
pypkg_keys = PythonPackage.extra_options().keys()
for key in pypkg_keys:
if key not in self.cfg['exts_default_options']:
self.cfg['exts_default_options'][key] = self.cfg[key]
self.cfg['exts_default_options']['download_dep_fail'] = True
self.log.info("Detection of downloaded extension dependencies is enabled")
self.cfg.enable_templating = prev_enable_templating
self.log.info("exts_default_options: %s", self.cfg['exts_default_options'])
self.pylibdir = None
# figure out whether this bundle of Python packages is being installed for multiple Python versions
self.multi_python = 'Python' in self.cfg['multi_deps']
def extra_options():
extra_vars = {
# see https://developer.nvidia.com/cuda-gpus
'cuda_compute_capabilities': [[], "List of CUDA compute capabilities to build with", CUSTOM],
'with_jemalloc': [True, "Make TensorFlow use jemalloc", CUSTOM],
'with_mkl_dnn': [None, "Make TensorFlow use Intel MKL-DNN (enabled unless cuDNN is used)", CUSTOM],
}
return PythonPackage.extra_options(extra_vars)
def extra_options():
extra_vars = {
# see https://developer.nvidia.com/cuda-gpus
'cuda_compute_capabilities': [[], "List of CUDA compute capabilities to build with", CUSTOM],
'path_filter': [[], "List of patterns to be filtered out in paths in $CPATH and $LIBRARY_PATH", CUSTOM],
'with_jemalloc': [None, "Make TensorFlow use jemalloc (usually enabled by default)", CUSTOM],
'with_mkl_dnn': [None, "Make TensorFlow use Intel MKL-DNN (enabled unless cuDNN is used)", CUSTOM],
}
return PythonPackage.extra_options(extra_vars)
def __init__(self, *args, **kwargs):
"""Initialize custom class variables."""
super(EB_MXNet, self).__init__(*args, **kwargs)
self.mxnet_src_dir = None
self.py_ext = PythonPackage(self, {'name': self.name, 'version': self.version})
self.py_ext.module_generator = self.module_generator
self.r_ext = RPackage(self, {'name': self.name, 'version': self.version})
self.r_ext.module_generator = self.module_generator
def make_module_extra(self):
"""
Add python bindings to the pythonpath
"""
if self.with_python_bindings:
txt = PythonPackage.make_module_extra(self)
else:
txt = super(EB_libxml2, self).make_module_extra()
txt += self.module_generator.prepend_paths('CPATH', [os.path.join('include', 'libxml2')])
return txt
def configure_step(self):
"""
Configure libxml2 build
"""
# only build with Python bindings if Python is listed as a dependency
python = get_software_root('Python')
if python:
self.with_python_bindings = True
if self.toolchain.name != DUMMY_TOOLCHAIN_NAME:
self.cfg.update('configopts', "CC='%s' CXX='%s'" % (os.getenv('CC'), os.getenv('CXX')))
if self.toolchain.options.get('pic', False):
self.cfg.update('configopts', '--with-pic')
zlib = get_software_root('zlib')
if zlib:
self.cfg.update('configopts', '--with-zlib=%s' % zlib)
# enable building of Python bindings if Python is a dependency (or build them ourselves for old versions)
# disable building of Python bindings if Python is not a dependency
if self.with_python_bindings and LooseVersion(self.version) >= LooseVersion('2.9.2'):
libxml2_pylibdir = os.path.join(self.installdir, self.pylibdir)
self.cfg.update('configopts', "--with-python=%s" % os.path.join(python, 'bin', 'python'))
self.cfg.update('configopts', "--with-python-install-dir=%s" % libxml2_pylibdir)
else:
self.cfg.update('configopts', '--without-python')
ConfigureMake.configure_step(self)
if self.with_python_bindings:
# prepare for installing Python package
PythonPackage.prepare_python(self)
# test using 'make check' (done via test_step)
self.cfg['runtest'] = 'check'
def install_step(self):
"""
Install libxml2 and install python bindings
"""
ConfigureMake.install_step(self)
try:
# We can only do the python bindings after the initial installation
# since setup.py expects to find the include dir in the installation path
# and that only exists after installation
os.chdir('python')
PythonPackage.configure_step(self)
# set cflags to point to include folder for the compilation step to succeed
env.setvar('CFLAGS', "-I../include")
PythonPackage.build_step(self)
PythonPackage.install_step(self)
os.chdir('..')
except OSError, err:
raise EasyBuildError("Failed to install libxml2 Python bindings: %s", err)
def install_step(self):
"""
Custom install step for libxml2;
also build Python bindings ourselves if desired (only for older libxml2 versions
"""
ConfigureMake.install_step(self)
if self.with_python_bindings and LooseVersion(self.version) < LooseVersion('2.9.2'):
try:
# We can only do the Python bindings after the initial installation
# since setup.py expects to find the include dir in the installation path
# and that only exists after installation
os.chdir('python')
PythonPackage.configure_step(self)
# set cflags to point to include folder for the compilation step to succeed
env.setvar('CFLAGS', "-I../include")
PythonPackage.build_step(self)
PythonPackage.install_step(self)
os.chdir('..')
except OSError as err:
raise EasyBuildError("Failed to install libxml2 Python bindings: %s", err)
def install_step(self):
"""
Custom install step for libxml2;
also build Python bindings ourselves if desired (only for older libxml2 versions
"""
ConfigureMake.install_step(self)
if self.with_python_bindings and LooseVersion(
self.version) < LooseVersion('2.9.2'):
try:
# We can only do the Python bindings after the initial installation
# since setup.py expects to find the include dir in the installation path
# and that only exists after installation
os.chdir('python')
PythonPackage.configure_step(self)
# set cflags to point to include folder for the compilation step to succeed
env.setvar('CFLAGS', "-I../include")
PythonPackage.build_step(self)
PythonPackage.install_step(self)
os.chdir('..')
except OSError as err:
raise EasyBuildError(
"Failed to install libxml2 Python bindings: %s", err)
def make_module_extra(self):
"""
Add python bindings to the pythonpath
"""
return PythonPackage.make_module_extra(self)
def extra_options(extra_vars=None):
"""Easyconfig parameters specific to Python packages thar are configured/built/installed via CMake"""
extra_vars = PythonPackage.extra_options(extra_vars=extra_vars)
return CMakeMake.extra_options(extra_vars=extra_vars)
ConfigureMake.configure_step(self)
ConfigureMake.build_step(self)
ConfigureMake.install_step(self)
# header files and libraries must be found when building Python library
for varname, subdir in [('CPATH', 'include'), ('LIBRARY_PATH', 'lib')]:
env.setvar(varname, '%s:%s' % (os.path.join(self.installdir, subdir), os.environ.get(varname, '')))
# build/install Python package
py_subdir = os.path.join(self.builddir, 'egglib-py-%s' % self.version)
try:
os.chdir(py_subdir)
except OSError, err:
raise EasyBuildError("Failed to move to: %s", err)
PythonPackage.build_step(self)
self.cfg.update('installopts', "--install-lib %s" % os.path.join(self.installdir, self.pylibdir))
self.cfg.update('installopts', "--install-scripts %s" % os.path.join(self.installdir, 'bin'))
PythonPackage.install_step(self)
def sanity_check_step(self):
"""Custom sanity check for EggLib."""
custom_paths = {
'files': ['bin/egglib', 'lib/libegglib-cpp.a'],
'dirs': ['include/egglib-cpp', self.pylibdir],
}
super(EB_EggLib, self).sanity_check_step(custom_paths=custom_paths)
def __init__(self, *args, **kwargs):
"""
Constructor
init as a pythonpackage, since that is also an application
"""
PythonPackage.__init__(self, *args, **kwargs)
def test_step(self, *args, **kwargs):
"""Test Python package."""
PythonPackage.test_step(self, *args, **kwargs)
def make_module_extra(self):
"""Add extra Python package module parameters"""
return PythonPackage.make_module_extra(self)
def sanity_check_step(self, *args, **kwargs):
"""
Custom sanity check for Python packages
"""
return PythonPackage.sanity_check_step(self, *args, **kwargs)
def __init__(self, *args, **kwargs):
"""Initialize with PythonPackage."""
PythonPackage.__init__(self, *args, **kwargs)
def make_module_extra(self):
return PythonPackage.make_module_extra(self)
def __init__(self, *args, **kwargs):
"""Initialize with PythonPackage."""
PythonPackage.__init__(self, *args, **kwargs)
def __init__(self, *args, **kwargs):
"""Initialize with PythonPackage."""
PythonPackage.__init__(self, *args, **kwargs)
self._lib_ext = None # From CMakeMake.__init__
class EB_MXNet(MakeCp):
"""Easyblock to build and install MXNet"""
@staticmethod
def extra_options(extra_vars=None):
"""Change default values of options"""
extra = MakeCp.extra_options()
# files_to_copy is not mandatory here
extra['files_to_copy'][2] = CUSTOM
return extra
def __init__(self, *args, **kwargs):
"""Initialize custom class variables."""
super(EB_MXNet, self).__init__(*args, **kwargs)
self.mxnet_src_dir = None
self.py_ext = PythonPackage(self, {
'name': self.name,
'version': self.version
})
self.py_ext.module_generator = self.module_generator
self.r_ext = RPackage(self, {
'name': self.name,
'version': self.version
})
self.r_ext.module_generator = self.module_generator
def extract_step(self):
"""
Prepare a combined MXNet source tree. Move all submodules
to their right place.
"""
# Extract everything into separate directories.
super(EB_MXNet, self).extract_step()
mxnet_dirs = glob.glob(os.path.join(self.builddir, '*mxnet-*'))
if len(mxnet_dirs) == 1:
self.mxnet_src_dir = mxnet_dirs[0]
self.log.debug("MXNet dir is: %s", self.mxnet_src_dir)
else:
raise EasyBuildError(
"Failed to find/isolate MXNet source directory: %s",
mxnet_dirs)
for srcdir in [
d for d in os.listdir(self.builddir)
if d != os.path.basename(self.mxnet_src_dir)
]:
submodule, _, _ = srcdir.rpartition('-')
newdir = os.path.join(self.mxnet_src_dir, submodule)
olddir = os.path.join(self.builddir, srcdir)
# first remove empty existing directory
remove_dir(newdir)
try:
shutil.move(olddir, newdir)
except IOError as err:
raise EasyBuildError("Failed to move %s to %s: %s", olddir,
newdir, err)
# the nnvm submodules has dmlc-core as a submodule too. Let's put a symlink in place.
newdir = os.path.join(self.mxnet_src_dir, "nnvm", "dmlc-core")
olddir = os.path.join(self.mxnet_src_dir, "dmlc-core")
remove_dir(newdir)
symlink(olddir, newdir)
def prepare_step(self, *args, **kwargs):
"""Prepare for building and installing MXNet."""
super(EB_MXNet, self).prepare_step(*args, **kwargs)
self.py_ext.prepare_python()
def configure_step(self):
"""Patch 'config.mk' file to use EB stuff"""
for (var, env_var) in [('CC', 'CC'), ('CXX', 'CXX'),
('ADD_CFLAGS', 'CFLAGS'),
('ADD_LDFLAGS', 'LDFLAGS')]:
self.cfg.update('buildopts', '%s="%s"' % (var, os.getenv(env_var)))
toolchain_blas = self.toolchain.definition().get('BLAS', None)[0]
if toolchain_blas == 'imkl':
blas = "mkl"
imkl_version = get_software_version('imkl')
if LooseVersion(imkl_version) >= LooseVersion('17'):
self.cfg.update('buildopts', 'USE_MKL2017=1')
self.cfg.update('buildopts',
'MKLML_ROOT="%s"' % os.getenv("MKLROOT"))
elif toolchain_blas in ['ACML', 'ATLAS']:
blas = "atlas"
elif toolchain_blas == 'OpenBLAS':
blas = "openblas"
elif toolchain_blas is None:
raise EasyBuildError("No BLAS library found in the toolchain")
self.cfg.update('buildopts', 'USE_BLAS="%s"' % blas)
if get_software_root('NNPACK'):
self.cfg.update('buildopts', 'USE_NNPACK=1')
super(EB_MXNet, self).configure_step()
def install_step(self):
"""Specify list of files to copy"""
self.cfg['files_to_copy'] = [
'bin', 'include', 'lib',
(['dmlc-core/include/dmlc', 'nnvm/include/nnvm'], 'include')
]
super(EB_MXNet, self).install_step()
def extensions_step(self):
"""Build & Install both Python and R extension"""
# we start with the python bindings
self.py_ext.src = os.path.join(self.mxnet_src_dir, "python")
change_dir(self.py_ext.src)
self.py_ext.prerun()
self.py_ext.run(unpack_src=False)
self.py_ext.postrun()
# next up, the R bindings
self.r_ext.src = os.path.join(self.mxnet_src_dir, "R-package")
change_dir(self.r_ext.src)
mkdir("inst")
symlink(os.path.join(self.installdir, "lib"),
os.path.join("inst", "libs"))
symlink(os.path.join(self.installdir, "include"),
os.path.join("inst", "include"))
# MXNet doesn't provide a list of its R dependencies by default
write_file("NAMESPACE", R_NAMESPACE)
change_dir(self.mxnet_src_dir)
self.r_ext.prerun()
# MXNet is just weird. To install the R extension, we have to:
# - First install the extension like it is
# - Let R export the extension again. By doing this, all the dependencies get
# correctly filled and some mappings are done
# - Reinstal the exported version
self.r_ext.run()
run_cmd(
"R_LIBS=%s Rscript -e \"require(mxnet); mxnet:::mxnet.export(\\\"R-package\\\")\""
% self.installdir)
change_dir(self.r_ext.src)
self.r_ext.run()
self.r_ext.postrun()
def sanity_check_step(self):
"""Check for main library files for MXNet"""
custom_paths = {
'files':
['lib/libmxnet.a',
'lib/libmxnet.%s' % get_shared_lib_ext()],
'dirs': [],
}
super(EB_MXNet, self).sanity_check_step(custom_paths=custom_paths)
# for the extension we are doing the loading of the fake module ourself
try:
fake_mod_data = self.load_fake_module()
except EasyBuildError as err:
raise EasyBuildError("Loading fake module failed: %s", err)
if not self.py_ext.sanity_check_step():
raise EasyBuildError(
"The sanity check for the Python bindings failed")
self.r_ext.options['modulename'] = self.name.lower()
if not self.r_ext.sanity_check_step():
raise EasyBuildError("The sanity check for the R bindings failed")
self.clean_up_fake_module(fake_mod_data)
def make_module_extra(self, *args, **kwargs):
"""Custom variables for MXNet module."""
txt = super(EB_MXNet, self).make_module_extra(*args, **kwargs)
for path in self.py_ext.all_pylibdirs:
fullpath = os.path.join(self.installdir, path)
# only extend $PYTHONPATH with existing, non-empty directories
if os.path.exists(fullpath) and os.listdir(fullpath):
txt += self.module_generator.prepend_paths('PYTHONPATH', path)
txt += self.module_generator.prepend_paths(
"R_LIBS", ['']) # prepend R_LIBS with install path
return txt
def make_module_extra(self):
"""
Add python bindings to the pythonpath
"""
return PythonPackage.make_module_extra(self)
def configure_step(self, *args, **kwargs):
"""Main configuration using cmake"""
PythonPackage.configure_step(self, *args, **kwargs)
return CMakeMake.configure_step(self, *args, **kwargs)
def prepare_step(self):
"""Set pylibdir"""
self.pylibdir = 'lib'
PythonPackage.prepare_step(self)
def extra_options():
"""Extra easyconfig parameters for Python packages being installed with python configure/make/make install."""
extra = PythonPackage.extra_options()
return ConfigureMake.extra_options(extra_vars=extra)
def configure_step(self):
"""Configure EggLib build/install procedure."""
# only need to configure Python library here, configuration of C++ library is done in install step
PythonPackage.configure_step(self)
def configure_step(self, *args, **kwargs):
"""Configure build using 'python configure'."""
PythonPackage.configure_step(self, *args, **kwargs)
cmd = ' '.join([self.cfg['preconfigopts'], self.python_cmd, self.cfg['configopts']])
run_cmd(cmd, log_all=True)
def configure_step(self):
PythonPackage.configure_step(self)
return CMakeMake.configure_step(self)
def sanity_check_step(self, *args, **kwargs):
"""
Custom sanity check for Python packages
"""
return PythonPackage.sanity_check_step(self, *args, **kwargs)
def __init__(self, *args, **kwargs):
"""
Constructor
init as a pythonpackage, since that is also an application
"""
PythonPackage.__init__(self, *args, **kwargs)
ConfigureMake.install_step(self)
# header files and libraries must be found when building Python library
for varname, subdir in [('CPATH', 'include'), ('LIBRARY_PATH', 'lib')]:
env.setvar(
varname, '%s:%s' % (os.path.join(
self.installdir, subdir), os.environ.get(varname, '')))
# build/install Python package
py_subdir = os.path.join(self.builddir, 'egglib-py-%s' % self.version)
try:
os.chdir(py_subdir)
except OSError, err:
raise EasyBuildError("Failed to move to: %s", err)
PythonPackage.build_step(self)
self.cfg.update(
'installopts',
"--install-lib %s" % os.path.join(self.installdir, self.pylibdir))
self.cfg.update(
'installopts',
"--install-scripts %s" % os.path.join(self.installdir, 'bin'))
PythonPackage.install_step(self)
def sanity_check_step(self):
"""Custom sanity check for EggLib."""
custom_paths = {
'files': ['bin/egglib', 'lib/libegglib-cpp.a'],
'dirs': ['include/egglib-cpp', self.pylibdir],
def configure_step(self, *args, **kwargs):
"""Configure build using 'python configure'."""
PythonPackage.configure_step(self, *args, **kwargs)
cmd = ' '.join([self.cfg['preconfigopts'], self.python_cmd, self.cfg['configopts']])
run_cmd(cmd, log_all=True)
def __init__(self, *args, **kwargs):
PythonPackage.__init__(self, *args, **kwargs)
MakeCp.__init__(self, *args, **kwargs)
def extra_options(extra_vars=None):
"""Easyconfig parameters specific to libxml2."""
extra_vars = ConfigureMake.extra_options()
return PythonPackage.extra_options(extra_vars=extra_vars)
def install_step(self):
files_to_copy = self.cfg.get('files_to_copy', [])
self.log.info("Starting install_step with files_to_copy: %s" %
files_to_copy)
PythonPackage.install_step(self)
return MakeCp.install_step(self)
def configure_step(self):
"""Configure EggLib build/install procedure."""
# only need to configure Python library here, configuration of C++ library is done in install step
PythonPackage.configure_step(self)
def extra_options():
"""Extra easyconfig parameters for Python packages being installed with python configure/make/make install."""
extra = PythonPackage.extra_options()
return ConfigureMake.extra_options(extra_vars=extra)
def extra_options(extra_vars=None):
"""Easyconfig parameters specific to libxml2."""
extra_vars = ConfigureMake.extra_options()
return PythonPackage.extra_options(extra_vars=extra_vars)
def test_step(self, *args, **kwargs):
"""Test Python package."""
PythonPackage.test_step(self, *args, **kwargs)
def extra_options(extra_vars=None):
"""Easyconfig parameters specific to Python packages thar are configured/built/installed via CMake"""
extra_vars = PythonPackage.extra_options(extra_vars=extra_vars)
return CMakeMake.extra_options(extra_vars=extra_vars)
def make_module_extra(self):
"""Add extra Python package module parameters"""
return PythonPackage.make_module_extra(self)
def configure_step(self, *args, **kwargs):
"""Main configuration using cmake"""
PythonPackage.configure_step(self, *args, **kwargs)
return CMakeMake.configure_step(self, *args, **kwargs)