def install_step(self): """Custom install procedure for EggLib: first build/install C++ library, then build Python library.""" # build/install C++ library cpp_subdir = os.path.join(self.builddir, 'egglib-cpp-%s' % self.version) try: os.chdir(cpp_subdir) except OSError as err: raise EasyBuildError("Failed to move to: %s", err) ConfigureMake.configure_step(self) ConfigureMake.build_step(self) ConfigureMake.install_step(self) # header files and libraries must be found when building Python library for varname, subdir in [('CPATH', 'include'), ('LIBRARY_PATH', 'lib')]: env.setvar(varname, '%s:%s' % (os.path.join(self.installdir, subdir), os.environ.get(varname, ''))) # build/install Python package py_subdir = os.path.join(self.builddir, 'egglib-py-%s' % self.version) try: os.chdir(py_subdir) except OSError as err: raise EasyBuildError("Failed to move to: %s", err) PythonPackage.build_step(self) self.cfg.update('installopts', "--install-lib %s" % os.path.join(self.installdir, self.pylibdir)) self.cfg.update('installopts', "--install-scripts %s" % os.path.join(self.installdir, 'bin')) PythonPackage.install_step(self)
def __init__(self, *args, **kwargs): """ Constructor: initialize via PythonPackage, to ensure everything is set up as needed to build with Python bindings """ PythonPackage.__init__(self, *args, **kwargs) self.with_python_bindings = False
def install_step(self): """ Install libxml2 and install python bindings """ ConfigureMake.install_step(self) try: os.chdir('python') PythonPackage.install_step(self) os.chdir('..') except OSError, err: self.log.error("Failed to install libxml2 Python bindings: %s" % err)
def configure_step(self): """ Configure and Test if python module is loaded """ if not get_software_root('Python'): raise EasyBuildError("Python module not loaded") # We will do the python bindings ourselves so force them off self.cfg.update('configopts', '--without-python') ConfigureMake.configure_step(self) # prepare for installing Python package PythonPackage.prepare_python(self)
def build_step(self): """ Make libxml2 first, then make python bindings """ ConfigureMake.build_step(self) try: os.chdir('python') # set cflags to point to include folder env.setvar('CFLAGS', "-I../include") PythonPackage.build_step(self) os.chdir('..') except OSError, err: self.log.error("Failed to build libxml2 Python bindings: %s" % err)
def extra_options(extra_vars=None): """Easyconfig parameters specific to bundles of Python packages.""" if extra_vars is None: extra_vars = {} # combine custom easyconfig parameters of Bundle & PythonPackage extra_vars = Bundle.extra_options(extra_vars) return PythonPackage.extra_options(extra_vars)
def extra_options(): """Define custom easyconfig parameters for TensorRT.""" # Combine extra variables from Binary and PythonPackage easyblocks extra_vars = Binary.extra_options() extra_vars = PythonPackage.extra_options(extra_vars) return EasyBlock.extra_options(extra_vars)
def configure_step(self): """ Configure and Test if python module is loaded """ if not get_software_root('Python'): self.log.error("Python module not loaded") ConfigureMake.configure_step(self) try: os.chdir('python') PythonPackage.configure_step(self) os.chdir('..') except OSError, err: self.log.error("Failed to configure libxml2 Python bindings: %s" % err)
def __init__(self, *args, **kwargs): """Initialize PythonBundle easyblock.""" super(PythonBundle, self).__init__(*args, **kwargs) self.cfg['exts_defaultclass'] = 'PythonPackage' # need to disable templating to ensure that actual value for exts_default_options is updated... prev_enable_templating = self.cfg.enable_templating self.cfg.enable_templating = False # set default options for extensions according to relevant top-level easyconfig parameters pypkg_keys = PythonPackage.extra_options().keys() for key in pypkg_keys: if key not in self.cfg['exts_default_options']: self.cfg['exts_default_options'][key] = self.cfg[key] self.cfg['exts_default_options']['download_dep_fail'] = True self.log.info("Detection of downloaded extension dependencies is enabled") self.cfg.enable_templating = prev_enable_templating self.log.info("exts_default_options: %s", self.cfg['exts_default_options']) self.pylibdir = None # figure out whether this bundle of Python packages is being installed for multiple Python versions self.multi_python = 'Python' in self.cfg['multi_deps']
def extra_options(): extra_vars = { # see https://developer.nvidia.com/cuda-gpus 'cuda_compute_capabilities': [[], "List of CUDA compute capabilities to build with", CUSTOM], 'with_jemalloc': [True, "Make TensorFlow use jemalloc", CUSTOM], 'with_mkl_dnn': [None, "Make TensorFlow use Intel MKL-DNN (enabled unless cuDNN is used)", CUSTOM], } return PythonPackage.extra_options(extra_vars)
def extra_options(): extra_vars = { # see https://developer.nvidia.com/cuda-gpus 'cuda_compute_capabilities': [[], "List of CUDA compute capabilities to build with", CUSTOM], 'path_filter': [[], "List of patterns to be filtered out in paths in $CPATH and $LIBRARY_PATH", CUSTOM], 'with_jemalloc': [None, "Make TensorFlow use jemalloc (usually enabled by default)", CUSTOM], 'with_mkl_dnn': [None, "Make TensorFlow use Intel MKL-DNN (enabled unless cuDNN is used)", CUSTOM], } return PythonPackage.extra_options(extra_vars)
def __init__(self, *args, **kwargs): """Initialize custom class variables.""" super(EB_MXNet, self).__init__(*args, **kwargs) self.mxnet_src_dir = None self.py_ext = PythonPackage(self, {'name': self.name, 'version': self.version}) self.py_ext.module_generator = self.module_generator self.r_ext = RPackage(self, {'name': self.name, 'version': self.version}) self.r_ext.module_generator = self.module_generator
def make_module_extra(self): """ Add python bindings to the pythonpath """ if self.with_python_bindings: txt = PythonPackage.make_module_extra(self) else: txt = super(EB_libxml2, self).make_module_extra() txt += self.module_generator.prepend_paths('CPATH', [os.path.join('include', 'libxml2')]) return txt
def configure_step(self): """ Configure libxml2 build """ # only build with Python bindings if Python is listed as a dependency python = get_software_root('Python') if python: self.with_python_bindings = True if self.toolchain.name != DUMMY_TOOLCHAIN_NAME: self.cfg.update('configopts', "CC='%s' CXX='%s'" % (os.getenv('CC'), os.getenv('CXX'))) if self.toolchain.options.get('pic', False): self.cfg.update('configopts', '--with-pic') zlib = get_software_root('zlib') if zlib: self.cfg.update('configopts', '--with-zlib=%s' % zlib) # enable building of Python bindings if Python is a dependency (or build them ourselves for old versions) # disable building of Python bindings if Python is not a dependency if self.with_python_bindings and LooseVersion(self.version) >= LooseVersion('2.9.2'): libxml2_pylibdir = os.path.join(self.installdir, self.pylibdir) self.cfg.update('configopts', "--with-python=%s" % os.path.join(python, 'bin', 'python')) self.cfg.update('configopts', "--with-python-install-dir=%s" % libxml2_pylibdir) else: self.cfg.update('configopts', '--without-python') ConfigureMake.configure_step(self) if self.with_python_bindings: # prepare for installing Python package PythonPackage.prepare_python(self) # test using 'make check' (done via test_step) self.cfg['runtest'] = 'check'
def install_step(self): """ Install libxml2 and install python bindings """ ConfigureMake.install_step(self) try: # We can only do the python bindings after the initial installation # since setup.py expects to find the include dir in the installation path # and that only exists after installation os.chdir('python') PythonPackage.configure_step(self) # set cflags to point to include folder for the compilation step to succeed env.setvar('CFLAGS', "-I../include") PythonPackage.build_step(self) PythonPackage.install_step(self) os.chdir('..') except OSError, err: raise EasyBuildError("Failed to install libxml2 Python bindings: %s", err)
def install_step(self): """ Custom install step for libxml2; also build Python bindings ourselves if desired (only for older libxml2 versions """ ConfigureMake.install_step(self) if self.with_python_bindings and LooseVersion(self.version) < LooseVersion('2.9.2'): try: # We can only do the Python bindings after the initial installation # since setup.py expects to find the include dir in the installation path # and that only exists after installation os.chdir('python') PythonPackage.configure_step(self) # set cflags to point to include folder for the compilation step to succeed env.setvar('CFLAGS', "-I../include") PythonPackage.build_step(self) PythonPackage.install_step(self) os.chdir('..') except OSError as err: raise EasyBuildError("Failed to install libxml2 Python bindings: %s", err)
def install_step(self): """ Custom install step for libxml2; also build Python bindings ourselves if desired (only for older libxml2 versions """ ConfigureMake.install_step(self) if self.with_python_bindings and LooseVersion( self.version) < LooseVersion('2.9.2'): try: # We can only do the Python bindings after the initial installation # since setup.py expects to find the include dir in the installation path # and that only exists after installation os.chdir('python') PythonPackage.configure_step(self) # set cflags to point to include folder for the compilation step to succeed env.setvar('CFLAGS', "-I../include") PythonPackage.build_step(self) PythonPackage.install_step(self) os.chdir('..') except OSError as err: raise EasyBuildError( "Failed to install libxml2 Python bindings: %s", err)
def make_module_extra(self): """ Add python bindings to the pythonpath """ return PythonPackage.make_module_extra(self)
def extra_options(extra_vars=None): """Easyconfig parameters specific to Python packages thar are configured/built/installed via CMake""" extra_vars = PythonPackage.extra_options(extra_vars=extra_vars) return CMakeMake.extra_options(extra_vars=extra_vars)
ConfigureMake.configure_step(self) ConfigureMake.build_step(self) ConfigureMake.install_step(self) # header files and libraries must be found when building Python library for varname, subdir in [('CPATH', 'include'), ('LIBRARY_PATH', 'lib')]: env.setvar(varname, '%s:%s' % (os.path.join(self.installdir, subdir), os.environ.get(varname, ''))) # build/install Python package py_subdir = os.path.join(self.builddir, 'egglib-py-%s' % self.version) try: os.chdir(py_subdir) except OSError, err: raise EasyBuildError("Failed to move to: %s", err) PythonPackage.build_step(self) self.cfg.update('installopts', "--install-lib %s" % os.path.join(self.installdir, self.pylibdir)) self.cfg.update('installopts', "--install-scripts %s" % os.path.join(self.installdir, 'bin')) PythonPackage.install_step(self) def sanity_check_step(self): """Custom sanity check for EggLib.""" custom_paths = { 'files': ['bin/egglib', 'lib/libegglib-cpp.a'], 'dirs': ['include/egglib-cpp', self.pylibdir], } super(EB_EggLib, self).sanity_check_step(custom_paths=custom_paths)
def __init__(self, *args, **kwargs): """ Constructor init as a pythonpackage, since that is also an application """ PythonPackage.__init__(self, *args, **kwargs)
def test_step(self, *args, **kwargs): """Test Python package.""" PythonPackage.test_step(self, *args, **kwargs)
def make_module_extra(self): """Add extra Python package module parameters""" return PythonPackage.make_module_extra(self)
def sanity_check_step(self, *args, **kwargs): """ Custom sanity check for Python packages """ return PythonPackage.sanity_check_step(self, *args, **kwargs)
def __init__(self, *args, **kwargs): """Initialize with PythonPackage.""" PythonPackage.__init__(self, *args, **kwargs)
def make_module_extra(self): return PythonPackage.make_module_extra(self)
def __init__(self, *args, **kwargs): """Initialize with PythonPackage.""" PythonPackage.__init__(self, *args, **kwargs) self._lib_ext = None # From CMakeMake.__init__
class EB_MXNet(MakeCp): """Easyblock to build and install MXNet""" @staticmethod def extra_options(extra_vars=None): """Change default values of options""" extra = MakeCp.extra_options() # files_to_copy is not mandatory here extra['files_to_copy'][2] = CUSTOM return extra def __init__(self, *args, **kwargs): """Initialize custom class variables.""" super(EB_MXNet, self).__init__(*args, **kwargs) self.mxnet_src_dir = None self.py_ext = PythonPackage(self, { 'name': self.name, 'version': self.version }) self.py_ext.module_generator = self.module_generator self.r_ext = RPackage(self, { 'name': self.name, 'version': self.version }) self.r_ext.module_generator = self.module_generator def extract_step(self): """ Prepare a combined MXNet source tree. Move all submodules to their right place. """ # Extract everything into separate directories. super(EB_MXNet, self).extract_step() mxnet_dirs = glob.glob(os.path.join(self.builddir, '*mxnet-*')) if len(mxnet_dirs) == 1: self.mxnet_src_dir = mxnet_dirs[0] self.log.debug("MXNet dir is: %s", self.mxnet_src_dir) else: raise EasyBuildError( "Failed to find/isolate MXNet source directory: %s", mxnet_dirs) for srcdir in [ d for d in os.listdir(self.builddir) if d != os.path.basename(self.mxnet_src_dir) ]: submodule, _, _ = srcdir.rpartition('-') newdir = os.path.join(self.mxnet_src_dir, submodule) olddir = os.path.join(self.builddir, srcdir) # first remove empty existing directory remove_dir(newdir) try: shutil.move(olddir, newdir) except IOError as err: raise EasyBuildError("Failed to move %s to %s: %s", olddir, newdir, err) # the nnvm submodules has dmlc-core as a submodule too. Let's put a symlink in place. newdir = os.path.join(self.mxnet_src_dir, "nnvm", "dmlc-core") olddir = os.path.join(self.mxnet_src_dir, "dmlc-core") remove_dir(newdir) symlink(olddir, newdir) def prepare_step(self, *args, **kwargs): """Prepare for building and installing MXNet.""" super(EB_MXNet, self).prepare_step(*args, **kwargs) self.py_ext.prepare_python() def configure_step(self): """Patch 'config.mk' file to use EB stuff""" for (var, env_var) in [('CC', 'CC'), ('CXX', 'CXX'), ('ADD_CFLAGS', 'CFLAGS'), ('ADD_LDFLAGS', 'LDFLAGS')]: self.cfg.update('buildopts', '%s="%s"' % (var, os.getenv(env_var))) toolchain_blas = self.toolchain.definition().get('BLAS', None)[0] if toolchain_blas == 'imkl': blas = "mkl" imkl_version = get_software_version('imkl') if LooseVersion(imkl_version) >= LooseVersion('17'): self.cfg.update('buildopts', 'USE_MKL2017=1') self.cfg.update('buildopts', 'MKLML_ROOT="%s"' % os.getenv("MKLROOT")) elif toolchain_blas in ['ACML', 'ATLAS']: blas = "atlas" elif toolchain_blas == 'OpenBLAS': blas = "openblas" elif toolchain_blas is None: raise EasyBuildError("No BLAS library found in the toolchain") self.cfg.update('buildopts', 'USE_BLAS="%s"' % blas) if get_software_root('NNPACK'): self.cfg.update('buildopts', 'USE_NNPACK=1') super(EB_MXNet, self).configure_step() def install_step(self): """Specify list of files to copy""" self.cfg['files_to_copy'] = [ 'bin', 'include', 'lib', (['dmlc-core/include/dmlc', 'nnvm/include/nnvm'], 'include') ] super(EB_MXNet, self).install_step() def extensions_step(self): """Build & Install both Python and R extension""" # we start with the python bindings self.py_ext.src = os.path.join(self.mxnet_src_dir, "python") change_dir(self.py_ext.src) self.py_ext.prerun() self.py_ext.run(unpack_src=False) self.py_ext.postrun() # next up, the R bindings self.r_ext.src = os.path.join(self.mxnet_src_dir, "R-package") change_dir(self.r_ext.src) mkdir("inst") symlink(os.path.join(self.installdir, "lib"), os.path.join("inst", "libs")) symlink(os.path.join(self.installdir, "include"), os.path.join("inst", "include")) # MXNet doesn't provide a list of its R dependencies by default write_file("NAMESPACE", R_NAMESPACE) change_dir(self.mxnet_src_dir) self.r_ext.prerun() # MXNet is just weird. To install the R extension, we have to: # - First install the extension like it is # - Let R export the extension again. By doing this, all the dependencies get # correctly filled and some mappings are done # - Reinstal the exported version self.r_ext.run() run_cmd( "R_LIBS=%s Rscript -e \"require(mxnet); mxnet:::mxnet.export(\\\"R-package\\\")\"" % self.installdir) change_dir(self.r_ext.src) self.r_ext.run() self.r_ext.postrun() def sanity_check_step(self): """Check for main library files for MXNet""" custom_paths = { 'files': ['lib/libmxnet.a', 'lib/libmxnet.%s' % get_shared_lib_ext()], 'dirs': [], } super(EB_MXNet, self).sanity_check_step(custom_paths=custom_paths) # for the extension we are doing the loading of the fake module ourself try: fake_mod_data = self.load_fake_module() except EasyBuildError as err: raise EasyBuildError("Loading fake module failed: %s", err) if not self.py_ext.sanity_check_step(): raise EasyBuildError( "The sanity check for the Python bindings failed") self.r_ext.options['modulename'] = self.name.lower() if not self.r_ext.sanity_check_step(): raise EasyBuildError("The sanity check for the R bindings failed") self.clean_up_fake_module(fake_mod_data) def make_module_extra(self, *args, **kwargs): """Custom variables for MXNet module.""" txt = super(EB_MXNet, self).make_module_extra(*args, **kwargs) for path in self.py_ext.all_pylibdirs: fullpath = os.path.join(self.installdir, path) # only extend $PYTHONPATH with existing, non-empty directories if os.path.exists(fullpath) and os.listdir(fullpath): txt += self.module_generator.prepend_paths('PYTHONPATH', path) txt += self.module_generator.prepend_paths( "R_LIBS", ['']) # prepend R_LIBS with install path return txt
def configure_step(self, *args, **kwargs): """Main configuration using cmake""" PythonPackage.configure_step(self, *args, **kwargs) return CMakeMake.configure_step(self, *args, **kwargs)
def prepare_step(self): """Set pylibdir""" self.pylibdir = 'lib' PythonPackage.prepare_step(self)
def extra_options(): """Extra easyconfig parameters for Python packages being installed with python configure/make/make install.""" extra = PythonPackage.extra_options() return ConfigureMake.extra_options(extra_vars=extra)
def configure_step(self): """Configure EggLib build/install procedure.""" # only need to configure Python library here, configuration of C++ library is done in install step PythonPackage.configure_step(self)
def configure_step(self, *args, **kwargs): """Configure build using 'python configure'.""" PythonPackage.configure_step(self, *args, **kwargs) cmd = ' '.join([self.cfg['preconfigopts'], self.python_cmd, self.cfg['configopts']]) run_cmd(cmd, log_all=True)
def configure_step(self): PythonPackage.configure_step(self) return CMakeMake.configure_step(self)
ConfigureMake.install_step(self) # header files and libraries must be found when building Python library for varname, subdir in [('CPATH', 'include'), ('LIBRARY_PATH', 'lib')]: env.setvar( varname, '%s:%s' % (os.path.join( self.installdir, subdir), os.environ.get(varname, ''))) # build/install Python package py_subdir = os.path.join(self.builddir, 'egglib-py-%s' % self.version) try: os.chdir(py_subdir) except OSError, err: raise EasyBuildError("Failed to move to: %s", err) PythonPackage.build_step(self) self.cfg.update( 'installopts', "--install-lib %s" % os.path.join(self.installdir, self.pylibdir)) self.cfg.update( 'installopts', "--install-scripts %s" % os.path.join(self.installdir, 'bin')) PythonPackage.install_step(self) def sanity_check_step(self): """Custom sanity check for EggLib.""" custom_paths = { 'files': ['bin/egglib', 'lib/libegglib-cpp.a'], 'dirs': ['include/egglib-cpp', self.pylibdir],
def __init__(self, *args, **kwargs): PythonPackage.__init__(self, *args, **kwargs) MakeCp.__init__(self, *args, **kwargs)
def extra_options(extra_vars=None): """Easyconfig parameters specific to libxml2.""" extra_vars = ConfigureMake.extra_options() return PythonPackage.extra_options(extra_vars=extra_vars)
def install_step(self): files_to_copy = self.cfg.get('files_to_copy', []) self.log.info("Starting install_step with files_to_copy: %s" % files_to_copy) PythonPackage.install_step(self) return MakeCp.install_step(self)