def configure_step(self): """ Configure and Test if python module is loaded """ if not get_software_root('Python'): raise EasyBuildError("Python module not loaded") # We will do the python bindings ourselves so force them off self.cfg.update('configopts', '--without-python') ConfigureMake.configure_step(self) # prepare for installing Python package PythonPackage.prepare_python(self)
def configure_step(self): """ Configure and Test if python module is loaded """ if not get_software_root('Python'): raise EasyBuildError("Python module not loaded") # We will do the python bindings ourselves so force them off self.cfg.update('configopts', '--without-python') ConfigureMake.configure_step(self) # prepare for installing Python package PythonPackage.prepare_python(self)
def configure_step(self): """ Configure libxml2 build """ # only build with Python bindings if Python is listed as a dependency python = get_software_root('Python') if python: self.with_python_bindings = True self.require_python = True if not self.toolchain.is_system_toolchain(): self.cfg.update( 'configopts', "CC='%s' CXX='%s'" % (os.getenv('CC'), os.getenv('CXX'))) if self.toolchain.options.get('pic', False): self.cfg.update('configopts', '--with-pic') zlib = get_software_root('zlib') if zlib: self.cfg.update('configopts', '--with-zlib=%s' % zlib) xz_root = get_software_root('XZ') if xz_root: self.cfg.update('configopts', '--with-lzma=%s' % xz_root) # enable building of Python bindings if Python is a dependency (or build them ourselves for old versions) # disable building of Python bindings if Python is not a dependency if self.with_python_bindings and LooseVersion( self.version) >= LooseVersion('2.9.2'): libxml2_pylibdir = os.path.join(self.installdir, self.pylibdir) self.cfg.update( 'configopts', "--with-python=%s" % os.path.join(python, 'bin', 'python')) self.cfg.update('configopts', "--with-python-install-dir=%s" % libxml2_pylibdir) else: self.cfg.update('configopts', '--without-python') ConfigureMake.configure_step(self) if self.with_python_bindings: # prepare for installing Python package PythonPackage.prepare_python(self) # test using 'make check' (done via test_step) self.cfg['runtest'] = 'check'
def configure_step(self): """ Configure libxml2 build """ # only build with Python bindings if Python is listed as a dependency python = get_software_root('Python') if python: self.with_python_bindings = True if self.toolchain.name != DUMMY_TOOLCHAIN_NAME: self.cfg.update('configopts', "CC='%s' CXX='%s'" % (os.getenv('CC'), os.getenv('CXX'))) if self.toolchain.options.get('pic', False): self.cfg.update('configopts', '--with-pic') zlib = get_software_root('zlib') if zlib: self.cfg.update('configopts', '--with-zlib=%s' % zlib) # enable building of Python bindings if Python is a dependency (or build them ourselves for old versions) # disable building of Python bindings if Python is not a dependency if self.with_python_bindings and LooseVersion(self.version) >= LooseVersion('2.9.2'): libxml2_pylibdir = os.path.join(self.installdir, self.pylibdir) self.cfg.update('configopts', "--with-python=%s" % os.path.join(python, 'bin', 'python')) self.cfg.update('configopts', "--with-python-install-dir=%s" % libxml2_pylibdir) else: self.cfg.update('configopts', '--without-python') ConfigureMake.configure_step(self) if self.with_python_bindings: # prepare for installing Python package PythonPackage.prepare_python(self) # test using 'make check' (done via test_step) self.cfg['runtest'] = 'check'
class EB_MXNet(MakeCp): """Easyblock to build and install MXNet""" @staticmethod def extra_options(extra_vars=None): """Change default values of options""" extra = MakeCp.extra_options() # files_to_copy is not mandatory here extra['files_to_copy'][2] = CUSTOM return extra def __init__(self, *args, **kwargs): """Initialize custom class variables.""" super(EB_MXNet, self).__init__(*args, **kwargs) self.mxnet_src_dir = None self.py_ext = PythonPackage(self, { 'name': self.name, 'version': self.version }) self.py_ext.module_generator = self.module_generator self.r_ext = RPackage(self, { 'name': self.name, 'version': self.version }) self.r_ext.module_generator = self.module_generator def extract_step(self): """ Prepare a combined MXNet source tree. Move all submodules to their right place. """ # Extract everything into separate directories. super(EB_MXNet, self).extract_step() mxnet_dirs = glob.glob(os.path.join(self.builddir, '*mxnet-*')) if len(mxnet_dirs) == 1: self.mxnet_src_dir = mxnet_dirs[0] self.log.debug("MXNet dir is: %s", self.mxnet_src_dir) else: raise EasyBuildError( "Failed to find/isolate MXNet source directory: %s", mxnet_dirs) for srcdir in [ d for d in os.listdir(self.builddir) if d != os.path.basename(self.mxnet_src_dir) ]: submodule, _, _ = srcdir.rpartition('-') newdir = os.path.join(self.mxnet_src_dir, submodule) olddir = os.path.join(self.builddir, srcdir) # first remove empty existing directory remove_dir(newdir) try: shutil.move(olddir, newdir) except IOError as err: raise EasyBuildError("Failed to move %s to %s: %s", olddir, newdir, err) # the nnvm submodules has dmlc-core as a submodule too. Let's put a symlink in place. newdir = os.path.join(self.mxnet_src_dir, "nnvm", "dmlc-core") olddir = os.path.join(self.mxnet_src_dir, "dmlc-core") remove_dir(newdir) symlink(olddir, newdir) def prepare_step(self, *args, **kwargs): """Prepare for building and installing MXNet.""" super(EB_MXNet, self).prepare_step(*args, **kwargs) self.py_ext.prepare_python() def configure_step(self): """Patch 'config.mk' file to use EB stuff""" for (var, env_var) in [('CC', 'CC'), ('CXX', 'CXX'), ('ADD_CFLAGS', 'CFLAGS'), ('ADD_LDFLAGS', 'LDFLAGS')]: self.cfg.update('buildopts', '%s="%s"' % (var, os.getenv(env_var))) toolchain_blas = self.toolchain.definition().get('BLAS', None)[0] if toolchain_blas == 'imkl': blas = "mkl" imkl_version = get_software_version('imkl') if LooseVersion(imkl_version) >= LooseVersion('17'): self.cfg.update('buildopts', 'USE_MKL2017=1') self.cfg.update('buildopts', 'MKLML_ROOT="%s"' % os.getenv("MKLROOT")) elif toolchain_blas in ['ACML', 'ATLAS']: blas = "atlas" elif toolchain_blas == 'OpenBLAS': blas = "openblas" elif toolchain_blas is None: raise EasyBuildError("No BLAS library found in the toolchain") self.cfg.update('buildopts', 'USE_BLAS="%s"' % blas) if get_software_root('NNPACK'): self.cfg.update('buildopts', 'USE_NNPACK=1') super(EB_MXNet, self).configure_step() def install_step(self): """Specify list of files to copy""" self.cfg['files_to_copy'] = [ 'bin', 'include', 'lib', (['dmlc-core/include/dmlc', 'nnvm/include/nnvm'], 'include') ] super(EB_MXNet, self).install_step() def extensions_step(self): """Build & Install both Python and R extension""" # we start with the python bindings self.py_ext.src = os.path.join(self.mxnet_src_dir, "python") change_dir(self.py_ext.src) self.py_ext.prerun() self.py_ext.run(unpack_src=False) self.py_ext.postrun() # next up, the R bindings self.r_ext.src = os.path.join(self.mxnet_src_dir, "R-package") change_dir(self.r_ext.src) mkdir("inst") symlink(os.path.join(self.installdir, "lib"), os.path.join("inst", "libs")) symlink(os.path.join(self.installdir, "include"), os.path.join("inst", "include")) # MXNet doesn't provide a list of its R dependencies by default write_file("NAMESPACE", R_NAMESPACE) change_dir(self.mxnet_src_dir) self.r_ext.prerun() # MXNet is just weird. To install the R extension, we have to: # - First install the extension like it is # - Let R export the extension again. By doing this, all the dependencies get # correctly filled and some mappings are done # - Reinstal the exported version self.r_ext.run() run_cmd( "R_LIBS=%s Rscript -e \"require(mxnet); mxnet:::mxnet.export(\\\"R-package\\\")\"" % self.installdir) change_dir(self.r_ext.src) self.r_ext.run() self.r_ext.postrun() def sanity_check_step(self): """Check for main library files for MXNet""" custom_paths = { 'files': ['lib/libmxnet.a', 'lib/libmxnet.%s' % get_shared_lib_ext()], 'dirs': [], } super(EB_MXNet, self).sanity_check_step(custom_paths=custom_paths) # for the extension we are doing the loading of the fake module ourself try: fake_mod_data = self.load_fake_module() except EasyBuildError as err: raise EasyBuildError("Loading fake module failed: %s", err) if not self.py_ext.sanity_check_step(): raise EasyBuildError( "The sanity check for the Python bindings failed") self.r_ext.options['modulename'] = self.name.lower() if not self.r_ext.sanity_check_step(): raise EasyBuildError("The sanity check for the R bindings failed") self.clean_up_fake_module(fake_mod_data) def make_module_extra(self, *args, **kwargs): """Custom variables for MXNet module.""" txt = super(EB_MXNet, self).make_module_extra(*args, **kwargs) for path in self.py_ext.all_pylibdirs: fullpath = os.path.join(self.installdir, path) # only extend $PYTHONPATH with existing, non-empty directories if os.path.exists(fullpath) and os.listdir(fullpath): txt += self.module_generator.prepend_paths('PYTHONPATH', path) txt += self.module_generator.prepend_paths( "R_LIBS", ['']) # prepend R_LIBS with install path return txt
class EB_MXNet(MakeCp): """Easyblock to build and install MXNet""" @staticmethod def extra_options(extra_vars=None): """Change default values of options""" extra = MakeCp.extra_options() # files_to_copy is not mandatory here extra['files_to_copy'][2] = CUSTOM return extra def __init__(self, *args, **kwargs): """Initialize custom class variables.""" super(EB_MXNet, self).__init__(*args, **kwargs) self.mxnet_src_dir = None self.py_ext = PythonPackage(self, {'name': self.name, 'version': self.version}) self.py_ext.module_generator = self.module_generator self.r_ext = RPackage(self, {'name': self.name, 'version': self.version}) self.r_ext.module_generator = self.module_generator def extract_step(self): """ Prepare a combined MXNet source tree. Move all submodules to their right place. """ # Extract everything into separate directories. super(EB_MXNet, self).extract_step() mxnet_dirs = glob.glob(os.path.join(self.builddir, '*mxnet-*')) if len(mxnet_dirs) == 1: self.mxnet_src_dir = mxnet_dirs[0] self.log.debug("MXNet dir is: %s", self.mxnet_src_dir) else: raise EasyBuildError("Failed to find/isolate MXNet source directory: %s", mxnet_dirs) for srcdir in [d for d in os.listdir(self.builddir) if d != os.path.basename(self.mxnet_src_dir)]: submodule, _, _ = srcdir.rpartition('-') newdir = os.path.join(self.mxnet_src_dir, submodule) olddir = os.path.join(self.builddir, srcdir) # first remove empty existing directory rmtree2(newdir) try: shutil.move(olddir, newdir) except IOError as err: raise EasyBuildError("Failed to move %s to %s: %s", olddir, newdir, err) # the nnvm submodules has dmlc-core as a submodule too. Let's put a symlink in place. newdir = os.path.join(self.mxnet_src_dir, "nnvm", "dmlc-core") olddir = os.path.join(self.mxnet_src_dir, "dmlc-core") rmtree2(newdir) symlink(olddir, newdir) def prepare_step(self, *args, **kwargs): """Prepare for building and installing MXNet.""" super(EB_MXNet, self).prepare_step(*args, **kwargs) self.py_ext.prepare_python() def configure_step(self): """Patch 'config.mk' file to use EB stuff""" for (var, env_var) in [('CC', 'CC'), ('CXX', 'CXX'), ('ADD_CFLAGS', 'CFLAGS'), ('ADD_LDFLAGS', 'LDFLAGS')]: self.cfg.update('buildopts', '%s="%s"' % (var, os.getenv(env_var))) toolchain_blas = self.toolchain.definition().get('BLAS', None)[0] if toolchain_blas == 'imkl': blas = "mkl" imkl_version = get_software_version('imkl') if LooseVersion(imkl_version) >= LooseVersion('17'): self.cfg.update('buildopts', 'USE_MKL2017=1') self.cfg.update('buildopts', 'MKLML_ROOT="%s"' % os.getenv("MKLROOT")) elif toolchain_blas in ['ACML', 'ATLAS']: blas = "atlas" elif toolchain_blas == 'OpenBLAS': blas = "openblas" elif toolchain_blas is None: raise EasyBuildError("No BLAS library found in the toolchain") self.cfg.update('buildopts', 'USE_BLAS="%s"' % blas) if get_software_root('NNPACK'): self.cfg.update('buildopts', 'USE_NNPACK=1') super(EB_MXNet, self).configure_step() def install_step(self): """Specify list of files to copy""" self.cfg['files_to_copy'] = ['bin', 'include', 'lib', (['dmlc-core/include/dmlc', 'nnvm/include/nnvm'], 'include')] super(EB_MXNet, self).install_step() def extensions_step(self): """Build & Install both Python and R extension""" # we start with the python bindings self.py_ext.src = os.path.join(self.mxnet_src_dir, "python") change_dir(self.py_ext.src) self.py_ext.prerun() self.py_ext.run(unpack_src=False) self.py_ext.postrun() # next up, the R bindings self.r_ext.src = os.path.join(self.mxnet_src_dir, "R-package") change_dir(self.r_ext.src) mkdir("inst") symlink(os.path.join(self.installdir, "lib"), os.path.join("inst", "libs")) symlink(os.path.join(self.installdir, "include"), os.path.join("inst", "include")) # MXNet doesn't provide a list of its R dependencies by default write_file("NAMESPACE", R_NAMESPACE) change_dir(self.mxnet_src_dir) self.r_ext.prerun() # MXNet is just weird. To install the R extension, we have to: # - First install the extension like it is # - Let R export the extension again. By doing this, all the dependencies get # correctly filled and some mappings are done # - Reinstal the exported version self.r_ext.run() run_cmd("R_LIBS=%s Rscript -e \"require(mxnet); mxnet:::mxnet.export(\\\"R-package\\\")\"" % self.installdir) change_dir(self.r_ext.src) self.r_ext.run() self.r_ext.postrun() def sanity_check_step(self): """Check for main library files for MXNet""" custom_paths = { 'files': ['lib/libmxnet.a', 'lib/libmxnet.%s' % get_shared_lib_ext()], 'dirs': [], } super(EB_MXNet, self).sanity_check_step(custom_paths=custom_paths) # for the extension we are doing the loading of the fake module ourself try: fake_mod_data = self.load_fake_module() except EasyBuildError as err: raise EasyBuildError("Loading fake module failed: %s", err) if not self.py_ext.sanity_check_step(): raise EasyBuildError("The sanity check for the Python bindings failed") self.r_ext.options['modulename'] = self.name.lower() if not self.r_ext.sanity_check_step(): raise EasyBuildError("The sanity check for the R bindings failed") self.clean_up_fake_module(fake_mod_data) def make_module_extra(self, *args, **kwargs): """Custom variables for MXNet module.""" txt = super(EB_MXNet, self).make_module_extra(*args, **kwargs) for path in self.py_ext.all_pylibdirs: fullpath = os.path.join(self.installdir, path) # only extend $PYTHONPATH with existing, non-empty directories if os.path.exists(fullpath) and os.listdir(fullpath): txt += self.module_generator.prepend_paths('PYTHONPATH', path) txt += self.module_generator.prepend_paths("R_LIBS", ['']) # prepend R_LIBS with install path return txt