def _simulated_load_dependency_module(self, name, version, metadata, verbose=False): """ Set environment variables picked up by utility functions for dependencies specified as external modules. @param name: software name @param version: software version @param metadata: dictionary with software metadata ('prefix' for software installation prefix) """ self.log.debug("Defining $EB* environment variables for software named %s", name) # define $EBROOT env var for install prefix, picked up by get_software_root prefix = metadata.get('prefix') if prefix is not None: # the prefix can be specified in a number of ways # * name of environment variable (+ optional relative path to combine it with; format: <name>/<relpath> # * filepath (assumed if environment variable is not defined) parts = prefix.split(os.path.sep) env_var = parts[0] if env_var in os.environ: prefix = os.environ[env_var] rel_path = os.path.sep.join(parts[1:]) if rel_path: prefix = os.path.join(prefix, rel_path, '') self.log.debug("Derived prefix for software named %s from $%s (rel path: %s): %s", name, env_var, rel_path, prefix) else: self.log.debug("Using specified path as prefix for software named %s: %s", name, prefix) setvar(get_software_root_env_var_name(name), prefix, verbose=verbose) # define $EBVERSION env var for software version, picked up by get_software_version if version is not None: setvar(get_software_version_env_var_name(name), version, verbose=verbose)
def _simulated_load_dependency_module(self, name, version, metadata, verbose=False): """ Set environment variables picked up by utility functions for dependencies specified as external modules. :param name: software name :param version: software version :param metadata: dictionary with software metadata ('prefix' for software installation prefix) """ self.log.debug("Defining $EB* environment variables for software named %s", name) # define $EBROOT env var for install prefix, picked up by get_software_root prefix = metadata.get('prefix') if prefix is not None: # the prefix can be specified in a number of ways # * name of environment variable (+ optional relative path to combine it with; format: <name>/<relpath> # * filepath (assumed if environment variable is not defined) parts = prefix.split(os.path.sep) env_var = parts[0] if env_var in os.environ: prefix = os.environ[env_var] rel_path = os.path.sep.join(parts[1:]) if rel_path: prefix = os.path.join(prefix, rel_path, '') self.log.debug("Derived prefix for software named %s from $%s (rel path: %s): %s", name, env_var, rel_path, prefix) else: self.log.debug("Using specified path as prefix for software named %s: %s", name, prefix) setvar(get_software_root_env_var_name(name), prefix, verbose=verbose) # define $EBVERSION env var for software version, picked up by get_software_version if version is not None: setvar(get_software_version_env_var_name(name), version, verbose=verbose)
def _prepare_dependency_external_module(self, dep): """Set environment variables picked up by utility functions for dependencies specified as external modules.""" mod_name = dep['full_mod_name'] metadata = dep['external_module_metadata'] self.log.debug("Defining $EB* environment variables for external module %s", mod_name) names = metadata.get('name', []) versions = metadata.get('version', [None]*len(names)) self.log.debug("Metadata for external module %s: %s", mod_name, metadata) for name, version in zip(names, versions): self.log.debug("Defining $EB* environment variables for external module %s under name %s", mod_name, name) # define $EBROOT env var for install prefix, picked up by get_software_root prefix = metadata.get('prefix') if prefix is not None: if prefix in os.environ: val = os.environ[prefix] self.log.debug("Using value of $%s as prefix for external module %s: %s", prefix, mod_name, val) else: val = prefix self.log.debug("Using specified prefix for external module %s: %s", mod_name, val) setvar(get_software_root_env_var_name(name), val) # define $EBVERSION env var for software version, picked up by get_software_version if version is not None: setvar(get_software_version_env_var_name(name), version)
def env_vars_external_module(name, version, metadata): """ Determine $EBROOT* and/or $EBVERSION* environment variables that can be set for external module, based on the provided name, version and metadata. """ env_vars = {} # define $EBROOT env var for install prefix, picked up by get_software_root prefix = metadata.get('prefix') if prefix is not None: # the prefix can be specified in a number of ways # * name of environment variable (+ optional relative path to combine it with; format: <name>/<relpath> # * filepath (assumed if environment variable is not defined) parts = prefix.split(os.path.sep) env_var = parts[0] if env_var in os.environ: prefix = os.environ[env_var] rel_path = os.path.sep.join(parts[1:]) if rel_path: prefix = os.path.join(prefix, rel_path, '') _log.debug("Derived prefix for software named %s from $%s (rel path: %s): %s", name, env_var, rel_path, prefix) else: _log.debug("Using specified path as prefix for software named %s: %s", name, prefix) env_vars[get_software_root_env_var_name(name)] = prefix # define $EBVERSION env var for software version, picked up by get_software_version if version is not None: env_vars[get_software_version_env_var_name(name)] = version return env_vars
def _simulated_load_dependency_module(self, name, version, metadata, verbose=False): """ Set environment variables picked up by utility functions for dependencies specified as external modules. @param name: software name @param version: software version @param metadata: dictionary with software metadata ('prefix' for software installation prefix) """ self.log.debug("Defining $EB* environment variables for software named %s", name) # define $EBROOT env var for install prefix, picked up by get_software_root prefix = metadata.get('prefix') if prefix is not None: if prefix in os.environ: val = os.environ[prefix] self.log.debug("Using value of $%s as prefix for software named %s: %s", prefix, name, val) else: val = prefix self.log.debug("Using specified prefix for software named %s: %s", name, val) setvar(get_software_root_env_var_name(name), val, verbose=verbose) # define $EBVERSION env var for software version, picked up by get_software_version if version is not None: setvar(get_software_version_env_var_name(name), version, verbose=verbose)
def handle_allowed_system_deps(self): """Handle allowed system dependencies.""" for (name, version) in self['allow_system_deps']: env.setvar(get_software_root_env_var_name(name), name) # root is set to name, not an actual path env.setvar( get_software_version_env_var_name(name), version ) # version is expected to be something that makes sense
def _simulated_load_dependency_module(self, name, version, metadata, verbose=False): """ Set environment variables picked up by utility functions for dependencies specified as external modules. @param name: software name @param version: software version @param metadata: dictionary with software metadata ('prefix' for software installation prefix) """ self.log.debug( "Defining $EB* environment variables for software named %s", name) # define $EBROOT env var for install prefix, picked up by get_software_root prefix = metadata.get('prefix') if prefix is not None: if prefix in os.environ: val = os.environ[prefix] self.log.debug( "Using value of $%s as prefix for software named %s: %s", prefix, name, val) else: val = prefix self.log.debug( "Using specified prefix for software named %s: %s", name, val) setvar(get_software_root_env_var_name(name), val, verbose=verbose) # define $EBVERSION env var for software version, picked up by get_software_version if version is not None: setvar(get_software_version_env_var_name(name), version, verbose=verbose)
def template_module_only_test(self, easyblock, name='foo', version='1.3.2', extra_txt=''): """Test whether all easyblocks are compatible with --module-only.""" tmpdir = tempfile.mkdtemp() class_regex = re.compile("^class (.*)\(.*", re.M) self.log.debug("easyblock: %s" % easyblock) # read easyblock Python module f = open(easyblock, "r") txt = f.read() f.close() # obtain easyblock class name using regex res = class_regex.search(txt) if res: ebname = res.group(1) self.log.debug("Found class name for easyblock %s: %s" % (easyblock, ebname)) toolchain = None # figure out list of mandatory variables, and define with dummy values as necessary app_class = get_easyblock_class(ebname) # easyblocks deriving from IntelBase require a license file to be found for --module-only bases = list(app_class.__bases__) for base in copy.copy(bases): bases.extend(base.__bases__) if app_class == IntelBase or IntelBase in bases: os.environ['INTEL_LICENSE_FILE'] = os.path.join( tmpdir, 'intel.lic') write_file(os.environ['INTEL_LICENSE_FILE'], '# dummy license') elif app_class == EB_IMOD: # $JAVA_HOME must be set for IMOD os.environ['JAVA_HOME'] = tmpdir elif app_class == PythonBundle: # $EBROOTPYTHON must be set for PythonBundle easyblock os.environ[ 'EBROOTPYTHON'] = '/fake/install/prefix/Python/2.7.14-foss-2018a' elif app_class == EB_OpenFOAM: # proper toolchain must be used for OpenFOAM(-Extend), to determine value to set for $WM_COMPILER write_file( os.path.join(tmpdir, 'GCC', '4.9.3-2.25'), '\n'.join([ '#%Module', 'setenv EBROOTGCC %s' % tmpdir, 'setenv EBVERSIONGCC 4.9.3', ])) write_file( os.path.join(tmpdir, 'OpenMPI', '1.10.2-GCC-4.9.3-2.25'), '\n'.join([ '#%Module', 'setenv EBROOTOPENMPI %s' % tmpdir, 'setenv EBVERSIONOPENMPI 1.10.2', ])) write_file( os.path.join(tmpdir, 'gompi', '2016a'), '\n'.join([ '#%Module', 'module load GCC/4.9.3-2.25', 'module load OpenMPI/1.10.2-GCC-4.9.3-2.25', ])) os.environ['MODULEPATH'] = tmpdir toolchain = {'name': 'gompi', 'version': '2016a'} # extend easyconfig to make sure mandatory custom easyconfig paramters are defined extra_options = app_class.extra_options() for (key, val) in extra_options.items(): if val[2] == MANDATORY: extra_txt += '%s = "foo"\n' % key # write easyconfig file self.writeEC(ebname, name=name, version=version, extratxt=extra_txt, toolchain=toolchain) # take into account that for some easyblock, particular dependencies are hard required early on # (in prepare_step for exampel); # we just set the corresponding $EBROOT* environment variables here to fool it... req_deps = { # QScintilla easyblock requires that either PyQt or PyQt5 are available as dependency # (PyQt is easier, since PyQt5 is only supported for sufficiently recent QScintilla versions) 'qscintilla.py': [('PyQt', '4.12')], # MotionCor2 and Gctf easyblock requires CUDA as dependency 'motioncor2.py': [('CUDA', '10.1.105')], 'gctf.py': [('CUDA', '10.1.105')], } easyblock_fn = os.path.basename(easyblock) for (dep_name, dep_version) in req_deps.get(easyblock_fn, []): dep_root_envvar = get_software_root_env_var_name(dep_name) os.environ[dep_root_envvar] = '/value/should/not/matter' dep_version_envvar = get_software_version_env_var_name(dep_name) os.environ[dep_version_envvar] = dep_version # initialize easyblock # if this doesn't fail, the test succeeds app = app_class(EasyConfig(self.eb_file)) # run all steps, most should be skipped orig_workdir = os.getcwd() try: app.run_all_steps(run_test_cases=False) finally: change_dir(orig_workdir) if os.path.basename(easyblock) == 'modulerc.py': # .modulerc must be cleaned up to avoid causing trouble (e.g. "Duplicate version symbol" errors) modulerc = os.path.join(TMPDIR, 'modules', 'all', name, '.modulerc') if os.path.exists(modulerc): remove_file(modulerc) modulerc += '.lua' if os.path.exists(modulerc): remove_file(modulerc) else: modfile = os.path.join(TMPDIR, 'modules', 'all', name, version) luamodfile = '%s.lua' % modfile self.assertTrue( os.path.exists(modfile) or os.path.exists(luamodfile), "Module file %s or %s was generated" % (modfile, luamodfile)) if os.path.exists(modfile): modtxt = read_file(modfile) else: modtxt = read_file(luamodfile) none_regex = re.compile('None') self.assertFalse(none_regex.search(modtxt), "None not found in module file: %s" % modtxt) # cleanup app.close_log() remove_file(app.logfile) remove_dir(tmpdir) else: self.assertTrue(False, "Class found in easyblock %s" % easyblock)
def handle_allowed_system_deps(self): """Handle allowed system dependencies.""" for (name, version) in self['allow_system_deps']: env.setvar(get_software_root_env_var_name(name), name) # root is set to name, not an actual path env.setvar(get_software_version_env_var_name(name), version) # version is expected to be something that makes sense