def add_module(self, modules): """ Check if module exist, if so add to list. """ self.log.deprecated("Use of add_module function should be replaced by load([<list of modules>])", '2.0') for mod in modules: if isinstance(mod, (list, tuple)): mod_dict = { 'name': mod[0], 'version': mod[1], 'versionsuffix': '', 'toolchain': { 'name': DUMMY_TOOLCHAIN_NAME, 'version': DUMMY_TOOLCHAIN_VERSION, }, } mod_name = det_full_module_name(mod_dict) elif isinstance(mod, basestring): mod_name = mod elif isinstance(mod, dict): mod_name = det_full_module_name(mod) else: self.log.error("Can't add module %s: unknown type" % str(mod)) mods = self.available(mod_name) if mod_name in mods: # ok self._modules.append(mod_name) else: if len(mods) == 0: self.log.warning('No module %s available' % str(mod)) else: self.log.warning('More than one module found for %s: %s' % (mod, mods)) continue
def test_buildininstalldir(self): """Test specifying build in install dir.""" config.variables['buildpath'] = tempfile.mkdtemp() config.variables['installpath'] = tempfile.mkdtemp() self.contents = '\n'.join([ 'name = "pi"', 'version = "3.14"', 'homepage = "http://example.com"', 'description = "test easyconfig"', 'toolchain = {"name": "dummy", "version": "dummy"}', 'buildininstalldir = True', ]) self.prep() eb = EasyBlock(self.eb_file) eb.gen_builddir() eb.mod_name = det_full_module_name(eb.cfg) # required by gen_installdir() eb.gen_installdir() eb.make_builddir() eb.make_installdir() self.assertEqual(eb.builddir, eb.installdir) self.assertTrue(os.path.isdir(eb.builddir)) # cleanup shutil.rmtree(config.variables['buildpath']) shutil.rmtree(config.variables['installpath'])
def process_easyconfig(path, build_options=None, build_specs=None): """ Process easyconfig, returning some information for each block @param path: path to easyconfig file @param build_options: dictionary specifying build options (e.g. robot_path, check_osdeps, ...) @param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...) """ blocks = retrieve_blocks_in_spec(path, build_options.get("only_blocks", None)) easyconfigs = [] for spec in blocks: # process for dependencies and real installversionname _log.debug("Processing easyconfig %s" % spec) # create easyconfig try: ec = EasyConfig(spec, build_options=build_options, build_specs=build_specs) except EasyBuildError, err: msg = "Failed to process easyconfig %s:\n%s" % (spec, err.msg) _log.exception(msg) name = ec["name"] # this app will appear as following module in the list easyconfig = { "ec": ec, "spec": spec, "module": det_full_module_name(ec), "dependencies": [], "builddependencies": [], } if len(blocks) > 1: easyconfig["originalSpec"] = path # add build dependencies for dep in ec.builddependencies(): _log.debug("Adding build dependency %s for app %s." % (dep, name)) easyconfig["builddependencies"].append(dep) # add dependencies (including build dependencies) for dep in ec.dependencies(): _log.debug("Adding dependency %s for app %s." % (dep, name)) easyconfig["dependencies"].append(dep) # add toolchain as dependency too if ec.toolchain.name != DUMMY_TOOLCHAIN_NAME: dep = ec.toolchain.as_dict() _log.debug("Adding toolchain %s as dependency for app %s." % (dep, name)) easyconfig["dependencies"].append(dep) del ec # this is used by the parallel builder easyconfig["unresolved_deps"] = copy.deepcopy(easyconfig["dependencies"]) easyconfigs.append(easyconfig)
def add_dependencies(self, dependencies): """ Verify if the given dependencies exist and add them """ self.log.debug("add_dependencies: adding toolchain dependencies %s" % dependencies) for dep in dependencies: mod_name = det_full_module_name(dep) if not self.modules_tool.exists(mod_name): self.log.error("add_dependencies: no module found for dependency %s" % str(dep)) else: self.dependencies.append(dep) self.log.debug("add_dependencies: added toolchain dependency %s" % dep)
def process_easyconfig(path, onlyBlocks=None, regtest_online=False, validate=True): """ Process easyconfig, returning some information for each block """ blocks = retrieve_blocks_in_spec(path, onlyBlocks) easyconfigs = [] for spec in blocks: # process for dependencies and real installversionname # - use mod? __init__ and importCfg are ignored. _log.debug("Processing easyconfig %s" % spec) # create easyconfig try: all_stops = [x[0] for x in EasyBlock.get_steps()] ec = EasyConfig(spec, validate=validate, valid_module_classes=module_classes(), valid_stops=all_stops) except EasyBuildError, err: msg = "Failed to process easyconfig %s:\n%s" % (spec, err.msg) _log.exception(msg) name = ec['name'] # this app will appear as following module in the list easyconfig = { 'ec': ec, 'spec': spec, 'module': det_full_module_name(ec), 'dependencies': [], 'builddependencies': [], } if len(blocks) > 1: easyconfig['originalSpec'] = path # add build dependencies for dep in ec.builddependencies(): _log.debug("Adding build dependency %s for app %s." % (dep, name)) easyconfig['builddependencies'].append(dep) # add dependencies (including build dependencies) for dep in ec.dependencies(): _log.debug("Adding dependency %s for app %s." % (dep, name)) easyconfig['dependencies'].append(dep) # add toolchain as dependency too if ec.toolchain.name != DUMMY_TOOLCHAIN_NAME: dep = ec.toolchain.as_dict() _log.debug("Adding toolchain %s as dependency for app %s." % (dep, name)) easyconfig['dependencies'].append(dep) del ec # this is used by the parallel builder easyconfig['unresolved_deps'] = copy.deepcopy(easyconfig['dependencies']) easyconfigs.append(easyconfig)
def create_job(build_command, easyconfig, output_dir="", conn=None, ppn=None): """ Creates a job, to build a *single* easyconfig build_command is a format string in which a full path to an eb file will be substituted easyconfig should be in the format as processEasyConfig returns them output_dir is an optional path. EASYBUILDTESTOUTPUT will be set inside the job with this variable returns the job """ # create command based on build_command template command = build_command % {'spec': easyconfig['spec']} # capture PYTHONPATH, MODULEPATH and all variables starting with EASYBUILD easybuild_vars = {} for name in os.environ: if name.startswith("EASYBUILD"): easybuild_vars[name] = os.environ[name] others = ["PYTHONPATH", "MODULEPATH"] for env_var in others: if env_var in os.environ: easybuild_vars[env_var] = os.environ[env_var] _log.info("Dictionary of environment variables passed to job: %s" % easybuild_vars) # obtain unique name based on name/easyconfig version tuple ec_tuple = (easyconfig['ec']['name'], det_full_ec_version(easyconfig['ec'])) name = '-'.join(ec_tuple) var = config.oldstyle_environment_variables['test_output_path'] easybuild_vars[var] = os.path.join(os.path.abspath(output_dir), name) # just use latest build stats repo = init_repository(get_repository(), get_repositorypath()) buildstats = repo.get_buildstats(*ec_tuple) resources = {} if buildstats: previous_time = buildstats[-1]['build_time'] resources['hours'] = int(math.ceil(previous_time * 2 / 60)) job = PbsJob(command, name, easybuild_vars, resources=resources, conn=conn, ppn=ppn) job.module = det_full_module_name(easyconfig['ec']) return job
def add_dependencies(self, dependencies): """ Verify if the given dependencies exist and add them """ self.log.debug("add_dependencies: adding toolchain dependencies %s" % dependencies) for dep in dependencies: mod_name = det_full_module_name(dep) if not self.modules_tool.exists(mod_name): self.log.error( 'add_dependencies: no module found for dependency %s' % str(dep)) else: self.dependencies.append(dep) self.log.debug( 'add_dependencies: added toolchain dependency %s' % dep)
def add_module(self, modules): """ Check if module exist, if so add to list. """ self.log.deprecated( "Use of add_module function should be replaced by load([<list of modules>])", '2.0') for mod in modules: if isinstance(mod, (list, tuple)): mod_dict = { 'name': mod[0], 'version': mod[1], 'versionsuffix': '', 'toolchain': { 'name': DUMMY_TOOLCHAIN_NAME, 'version': DUMMY_TOOLCHAIN_VERSION, }, } mod_name = det_full_module_name(mod_dict) elif isinstance(mod, basestring): mod_name = mod elif isinstance(mod, dict): mod_name = det_full_module_name(mod) else: self.log.error("Can't add module %s: unknown type" % str(mod)) mods = self.available(mod_name) if mod_name in mods: # ok self._modules.append(mod_name) else: if len(mods) == 0: self.log.warning('No module %s available' % str(mod)) else: self.log.warning('More then one module found for %s: %s' % (mod, mods)) continue
def create_job(build_command, easyconfig, output_dir=None, conn=None, ppn=None): """ Creates a job, to build a *single* easyconfig @param build_command: format string for command, full path to an easyconfig file will be substituted in it @param easyconfig: easyconfig as processed by process_easyconfig @param output_dir: optional output path; $EASYBUILDTESTOUTPUT will be set inside the job with this variable @param conn: open connection to PBS server @param ppn: ppn setting to use (# 'processors' (cores) per node to use) returns the job """ if output_dir is None: output_dir = 'easybuild-build' # create command based on build_command template command = build_command % {'spec': easyconfig['spec']} # capture PYTHONPATH, MODULEPATH and all variables starting with EASYBUILD easybuild_vars = {} for name in os.environ: if name.startswith("EASYBUILD"): easybuild_vars[name] = os.environ[name] others = ["PYTHONPATH", "MODULEPATH"] for env_var in others: if env_var in os.environ: easybuild_vars[env_var] = os.environ[env_var] _log.info("Dictionary of environment variables passed to job: %s" % easybuild_vars) # obtain unique name based on name/easyconfig version tuple ec_tuple = (easyconfig['ec']['name'], det_full_ec_version(easyconfig['ec'])) name = '-'.join(ec_tuple) var = config.oldstyle_environment_variables['test_output_path'] easybuild_vars[var] = os.path.join(os.path.abspath(output_dir), name) # just use latest build stats repo = init_repository(get_repository(), get_repositorypath()) buildstats = repo.get_buildstats(*ec_tuple) resources = {} if buildstats: previous_time = buildstats[-1]['build_time'] resources['hours'] = int(math.ceil(previous_time * 2 / 60)) job = PbsJob(command, name, easybuild_vars, resources=resources, conn=conn, ppn=ppn) job.module = det_full_module_name(easyconfig['ec']) return job
def test_gen_dirs(self): """Test methods that generate/set build/install directory names.""" self.contents = '\n'.join([ "name = 'pi'", "version = '3.14'", "homepage = 'http://example.com'", "description = 'test easyconfig'", "toolchain = {'name': 'dummy', 'version': 'dummy'}", ]) self.writeEC() stdoutorig = sys.stdout sys.stdout = open("/dev/null", 'w') eb = EasyBlock(EasyConfig(self.eb_file)) resb = eb.gen_builddir() eb.mod_name = det_full_module_name( eb.cfg) # required by gen_installdir() resi = eb.gen_installdir() eb.make_builddir() eb.make_installdir() # doesn't return anything self.assertEqual(resb, None) self.assertEqual(resi, None) # directories are set, and exist self.assertTrue(os.path.isdir(eb.builddir)) self.assertTrue(os.path.isdir(eb.installdir)) # make sure cleaning up old build dir is default self.assertTrue(eb.cfg['cleanupoldbuild'] or eb.cfg.get('cleanupoldbuild', True)) builddir = eb.builddir eb.gen_builddir() self.assertEqual(builddir, eb.builddir) eb.cfg['cleanupoldbuild'] = True eb.gen_builddir() self.assertEqual(builddir, eb.builddir) # make sure build dir is unique eb.cfg['cleanupoldbuild'] = False builddir = eb.builddir for i in range(3): eb.gen_builddir() self.assertEqual(eb.builddir, "%s.%d" % (builddir, i)) eb.make_builddir() # cleanup sys.stdout.close() sys.stdout = stdoutorig eb.close_log()
def print_dry_run(easyconfigs, robot=None): if robot is None: print_msg("Dry run: printing build status of easyconfigs") all_specs = easyconfigs else: print_msg("Dry run: printing build status of easyconfigs and dependencies") all_specs = resolve_dependencies(easyconfigs, robot, True) unbuilt_specs = skip_available(all_specs, True) dry_run_fmt = "%3s %s (module: %s)" for spec in all_specs: if spec in unbuilt_specs: ans = '[ ]' else: ans = '[x]' mod = det_full_module_name(spec['ec']) print dry_run_fmt % (ans, spec['spec'], mod)
def find_resolved_modules(unprocessed, processed): """ Find easyconfigs in unprocessed which can be fully resolved using easyconfigs in processed """ ordered_ecs = [] for ec in unprocessed: ec['dependencies'] = [d for d in ec['dependencies'] if not det_full_module_name(d) in processed] if len(ec['dependencies']) == 0: _log.debug("Adding easyconfig %s to final list" % ec['spec']) ordered_ecs.append(ec) processed.append(ec['module']) unprocessed[:] = [m for m in unprocessed if len(m['dependencies']) > 0] return ordered_ecs
def test_gen_dirs(self): """Test methods that generate/set build/install directory names.""" self.contents = '\n'.join([ "name = 'pi'", "version = '3.14'", "homepage = 'http://example.com'", "description = 'test easyconfig'", "toolchain = {'name': 'dummy', 'version': 'dummy'}", ]) self.writeEC() stdoutorig = sys.stdout sys.stdout = open("/dev/null", 'w') eb = EasyBlock(EasyConfig(self.eb_file)) resb = eb.gen_builddir() eb.mod_name = det_full_module_name(eb.cfg) # required by gen_installdir() resi = eb.gen_installdir() eb.make_builddir() eb.make_installdir() # doesn't return anything self.assertEqual(resb, None) self.assertEqual(resi, None) # directories are set, and exist self.assertTrue(os.path.isdir(eb.builddir)) self.assertTrue(os.path.isdir(eb.installdir)) # make sure cleaning up old build dir is default self.assertTrue(eb.cfg['cleanupoldbuild'] or eb.cfg.get('cleanupoldbuild', True)) builddir = eb.builddir eb.gen_builddir() self.assertEqual(builddir, eb.builddir) eb.cfg['cleanupoldbuild'] = True eb.gen_builddir() self.assertEqual(builddir, eb.builddir) # make sure build dir is unique eb.cfg['cleanupoldbuild'] = False builddir = eb.builddir for i in range(3): eb.gen_builddir() self.assertEqual(eb.builddir, "%s.%d" % (builddir, i)) eb.make_builddir() # cleanup sys.stdout.close() sys.stdout = stdoutorig eb.close_log()
def print_dry_run(easyconfigs, short=False, build_options=None, build_specs=None): """ Print dry run information @param easyconfigs: list of easyconfig files @param short: print short output (use a variable for the common prefix) @param build_options: dictionary specifying build options (e.g. robot_path, check_osdeps, ...) @param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...) """ lines = [] if build_options.get("robot_path", None) is None: lines.append("Dry run: printing build status of easyconfigs") all_specs = easyconfigs else: lines.append("Dry run: printing build status of easyconfigs and dependencies") build_options = copy.deepcopy(build_options) build_options.update({"retain_all_deps": True, "check_osdeps": False}) all_specs = resolve_dependencies(easyconfigs, build_options=build_options, build_specs=build_specs) unbuilt_specs = skip_available(all_specs, testing=True) dry_run_fmt = " * [%1s] %s (module: %s)" # markdown compatible (list of items with checkboxes in front) var_name = "CFGS" common_prefix = det_common_path_prefix([spec["spec"] for spec in all_specs]) # only allow short if common prefix is long enough short = short and common_prefix is not None and len(common_prefix) > len(var_name) * 2 for spec in all_specs: if spec in unbuilt_specs: ans = " " else: ans = "x" mod = det_full_module_name(spec["ec"]) if short: item = os.path.join("$%s" % var_name, spec["spec"][len(common_prefix) + 1 :]) else: item = spec["spec"] lines.append(dry_run_fmt % (ans, item, mod)) if short: # insert after 'Dry run:' message lines.insert(1, "%s=%s" % (var_name, common_prefix)) silent = build_options.get("silent", False) print_msg("\n".join(lines), log=_log, silent=silent, prefix=False)
def test_buildininstalldir(self): """Test specifying build in install dir.""" self.contents = '\n'.join([ 'name = "pi"', 'version = "3.14"', 'homepage = "http://example.com"', 'description = "test easyconfig"', 'toolchain = {"name": "dummy", "version": "dummy"}', 'buildininstalldir = True', ]) self.prep() ec = EasyConfig(self.eb_file) eb = EasyBlock(ec) eb.gen_builddir() eb.mod_name = det_full_module_name(eb.cfg) # required by gen_installdir() eb.gen_installdir() eb.make_builddir() eb.make_installdir() self.assertEqual(eb.builddir, eb.installdir) self.assertTrue(os.path.isdir(eb.builddir))
def find_resolved_modules(unprocessed, avail_modules): """ Find easyconfigs in 1st argument which can be fully resolved using modules specified in 2nd argument """ ordered_ecs = [] new_avail_modules = avail_modules[:] new_unprocessed = [] for ec in unprocessed: new_ec = ec.copy() new_ec["dependencies"] = [d for d in new_ec["dependencies"] if not det_full_module_name(d) in new_avail_modules] if len(new_ec["dependencies"]) == 0: _log.debug("Adding easyconfig %s to final list" % new_ec["spec"]) ordered_ecs.append(new_ec) new_avail_modules.append(ec["module"]) else: new_unprocessed.append(new_ec) return ordered_ecs, new_unprocessed, new_avail_modules
def resolve_dependencies(unprocessed, robot, force=False): """ Work through the list of easyconfigs to determine an optimal order enabling force results in retaining all dependencies and skipping validation of easyconfigs """ if force: # assume that no modules are available when forced available_modules = [] _log.info("Forcing all dependencies to be retained.") else: # Get a list of all available modules (format: [(name, installversion), ...]) available_modules = modules_tool().available() if len(available_modules) == 0: _log.warning("No installed modules. Your MODULEPATH is probably incomplete: %s" % os.getenv('MODULEPATH')) ordered_ecs = [] # All available modules can be used for resolving dependencies except # those that will be installed being_installed = [p['module'] for p in unprocessed] processed = [m for m in available_modules if not m in being_installed] _log.debug('unprocessed before resolving deps: %s' % unprocessed) # as long as there is progress in processing the modules, keep on trying loopcnt = 0 maxloopcnt = 10000 robot_add_dep = True while robot_add_dep: robot_add_dep = False # make sure this stops, we really don't want to get stuck in an infinite loop loopcnt += 1 if loopcnt > maxloopcnt: msg = "Maximum loop cnt %s reached, so quitting." % maxloopcnt _log.error(msg) # first try resolving dependencies without using external dependencies last_processed_count = -1 while len(processed) > last_processed_count: last_processed_count = len(processed) ordered_ecs.extend(find_resolved_modules(unprocessed, processed)) # robot: look for an existing dependency, add one if robot and len(unprocessed) > 0: being_installed = [det_full_module_name(p['ec'], eb_ns=True) for p in unprocessed] for entry in unprocessed: # do not choose an entry that is being installed in the current run # if they depend, you probably want to rebuild them using the new dependency deps = entry['dependencies'] candidates = [d for d in deps if not det_full_module_name(d, eb_ns=True) in being_installed] if len(candidates) > 0: cand_dep = candidates[0] # find easyconfig, might not find any _log.debug("Looking for easyconfig for %s" % str(cand_dep)) path = robot_find_easyconfig(robot, cand_dep['name'], det_full_ec_version(cand_dep)) else: path = None mod_name = det_full_module_name(entry['ec'], eb_ns=True) _log.debug("No more candidate dependencies to resolve for %s" % mod_name) if path is not None: cand_dep = candidates[0] _log.info("Robot: resolving dependency %s with %s" % (cand_dep, path)) processed_ecs = process_easyconfig(path, validate=(not force)) # ensure that selected easyconfig provides required dependency mods = [det_full_module_name(spec['ec']) for spec in processed_ecs] dep_mod_name = det_full_module_name(cand_dep) if not dep_mod_name in mods: _log.error("easyconfig file %s does not contain module %s (mods: %s)" % (path, dep_mod_name, mods)) unprocessed.extend(processed_ecs) robot_add_dep = True break _log.debug('unprocessed after resolving deps: %s' % unprocessed) # there are dependencies that cannot be resolved if len(unprocessed) > 0: _log.debug("List of unresolved dependencies: %s" % unprocessed) missing_dependencies = [] for ec in unprocessed: for dep in ec['dependencies']: missing_dependencies.append('%s for %s' % (det_full_module_name(dep, eb_ns=True), dep)) msg = "Dependencies not met. Cannot resolve %s" % missing_dependencies _log.error(msg) _log.info("Dependency resolution complete, building as follows:\n%s" % ordered_ecs) return ordered_ecs
def det_module_name(self, name=None, version=None): """Determine module name for this toolchain.""" return det_full_module_name(self.as_dict(name, version))
def mk_dep_mod_name(spec): return tuple(det_full_module_name(spec).split(os.path.sep))
def tokey(dep): """Determine key for specified dependency.""" return det_full_module_name(dep)
def prepare(self, onlymod=None): """ Prepare a set of environment parameters based on name/version of toolchain - load modules for toolchain and dependencies - generate extra variables and set them in the environment onlymod: Boolean/string to indicate if the toolchain should only load the environment with module (True) or also set all other variables (False) like compiler CC etc (If string: comma separated list of variables that will be ignored). """ if self.modules_tool is None: self.log.raiseException("No modules tool defined.") if not self._toolchain_exists(): self.log.raiseException( "No module found for toolchain name '%s' (%s)" % (self.name, self.version)) if self.name == DUMMY_TOOLCHAIN_NAME: if self.version == DUMMY_TOOLCHAIN_VERSION: self.log.info( 'prepare: toolchain dummy mode, dummy version; not loading dependencies' ) else: self.log.info( 'prepare: toolchain dummy mode and loading dependencies') self.modules_tool.load( [det_full_module_name(dep) for dep in self.dependencies]) return # Load the toolchain and dependencies modules self.log.debug("Loading toolchain module and dependencies...") self.modules_tool.load([self.det_module_name()]) self.modules_tool.load( [det_full_module_name(dep) for dep in self.dependencies]) # determine direct toolchain dependencies mod_name = self.det_module_name() self.toolchain_dependencies = self.modules_tool.dependencies_for( mod_name, depth=0) self.log.debug('prepare: list of direct toolchain dependencies: %s' % self.toolchain_dependencies) # verify whether elements in toolchain definition match toolchain deps specified by loaded toolchain module toolchain_module_deps = set([ self.modules_tool.module_software_name(d) for d in self.toolchain_dependencies ]) toolchain_elements_mod_names = set([ y for x in dir(self) if x.endswith('_MODULE_NAME') for y in eval("self.%s" % x) ]) # filter out toolchain name (e.g. 'GCC') from list of toolchain elements toolchain_elements_mod_names = set( [x for x in toolchain_elements_mod_names if not x == self.name]) # filter out optional toolchain elements if they're not used in the module for mod_name in toolchain_elements_mod_names.copy(): if not self.is_required(mod_name): if not mod_name in toolchain_module_deps: self.log.debug( "Removing optional module %s from list of toolchain elements." % mod_name) toolchain_elements_mod_names.remove(mod_name) self.log.debug( "List of toolchain dependency modules from loaded toolchain module: %s" % toolchain_module_deps) self.log.debug( "List of toolchain elements from toolchain definition: %s" % toolchain_elements_mod_names) if toolchain_module_deps == toolchain_elements_mod_names: self.log.info( "List of toolchain dependency modules and toolchain definition match!" ) else: self.log.error("List of toolchain dependency modules and toolchain definition do not match " \ "(%s vs %s)" % (toolchain_module_deps, toolchain_elements_mod_names)) # Generate the variables to be set self.set_variables() # set the variables # onlymod can be comma-separated string of variables not to be set if onlymod == True: self.log.debug( "prepare: do not set additional variables onlymod=%s" % onlymod) self.generate_vars() else: self.log.debug("prepare: set additional variables onlymod=%s" % onlymod) # add LDFLAGS and CPPFLAGS from dependencies to self.vars self._add_dependency_variables() self.generate_vars() self._setenv_variables(onlymod)
def resolve_dependencies(unprocessed, build_options=None, build_specs=None): """ Work through the list of easyconfigs to determine an optimal order @param unprocessed: list of easyconfigs @param build_options: dictionary specifying build options (e.g. robot_path, check_osdeps, ...) @param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...) """ robot = build_options.get("robot_path", None) if build_options.get("retain_all_deps", False): # assume that no modules are available when forced, to retain all dependencies avail_modules = [] _log.info("Forcing all dependencies to be retained.") else: # Get a list of all available modules (format: [(name, installversion), ...]) avail_modules = modules_tool().available() if len(avail_modules) == 0: _log.warning("No installed modules. Your MODULEPATH is probably incomplete: %s" % os.getenv("MODULEPATH")) ordered_ecs = [] # all available modules can be used for resolving dependencies except those that will be installed being_installed = [p["module"] for p in unprocessed] avail_modules = [m for m in avail_modules if not m in being_installed] _log.debug("unprocessed before resolving deps: %s" % unprocessed) # resolve all dependencies, put a safeguard in place to avoid an infinite loop (shouldn't occur though) irresolvable = [] loopcnt = 0 maxloopcnt = 10000 while unprocessed: # make sure this stops, we really don't want to get stuck in an infinite loop loopcnt += 1 if loopcnt > maxloopcnt: tup = (maxloopcnt, unprocessed, irresolvable) msg = "Maximum loop cnt %s reached, so quitting (unprocessed: %s, irresolvable: %s)" % tup _log.error(msg) # first try resolving dependencies without using external dependencies last_processed_count = -1 while len(avail_modules) > last_processed_count: last_processed_count = len(avail_modules) more_ecs, unprocessed, avail_modules = find_resolved_modules(unprocessed, avail_modules) for ec in more_ecs: if not ec["module"] in [x["module"] for x in ordered_ecs]: ordered_ecs.append(ec) # robot: look for existing dependencies, add them if robot and unprocessed: being_installed = [det_full_module_name(p["ec"], eb_ns=True) for p in unprocessed] additional = [] for i, entry in enumerate(unprocessed): # do not choose an entry that is being installed in the current run # if they depend, you probably want to rebuild them using the new dependency deps = entry["dependencies"] candidates = [d for d in deps if not det_full_module_name(d, eb_ns=True) in being_installed] if len(candidates) > 0: cand_dep = candidates[0] # find easyconfig, might not find any _log.debug("Looking for easyconfig for %s" % str(cand_dep)) # note: robot_find_easyconfig may return None path = robot_find_easyconfig(robot, cand_dep["name"], det_full_ec_version(cand_dep)) if path is None: # no easyconfig found for dependency, add to list of irresolvable dependencies if cand_dep not in irresolvable: irresolvable.append(cand_dep) # remove irresolvable dependency from list of dependencies so we can continue entry["dependencies"].remove(cand_dep) else: _log.info("Robot: resolving dependency %s with %s" % (cand_dep, path)) processed_ecs = process_easyconfig(path, build_options=build_options, build_specs=build_specs) # ensure that selected easyconfig provides required dependency mods = [det_full_module_name(spec["ec"]) for spec in processed_ecs] dep_mod_name = det_full_module_name(cand_dep) if not dep_mod_name in mods: tup = (path, dep_mod_name, mods) _log.error("easyconfig file %s does not contain module %s (mods: %s)" % tup) for ec in processed_ecs: if not ec in unprocessed + additional: additional.append(ec) _log.debug("Added %s as dependency of %s" % (ec, entry)) else: mod_name = det_full_module_name(entry["ec"], eb_ns=True) _log.debug("No more candidate dependencies to resolve for %s" % mod_name) # add additional (new) easyconfigs to list of stuff to process unprocessed.extend(additional) if irresolvable: irresolvable_mod_deps = [(det_full_module_name(dep, eb_ns=True), dep) for dep in irresolvable] _log.error("Irresolvable dependencies encountered: %s" % irresolvable_mod_deps) _log.info("Dependency resolution complete, building as follows:\n%s" % ordered_ecs) return ordered_ecs
_log.debug("Build stats: %s" % buildstats) if app.cfg['stop']: ended = "STOPPED" newLogDir = os.path.join(app.builddir, config.log_path()) else: newLogDir = os.path.join(app.installdir, config.log_path()) try: # upload spec to central repository repo = init_repository(get_repository(), get_repositorypath()) if 'originalSpec' in module: block = det_full_ec_version(app.cfg) + ".block" repo.add_easyconfig(module['originalSpec'], app.name, block, buildstats, currentbuildstats) repo.add_easyconfig(spec, app.name, det_full_ec_version(app.cfg), buildstats, currentbuildstats) repo.commit("Built %s" % det_full_module_name(app.cfg)) del repo except EasyBuildError, err: _log.warn("Unable to commit easyconfig to repository: %s", err) exitCode = 0 succ = "successfully" summary = "COMPLETED" # cleanup logs app.close_log() try: if not os.path.isdir(newLogDir): os.makedirs(newLogDir) applicationLog = os.path.join(newLogDir, get_log_filename(app.name, app.version)) shutil.move(app.logfile, applicationLog)
def mk_node_name(spec): if omit_versions: return spec["name"] else: return det_full_module_name(spec)
def prepare(self, onlymod=None): """ Prepare a set of environment parameters based on name/version of toolchain - load modules for toolchain and dependencies - generate extra variables and set them in the environment onlymod: Boolean/string to indicate if the toolchain should only load the environment with module (True) or also set all other variables (False) like compiler CC etc (If string: comma separated list of variables that will be ignored). """ if self.modules_tool is None: self.log.raiseException("No modules tool defined.") if not self._toolchain_exists(): self.log.raiseException("No module found for toolchain name '%s' (%s)" % (self.name, self.version)) if self.name == DUMMY_TOOLCHAIN_NAME: if self.version == DUMMY_TOOLCHAIN_VERSION: self.log.info("prepare: toolchain dummy mode, dummy version; not loading dependencies") else: self.log.info("prepare: toolchain dummy mode and loading dependencies") self.modules_tool.load([det_full_module_name(dep) for dep in self.dependencies]) return # Load the toolchain and dependencies modules self.log.debug("Loading toolchain module and dependencies...") self.modules_tool.load([self.det_module_name()]) self.modules_tool.load([det_full_module_name(dep) for dep in self.dependencies]) # determine direct toolchain dependencies mod_name = self.det_module_name() self.toolchain_dependencies = self.modules_tool.dependencies_for(mod_name, depth=0) self.log.debug("prepare: list of direct toolchain dependencies: %s" % self.toolchain_dependencies) # verify whether elements in toolchain definition match toolchain deps specified by loaded toolchain module toolchain_module_deps = set([self.modules_tool.module_software_name(d) for d in self.toolchain_dependencies]) toolchain_elements_mod_names = set( [y for x in dir(self) if x.endswith("_MODULE_NAME") for y in eval("self.%s" % x)] ) # filter out toolchain name (e.g. 'GCC') from list of toolchain elements toolchain_elements_mod_names = set([x for x in toolchain_elements_mod_names if not x == self.name]) # filter out optional toolchain elements if they're not used in the module for mod_name in toolchain_elements_mod_names.copy(): if not self.is_required(mod_name): if not mod_name in toolchain_module_deps: self.log.debug("Removing optional module %s from list of toolchain elements." % mod_name) toolchain_elements_mod_names.remove(mod_name) self.log.debug("List of toolchain dependency modules from loaded toolchain module: %s" % toolchain_module_deps) self.log.debug("List of toolchain elements from toolchain definition: %s" % toolchain_elements_mod_names) if toolchain_module_deps == toolchain_elements_mod_names: self.log.info("List of toolchain dependency modules and toolchain definition match!") else: self.log.error( "List of toolchain dependency modules and toolchain definition do not match " "(%s vs %s)" % (toolchain_module_deps, toolchain_elements_mod_names) ) # Generate the variables to be set self.set_variables() # set the variables # onlymod can be comma-separated string of variables not to be set if onlymod == True: self.log.debug("prepare: do not set additional variables onlymod=%s" % onlymod) self.generate_vars() else: self.log.debug("prepare: set additional variables onlymod=%s" % onlymod) # add LDFLAGS and CPPFLAGS from dependencies to self.vars self._add_dependency_variables() self.generate_vars() self._setenv_variables(onlymod)