def test_installversion(self): """Test generation of install version.""" ver = "3.14" verpref = "myprefix|" versuff = "|mysuffix" tcname = "GCC" tcver = "4.6.3" dummy = "dummy" correct_installver = "%s%s-%s-%s%s" % (verpref, ver, tcname, tcver, versuff) cfg = { 'version': ver, 'toolchain': {'name': tcname, 'version': tcver}, 'versionprefix': verpref, 'versionsuffix': versuff, } installver = det_full_ec_version(cfg) self.assertEqual(installver, "%s%s-%s-%s%s" % (verpref, ver, tcname, tcver, versuff)) correct_installver = "%s%s%s" % (verpref, ver, versuff) cfg = { 'version': ver, 'toolchain': {'name': dummy, 'version': tcver}, 'versionprefix': verpref, 'versionsuffix': versuff, } installver = det_full_ec_version(cfg) self.assertEqual(installver, correct_installver)
def test_dependency(self): """ test all possible ways of specifying dependencies """ self.contents = '\n'.join([ 'name = "pi"', 'version = "3.14"', 'homepage = "http://google.com"', 'description = "test easyconfig"', 'toolchain = {"name":"GCC", "version": "4.6.3"}', 'dependencies = [("first", "1.1"), {"name": "second", "version": "2.2"}]', 'builddependencies = [("first", "1.1"), {"name": "second", "version": "2.2"}]', ]) self.prep() eb = EasyConfig(self.eb_file, valid_stops=self.all_stops) # should include builddependencies self.assertEqual(len(eb.dependencies()), 4) self.assertEqual(len(eb.builddependencies()), 2) first = eb.dependencies()[0] second = eb.dependencies()[1] self.assertEqual(first['name'], "first") self.assertEqual(second['name'], "second") self.assertEqual(first['version'], "1.1") self.assertEqual(second['version'], "2.2") self.assertEqual(det_full_ec_version(first), '1.1-GCC-4.6.3') self.assertEqual(det_full_ec_version(second), '2.2-GCC-4.6.3') # same tests for builddependencies first = eb.builddependencies()[0] second = eb.builddependencies()[1] self.assertEqual(first['name'], "first") self.assertEqual(second['name'], "second") self.assertEqual(first['version'], "1.1") self.assertEqual(second['version'], "2.2") self.assertEqual(det_full_ec_version(first), '1.1-GCC-4.6.3') self.assertEqual(det_full_ec_version(second), '2.2-GCC-4.6.3') eb['dependencies'] = ["wrong type"] self.assertErrorRegex(EasyBuildError, "wrong type from unsupported type", eb.dependencies) eb['dependencies'] = [()] self.assertErrorRegex(EasyBuildError, "without name", eb.dependencies) eb['dependencies'] = [{'name': "test"}] self.assertErrorRegex(EasyBuildError, "without version", eb.dependencies)
def test_dependency(self): """ test all possible ways of specifying dependencies """ self.contents = '\n'.join([ 'easyblock = "ConfigureMake"', 'name = "pi"', 'version = "3.14"', 'homepage = "http://example.com"', 'description = "test easyconfig"', 'toolchain = {"name":"GCC", "version": "4.6.3"}', 'dependencies = [("first", "1.1"), {"name": "second", "version": "2.2"}]', 'builddependencies = [("first", "1.1"), {"name": "second", "version": "2.2"}]', ]) self.prep() eb = EasyConfig(self.eb_file) # should include builddependencies self.assertEqual(len(eb.dependencies()), 4) self.assertEqual(len(eb.builddependencies()), 2) first = eb.dependencies()[0] second = eb.dependencies()[1] self.assertEqual(first['name'], "first") self.assertEqual(second['name'], "second") self.assertEqual(first['version'], "1.1") self.assertEqual(second['version'], "2.2") self.assertEqual(det_full_ec_version(first), '1.1-GCC-4.6.3') self.assertEqual(det_full_ec_version(second), '2.2-GCC-4.6.3') # same tests for builddependencies first = eb.builddependencies()[0] second = eb.builddependencies()[1] self.assertEqual(first['name'], "first") self.assertEqual(second['name'], "second") self.assertEqual(first['version'], "1.1") self.assertEqual(second['version'], "2.2") self.assertEqual(det_full_ec_version(first), '1.1-GCC-4.6.3') self.assertEqual(det_full_ec_version(second), '2.2-GCC-4.6.3') self.assertErrorRegex(EasyBuildError, "Dependency foo of unsupported type", eb._parse_dependency, "foo") self.assertErrorRegex(EasyBuildError, "without name", eb._parse_dependency, ()) self.assertErrorRegex(EasyBuildError, "without version", eb._parse_dependency, {'name': 'test'}) err_msg = "Incorrect external dependency specification" self.assertErrorRegex(EasyBuildError, err_msg, eb._parse_dependency, (EXTERNAL_MODULE_MARKER,)) self.assertErrorRegex(EasyBuildError, err_msg, eb._parse_dependency, ('foo', '1.2.3', EXTERNAL_MODULE_MARKER))
def test_validation(self): """ test other validations beside mandatory variables """ self.contents = '\n'.join([ 'name = "pi"', 'version = "3.14"', 'homepage = "http://example.com"', 'description = "test easyconfig"', 'toolchain = {"name":"dummy", "version": "dummy"}', 'stop = "notvalid"', ]) self.prep() ec = EasyConfig(self.eb_file, validate=False) self.assertErrorRegex(EasyBuildError, r"\w* provided '\w*' is not valid", ec.validate) ec['stop'] = 'patch' # this should now not crash ec.validate() ec['osdependencies'] = ['non-existent-dep'] self.assertErrorRegex(EasyBuildError, "OS dependencies were not found", ec.validate) # dummy toolchain, installversion == version self.assertEqual(det_full_ec_version(ec), "3.14") os.chmod(self.eb_file, 0000) self.assertErrorRegex(EasyBuildError, "Permission denied", EasyConfig, self.eb_file) os.chmod(self.eb_file, 0755) self.contents += "\nsyntax_error'" self.prep() self.assertErrorRegex(EasyBuildError, "SyntaxError", EasyConfig, self.eb_file)
def robot_find_minimal_easyconfig_for_dependency(dep): """ Find an easyconfig with minimal toolchain for a dependency """ newdep = copy.deepcopy(dep) toolchain_hierarchy = get_toolchain_hierarchy(dep['toolchain']) res = None # reversed search: start with subtoolchains first, i.e. first (dummy or) compiler-only toolchain, etc. for toolchain in toolchain_hierarchy: newdep['toolchain'] = toolchain eb_file = robot_find_easyconfig(newdep['name'], det_full_ec_version(newdep)) if eb_file is not None: if newdep['toolchain'] != dep['toolchain']: _log.info( "Minimally resolving dependency %s using toolchain %s with %s", dep, toolchain, eb_file) res = (newdep, eb_file) break if res is None: _log.debug("Irresolvable minimal dependency found: %s", dep) return res
def det_toolchain_element_details(tc, elem): """ Determine details of a particular toolchain element, for a given Toolchain instance. """ # check for cached version first tc_dict = tc.as_dict() key = (tc_dict['name'], tc_dict['version'] + tc_dict['versionsuffix'], elem) if key in _toolchain_details_cache: _log.debug("Obtained details for '%s' in toolchain '%s' from cache" % (elem, tc_dict)) return _toolchain_details_cache[key] # grab version from parsed easyconfig file for toolchain eb_file = robot_find_easyconfig(tc_dict['name'], det_full_ec_version(tc_dict)) tc_ec = process_easyconfig(eb_file, parse_only=True) if len(tc_ec) > 1: _log.warning("More than one toolchain specification found for %s, only retaining first" % tc_dict) _log.debug("Full list of toolchain specifications: %s" % tc_ec) tc_ec = tc_ec[0]['ec'] tc_deps = tc_ec['dependencies'] tc_elem_details = None for tc_dep in tc_deps: if tc_dep['name'] == elem: tc_elem_details = tc_dep _log.debug("Found details for toolchain element %s: %s" % (elem, tc_elem_details)) break if tc_elem_details is None: # for compiler-only toolchains, toolchain and compilers are one-and-the-same if tc_ec['name'] == elem: tc_elem_details = tc_ec else: raise EasyBuildError("No toolchain element '%s' found for toolchain %s: %s", elem, tc.as_dict(), tc_ec) _toolchain_details_cache[key] = tc_elem_details _log.debug("Obtained details for '%s' in toolchain '%s', added to cache" % (elem, tc_dict)) return _toolchain_details_cache[key]
def det_toolchain_element_details(tc, elem): """ Determine details of a particular toolchain element, for a given Toolchain instance. """ # check for cached version first tc_dict = tc.as_dict() key = (tc_dict['name'], tc_dict['version'] + tc_dict['versionsuffix'], elem) if key in _toolchain_details_cache: _log.debug("Obtained details for '%s' in toolchain '%s' from cache" % (elem, tc_dict)) return _toolchain_details_cache[key] # grab version from parsed easyconfig file for toolchain eb_file = robot_find_easyconfig(tc_dict['name'], det_full_ec_version(tc_dict)) tc_ec = process_easyconfig(eb_file, parse_only=True) if len(tc_ec) > 1: _log.warning("More than one toolchain specification found for %s, only retaining first" % tc_dict) _log.debug("Full list of toolchain specifications: %s" % tc_ec) tc_ec = tc_ec[0]['ec'] tc_deps = tc_ec['dependencies'] tc_elem_details = None for tc_dep in tc_deps: if tc_dep['name'] == elem: tc_elem_details = tc_dep _log.debug("Found details for toolchain element %s: %s" % (elem, tc_elem_details)) break if tc_elem_details is None: # for compiler-only toolchains, toolchain and compilers are one-and-the-same if tc_ec['name'] == elem: tc_elem_details = tc_ec else: _log.error("No toolchain element '%s' found for toolchain %s: %s" % (elem, tc.as_dict(), tc_ec)) _toolchain_details_cache[key] = tc_elem_details _log.debug("Obtained details for '%s' in toolchain '%s', added to cache" % (elem, tc_dict)) return _toolchain_details_cache[key]
def det_full_module_name(ec, eb_ns=False): """ Determine full module name following the currently active module naming scheme. First try to pass 'parsed' easyconfig as supplied, try and find a matching easyconfig file, parse it and supply it in case of a KeyError. """ try: mod_name = _det_full_module_name(ec, eb_ns=eb_ns) except KeyError, err: _log.debug("KeyError '%s' when determining module name for %s, trying fallback procedure..." % (err, ec)) # for dependencies, only name/version/versionsuffix/toolchain easyconfig parameters are available; # when a key error occurs, try and find an easyconfig file to parse via the robot, # and retry with the parsed easyconfig file (which will contains a full set of keys) robot = build_option('robot_path') eb_file = robot_find_easyconfig(robot, ec['name'], det_full_ec_version(ec)) if eb_file is None: _log.error("Failed to find an easyconfig file when determining module name for: %s" % ec) else: parsed_ec = process_easyconfig(eb_file) if len(parsed_ec) > 1: _log.warning("More than one parsed easyconfig obtained from %s, only retaining first" % eb_file) try: mod_name = _det_full_module_name(parsed_ec[0]['ec'], eb_ns=eb_ns) except KeyError, err: _log.error("A KeyError '%s' occured when determining a module name for %s." % parsed_ec['ec'])
def create_job(build_command, easyconfig, output_dir=None, conn=None, ppn=None): """ Creates a job, to build a *single* easyconfig @param build_command: format string for command, full path to an easyconfig file will be substituted in it @param easyconfig: easyconfig as processed by process_easyconfig @param output_dir: optional output path; --regtest-output-dir will be used inside the job with this variable @param conn: open connection to PBS server @param ppn: ppn setting to use (# 'processors' (cores) per node to use) returns the job """ if output_dir is None: output_dir = 'easybuild-build' # capture PYTHONPATH, MODULEPATH and all variables starting with EASYBUILD easybuild_vars = {} for name in os.environ: if name.startswith("EASYBUILD"): easybuild_vars[name] = os.environ[name] others = ["PYTHONPATH", "MODULEPATH"] for env_var in others: if env_var in os.environ: easybuild_vars[env_var] = os.environ[env_var] _log.info("Dictionary of environment variables passed to job: %s" % easybuild_vars) # obtain unique name based on name/easyconfig version tuple ec_tuple = (easyconfig['ec']['name'], det_full_ec_version(easyconfig['ec'])) name = '-'.join(ec_tuple) # create command based on build_command template command = build_command % { 'spec': easyconfig['spec'], 'output_dir': os.path.join(os.path.abspath(output_dir), name), } # just use latest build stats repo = init_repository(get_repository(), get_repositorypath()) buildstats = repo.get_buildstats(*ec_tuple) resources = {} if buildstats: previous_time = buildstats[-1]['build_time'] resources['hours'] = int(math.ceil(previous_time * 2 / 60)) job = PbsJob(command, name, easybuild_vars, resources=resources, conn=conn, ppn=ppn) job.module = easyconfig['ec'].full_mod_name return job
def det_full_module_name(self, ec): """ Determine full module name from given easyconfig, according to the EasyBuild module naming scheme. :param ec: dict-like object with easyconfig parameter values (e.g. 'name', 'version', etc.) :return: string with full module name <name>/<installversion>, e.g.: 'gzip/1.5-goolf-1.4.10' """ return os.path.join(ec['name'], det_full_ec_version(ec)).lower()
def obtain_ec_for(specs, paths, fp): """ Obtain an easyconfig file to the given specifications. Either select between available ones, or use the best suited available one to generate a new easyconfig file. @param specs: list of available easyconfig files @param paths: a list of paths where easyconfig files can be found @param fp: the desired file name """ # ensure that at least name is specified if not specs.get('name'): _log.error( "Supplied 'specs' dictionary doesn't even contain a name of a software package?" ) # collect paths to search in if not paths: _log.error( "No paths to look for easyconfig files, specify a path with --robot." ) # create glob patterns based on supplied info # figure out the install version cfg = { 'version': specs.get('version', '*'), 'toolchain': { 'name': specs.get('toolchain_name', '*'), 'version': specs.get('toolchain_version', '*'), }, 'versionprefix': specs.get('versionprefix', '*'), 'versionsuffix': specs.get('versionsuffix', '*'), } installver = det_full_ec_version(cfg) # find easyconfigs that match a pattern easyconfig_files = [] for path in paths: patterns = create_paths(path, specs['name'], installver) for pattern in patterns: easyconfig_files.extend(glob.glob(pattern)) cnt = len(easyconfig_files) _log.debug("List of obtained easyconfig files (%d): %s" % (cnt, easyconfig_files)) # select best easyconfig, or try to generate one that fits the requirements res = select_or_generate_ec(fp, paths, specs) if res: return res else: _log.error( "No easyconfig found for requested software, and also failed to generate one." )
def create_job(job_backend, build_command, easyconfig, output_dir='easybuild-build'): """ Creates a job to build a *single* easyconfig. :param job_backend: A factory object for querying server parameters and creating actual job objects :param build_command: format string for command, full path to an easyconfig file will be substituted in it :param easyconfig: easyconfig as processed by process_easyconfig :param output_dir: optional output path; --regtest-output-dir will be used inside the job with this variable returns the job """ # capture PYTHONPATH, MODULEPATH and all variables starting with EASYBUILD easybuild_vars = {} for name in os.environ: if name.startswith("EASYBUILD"): easybuild_vars[name] = os.environ[name] for env_var in ["PYTHONPATH", "MODULEPATH"]: if env_var in os.environ: easybuild_vars[env_var] = os.environ[env_var] _log.info("Dictionary of environment variables passed to job: %s" % easybuild_vars) # obtain unique name based on name/easyconfig version tuple ec_tuple = (easyconfig['ec']['name'], det_full_ec_version(easyconfig['ec'])) name = '-'.join(ec_tuple) # determine whether additional options need to be passed to the 'eb' command add_opts = '' if easyconfig['hidden']: add_opts += ' --hidden' # create command based on build_command template command = build_command % { 'add_opts': add_opts, 'output_dir': os.path.join(os.path.abspath(output_dir), name), 'spec': easyconfig['spec'], } # just use latest build stats repo = init_repository(get_repository(), get_repositorypath()) buildstats = repo.get_buildstats(*ec_tuple) extra = {} if buildstats: previous_time = buildstats[-1]['build_time'] extra['hours'] = int(math.ceil(previous_time * 2 / 60)) if build_option('job_cores'): extra['cores'] = build_option('job_cores') job = job_backend.make_job(command, name, easybuild_vars, **extra) job.module = easyconfig['ec'].full_mod_name return job
def det_full_module_name(self, ec): """ Determine full module name from given easyconfig, according to the EasyBuild module naming scheme. :param ec: dict-like object with easyconfig parameter values (e.g. 'name', 'version', etc.) :return: string with full module name <name>/<installversion>, e.g.: 'gzip/1.5-goolf-1.4.10' """ return os.path.join(ec['name'], det_full_ec_version(ec))
def test_dependency(self): """ test all possible ways of specifying dependencies """ self.contents = '\n'.join([ 'easyblock = "ConfigureMake"', 'name = "pi"', 'version = "3.14"', 'homepage = "http://example.com"', 'description = "test easyconfig"', 'toolchain = {"name":"GCC", "version": "4.6.3"}', 'dependencies = [("first", "1.1"), {"name": "second", "version": "2.2"}]', 'builddependencies = [("first", "1.1"), {"name": "second", "version": "2.2"}]', ]) self.prep() eb = EasyConfig(self.eb_file) # should include builddependencies self.assertEqual(len(eb.dependencies()), 4) self.assertEqual(len(eb.builddependencies()), 2) first = eb.dependencies()[0] second = eb.dependencies()[1] self.assertEqual(first['name'], "first") self.assertEqual(second['name'], "second") self.assertEqual(first['version'], "1.1") self.assertEqual(second['version'], "2.2") self.assertEqual(det_full_ec_version(first), '1.1-GCC-4.6.3') self.assertEqual(det_full_ec_version(second), '2.2-GCC-4.6.3') # same tests for builddependencies first = eb.builddependencies()[0] second = eb.builddependencies()[1] self.assertEqual(first['name'], "first") self.assertEqual(second['name'], "second") self.assertEqual(first['version'], "1.1") self.assertEqual(second['version'], "2.2") self.assertEqual(det_full_ec_version(first), '1.1-GCC-4.6.3') self.assertEqual(det_full_ec_version(second), '2.2-GCC-4.6.3') self.assertErrorRegex(EasyBuildError, "Dependency foo of unsupported type", eb._parse_dependency, "foo") self.assertErrorRegex(EasyBuildError, "without name", eb._parse_dependency, ()) self.assertErrorRegex(EasyBuildError, "without version", eb._parse_dependency, {'name': 'test'})
def det_full_module_name(self, ec): """ Determine full module name from given easyconfig, according to the thematic module naming scheme. :param ec: dict-like object with easyconfig parameter values (e.g. 'name', 'version', etc.) :return: string representing full module name, e.g.: 'biology/ABySS/1.3.4-goolf-1.4.10' """ return os.path.join(ec['moduleclass'], ec['name'], det_full_ec_version(ec))
def ec_filename_for(path): """ Return a suiting file name for the easyconfig file at <path>, as determined by its contents. """ ec = EasyConfig(path, validate=False) fn = "%s-%s.eb" % (ec['name'], det_full_ec_version(ec)) return fn
def ec_filename_for(path): """ Return a suiting file name for the easyconfig file at <path>, as determined by its contents. """ ec = EasyConfig(path, build_options={"validate": False}) fn = "%s-%s.eb" % (ec["name"], det_full_ec_version(ec)) return fn
def check_ec_type(self, ec): """ Obtain a full parsed easyconfig file to pass to naming scheme methods if provided keys are insufficient. """ if not isinstance(ec, EasyConfig) and self.requires_full_easyconfig(ec.keys()): self.log.debug("A parsed easyconfig is required by the module naming scheme, so finding one for %s" % ec) # fetch/parse easyconfig file if deemed necessary eb_file = robot_find_easyconfig(ec['name'], det_full_ec_version(ec)) if eb_file is not None: parsed_ec = process_easyconfig(eb_file, parse_only=True, hidden=ec['hidden']) if len(parsed_ec) > 1: self.log.warning("More than one parsed easyconfig obtained from %s, only retaining first" % eb_file) self.log.debug("Full list of parsed easyconfigs: %s" % parsed_ec) ec = parsed_ec[0]['ec'] else: tup = (ec['name'], det_full_ec_version(ec), ec) self.log.error("Failed to find easyconfig file '%s-%s.eb' when determining module name for: %s" % tup) return ec
def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir=None): """ Take an easyconfig spec, parse it, map it to a target toolchain and dump it out :param ec_spec: Location of original easyconfig file :param toolchain_mapping: Mapping between source toolchain and target toolchain :param targetdir: Directory to dump the modified easyconfig file in :return: Location of the modified easyconfig file """ # Fully parse the original easyconfig parsed_ec = process_easyconfig(ec_spec, validate=False)[0] # Replace the toolchain if the mapping exists tc_name = parsed_ec['ec']['toolchain']['name'] if tc_name in toolchain_mapping: new_toolchain = toolchain_mapping[tc_name] _log.debug("Replacing parent toolchain %s with %s", parsed_ec['ec']['toolchain'], new_toolchain) parsed_ec['ec']['toolchain'] = new_toolchain # Replace the toolchains of all the dependencies for key in DEPENDENCY_PARAMETERS: # loop over a *copy* of dependency dicts (with resolved templates); # to update the original dep dict, we need to index with idx into self._config[key][0]... for idx, dep in enumerate(parsed_ec['ec'][key]): # reference to original dep dict, this is the one we should be updating orig_dep = parsed_ec['ec']._config[key][0][idx] # skip dependencies that are marked as external modules if dep['external_module']: continue dep_tc_name = dep['toolchain']['name'] if dep_tc_name in toolchain_mapping: orig_dep['toolchain'] = toolchain_mapping[dep_tc_name] # Replace the binutils version (if necessary) if 'binutils' in toolchain_mapping and ( dep['name'] == 'binutils' and dep_tc_name == GCCcore.NAME): orig_dep.update(toolchain_mapping['binutils']) # set module names orig_dep['short_mod_name'] = ActiveMNS().det_short_module_name( dep) orig_dep['full_mod_name'] = ActiveMNS().det_full_module_name( dep) # Determine the name of the modified easyconfig and dump it to target_dir ec_filename = '%s-%s.eb' % (parsed_ec['ec']['name'], det_full_ec_version(parsed_ec['ec'])) tweaked_spec = os.path.join(targetdir or tempfile.gettempdir(), ec_filename) parsed_ec['ec'].dump(tweaked_spec, always_overwrite=False, backup=True) _log.debug("Dumped easyconfig tweaked via --try-toolchain* to %s", tweaked_spec) return tweaked_spec
def det_installversion(version, toolchain_name, toolchain_version, prefix, suffix): """Deprecated 'det_installversion' function, to determine exact install version, based on supplied parameters.""" old_fn = 'framework.easyconfig.easyconfig.det_installversion' _log.deprecated('Use module_generator.det_full_ec_version instead of %s' % old_fn, '2.0') cfg = { 'version': version, 'toolchain': {'name': toolchain_name, 'version': toolchain_version}, 'versionprefix': prefix, 'versionsuffix': suffix, } return det_full_ec_version(cfg)
def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir=None): """ Take an easyconfig spec, parse it, map it to a target toolchain and dump it out :param ec_spec: Location of original easyconfig file :param toolchain_mapping: Mapping between source toolchain and target toolchain :param targetdir: Directory to dump the modified easyconfig file in :return: Location of the modified easyconfig file """ # Fully parse the original easyconfig parsed_ec = process_easyconfig(ec_spec, validate=False)[0]['ec'] # Replace the toolchain if the mapping exists tc_name = parsed_ec['toolchain']['name'] if tc_name in toolchain_mapping: new_toolchain = toolchain_mapping[tc_name] _log.debug("Replacing parent toolchain %s with %s", parsed_ec['toolchain'], new_toolchain) parsed_ec['toolchain'] = new_toolchain # Replace the toolchains of all the dependencies for key in DEPENDENCY_PARAMETERS: # loop over a *copy* of dependency dicts (with resolved templates); # to update the original dep dict, we need to get a reference with templating disabled... val = parsed_ec[key] orig_val = parsed_ec.get_ref(key) if key in parsed_ec.iterate_options: val = flatten(val) orig_val = flatten(orig_val) for idx, dep in enumerate(val): # reference to original dep dict, this is the one we should be updating orig_dep = orig_val[idx] # skip dependencies that are marked as external modules if dep['external_module']: continue dep_tc_name = dep['toolchain']['name'] if dep_tc_name in toolchain_mapping: orig_dep['toolchain'] = toolchain_mapping[dep_tc_name] # Replace the binutils version (if necessary) if 'binutils' in toolchain_mapping and (dep['name'] == 'binutils' and dep_tc_name == GCCcore.NAME): orig_dep.update(toolchain_mapping['binutils']) # set module names orig_dep['short_mod_name'] = ActiveMNS().det_short_module_name(dep) orig_dep['full_mod_name'] = ActiveMNS().det_full_module_name(dep) # Determine the name of the modified easyconfig and dump it to target_dir ec_filename = '%s-%s.eb' % (parsed_ec['name'], det_full_ec_version(parsed_ec)) tweaked_spec = os.path.join(targetdir or tempfile.gettempdir(), ec_filename) parsed_ec.dump(tweaked_spec, always_overwrite=False, backup=True) _log.debug("Dumped easyconfig tweaked via --try-toolchain* to %s", tweaked_spec) return tweaked_spec
def compose_ec_filenames(self): """Returns a list of all easyconfig names""" ec_filenames = [] for sw in self.software_list: full_ec_version = det_full_ec_version({ 'toolchain': {'name': sw.toolchain_name, 'version': sw.toolchain_version}, 'version': sw.version, 'versionsuffix': sw.versionsuffix, }) ec_filename = '%s-%s.eb' % (sw.name, full_ec_version) ec_filenames.append(ec_filename) return ec_filenames
def obtain_ec_for(specs, paths, fp): """ Obtain an easyconfig file to the given specifications. Either select between available ones, or use the best suited available one to generate a new easyconfig file. @param specs: list of available easyconfig files @param paths: a list of paths where easyconfig files can be found @param fp: the desired file name """ # ensure that at least name is specified if not specs.get('name'): _log.error("Supplied 'specs' dictionary doesn't even contain a name of a software package?") # collect paths to search in if not paths: _log.error("No paths to look for easyconfig files, specify a path with --robot.") # create glob patterns based on supplied info # figure out the install version cfg = { 'version': specs.get('version', '*'), 'toolchain': { 'name': specs.get('toolchain_name', '*'), 'version': specs.get('toolchain_version', '*'), }, 'versionprefix': specs.get('versionprefix', '*'), 'versionsuffix': specs.get('versionsuffix', '*'), } installver = det_full_ec_version(cfg) # find easyconfigs that match a pattern easyconfig_files = [] for path in paths: patterns = create_paths(path, specs['name'], installver) for pattern in patterns: easyconfig_files.extend(glob.glob(pattern)) cnt = len(easyconfig_files) _log.debug("List of obtained easyconfig files (%d): %s" % (cnt, easyconfig_files)) # select best easyconfig, or try to generate one that fits the requirements res = select_or_generate_ec(fp, paths, specs) if res: return res else: _log.error("No easyconfig found for requested software, and also failed to generate one.")
def create_job(build_command, easyconfig, output_dir="", conn=None, ppn=None): """ Creates a job, to build a *single* easyconfig build_command is a format string in which a full path to an eb file will be substituted easyconfig should be in the format as processEasyConfig returns them output_dir is an optional path. EASYBUILDTESTOUTPUT will be set inside the job with this variable returns the job """ # create command based on build_command template command = build_command % {'spec': easyconfig['spec']} # capture PYTHONPATH, MODULEPATH and all variables starting with EASYBUILD easybuild_vars = {} for name in os.environ: if name.startswith("EASYBUILD"): easybuild_vars[name] = os.environ[name] others = ["PYTHONPATH", "MODULEPATH"] for env_var in others: if env_var in os.environ: easybuild_vars[env_var] = os.environ[env_var] _log.info("Dictionary of environment variables passed to job: %s" % easybuild_vars) # obtain unique name based on name/easyconfig version tuple ec_tuple = (easyconfig['ec']['name'], det_full_ec_version(easyconfig['ec'])) name = '-'.join(ec_tuple) var = config.oldstyle_environment_variables['test_output_path'] easybuild_vars[var] = os.path.join(os.path.abspath(output_dir), name) # just use latest build stats repo = init_repository(get_repository(), get_repositorypath()) buildstats = repo.get_buildstats(*ec_tuple) resources = {} if buildstats: previous_time = buildstats[-1]['build_time'] resources['hours'] = int(math.ceil(previous_time * 2 / 60)) job = PbsJob(command, name, easybuild_vars, resources=resources, conn=conn, ppn=ppn) job.module = det_full_module_name(easyconfig['ec']) return job
def det_full_module_name(self, ec): """ Determine full module name from given easyconfig, according to the EasyBuild module naming scheme. :param ec: dict-like object with easyconfig parameter values (e.g. 'name', 'version', etc.) :return: string with lowercase of full module name <name>/<installversion>, e.g.: 'gzip/1.5-goolf-1.4.10 """ # fetch required values name = ec['name'] fversion = det_full_ec_version(ec) #version = ec['version'] #tc_name = ec['toolchain']['name'] #tc_version = ec['toolchain']['version'] # compose module name by lowercasing and stitching parts together return os.path.join(name.lower(), fversion.lower())
def check_ec_type(self, ec): """ Query module naming scheme using specified method and argument. Obtain and pass a full parsed easyconfig file if provided keys are insufficient. """ if not isinstance(ec, EasyConfig) and self.requires_full_easyconfig(ec.keys()): self.log.debug("A parsed easyconfig is required by the module naming scheme, so finding one for %s" % ec) # fetch/parse easyconfig file if deemed necessary eb_file = robot_find_easyconfig(ec['name'], det_full_ec_version(ec)) if eb_file is not None: parsed_ec = process_easyconfig(eb_file, parse_only=True) if len(parsed_ec) > 1: self.log.warning("More than one parsed easyconfig obtained from %s, only retaining first" % eb_file) self.log.debug("Full list of parsed easyconfigs: %s" % parsed_ec) ec = parsed_ec[0]['ec'] else: self.log.error("Failed to find an easyconfig file when determining module name for: %s" % ec) return ec
def robot_find_minimal_easyconfig_for_dependency(dep): """ Find an easyconfig with minimal toolchain for a dependency """ newdep = copy.deepcopy(dep) toolchain_hierarchy = get_toolchain_hierarchy(dep['toolchain']) res = None # reversed search: start with subtoolchains first, i.e. first (dummy or) compiler-only toolchain, etc. for toolchain in toolchain_hierarchy: newdep['toolchain'] = toolchain eb_file = robot_find_easyconfig(newdep['name'], det_full_ec_version(newdep)) if eb_file is not None: if newdep['toolchain'] != dep['toolchain']: _log.info("Minimally resolving dependency %s using toolchain %s with %s", dep, toolchain, eb_file) res = (newdep, eb_file) break if res is None: _log.debug("Irresolvable minimal dependency found: %s", dep) return res
def det_full_module_name(ec, eb_ns=False): """ Determine full module name following the currently active module naming scheme. First try to pass 'parsed' easyconfig as supplied, try and find a matching easyconfig file, parse it and supply it in case of a KeyError. """ try: mod_name = _det_full_module_name(ec, eb_ns=eb_ns) except KeyError, err: _log.debug( "KeyError '%s' when determining module name for %s, trying fallback procedure..." % (err, ec)) # for dependencies, only name/version/versionsuffix/toolchain easyconfig parameters are available; # when a key error occurs, try and find an easyconfig file to parse via the robot, # and retry with the parsed easyconfig file (which will contains a full set of keys) robot = build_option('robot_path') eb_file = robot_find_easyconfig(robot, ec['name'], det_full_ec_version(ec)) if eb_file is None: _log.error( "Failed to find an easyconfig file when determining module name for: %s" % ec) else: parsed_ec = process_easyconfig(eb_file) if len(parsed_ec) > 1: _log.warning( "More than one parsed easyconfig obtained from %s, only retaining first" % eb_file) try: mod_name = _det_full_module_name(parsed_ec[0]['ec'], eb_ns=eb_ns) except KeyError, err: _log.error( "A KeyError '%s' occured when determining a module name for %s." % parsed_ec['ec'])
def mk_key(spec): """Create key for dictionary with all dependencies.""" if "ec" in spec: spec = spec["ec"] return (spec["name"], det_full_ec_version(spec))
def select_or_generate_ec(fp, paths, specs): """ Select or generate an easyconfig file with the given requirements, from existing easyconfig files. If easyconfig files are available for the specified software package, then this function will first try to determine which toolchain to use. * if a toolchain is given, it will use it (possible using a template easyconfig file as base); * if not, and only a single toolchain is available, is will assume it can use that toolchain * else, it fails -- EasyBuild doesn't select between multiple available toolchains Next, it will trim down the selected easyconfig files to a single one, based on the following requirements (in order of preference): * toolchain version * software version * other parameters (e.g. versionprefix, versionsuffix, etc.) If a complete match is found, it will return that easyconfig. Else, it will generate a new easyconfig file based on the selected 'best matching' easyconfig file. """ specs = copy.deepcopy(specs) # ensure that at least name is specified if not specs.get('name'): raise EasyBuildError( "Supplied 'specs' dictionary doesn't even contain a name of a software package?" ) name = specs['name'] handled_params = ['name'] # find ALL available easyconfig files for specified software cfg = { 'version': '*', 'toolchain': { 'name': DUMMY_TOOLCHAIN_NAME, 'version': '*' }, 'versionprefix': '*', 'versionsuffix': '*', } installver = det_full_ec_version(cfg) ec_files = find_matching_easyconfigs(name, installver, paths) _log.debug("Unique ec_files: %s" % ec_files) # we need at least one config file to start from if len(ec_files) == 0: # look for a template file if no easyconfig for specified software name is available for path in paths: templ_file = os.path.join(path, "%s.eb" % EASYCONFIG_TEMPLATE) if os.path.isfile(templ_file): ec_files = [templ_file] break else: _log.debug("No template found at %s." % templ_file) if len(ec_files) == 0: raise EasyBuildError( "No easyconfig files found for software %s, and no templates available. " "I'm all out of ideas.", name) ecs_and_files = [(EasyConfig(f, validate=False), f) for f in ec_files] # TOOLCHAIN NAME # we can't rely on set, because we also need to be able to obtain a list of unique lists def unique(l): """Retain unique elements in a sorted list.""" l = sorted(l) if len(l) > 1: l2 = [l[0]] for x in l: if not x == l2[-1]: l2.append(x) return l2 else: return l # determine list of unique toolchain names tcnames = unique([x[0]['toolchain']['name'] for x in ecs_and_files]) _log.debug("Found %d unique toolchain names: %s" % (len(tcnames), tcnames)) # if a toolchain was selected, and we have no easyconfig files for it, try and use a template if specs.get('toolchain_name') and not specs['toolchain_name'] in tcnames: if EASYCONFIG_TEMPLATE in tcnames: _log.info( "No easyconfig file for specified toolchain, but template is available." ) else: raise EasyBuildError( "No easyconfig file for %s with toolchain %s, and no template available.", name, specs['toolchain_name']) tcname = specs.pop('toolchain_name', None) handled_params.append('toolchain_name') # trim down list according to selected toolchain if tcname in tcnames: # known toolchain, so only retain those selected_tcname = tcname else: if len(tcnames) == 1 and not tcnames[0] == EASYCONFIG_TEMPLATE: # only one (non-template) toolchain availble, so use that tcname = tcnames[0] selected_tcname = tcname elif len(tcnames) == 1 and tcnames[0] == EASYCONFIG_TEMPLATE: selected_tcname = tcnames[0] else: # fall-back: use template toolchain if a toolchain name was specified if tcname: selected_tcname = EASYCONFIG_TEMPLATE else: # if multiple toolchains are available, and none is specified, we quit # we can't just pick one, how would we prefer one over the other? raise EasyBuildError( "No toolchain name specified, and more than one available: %s.", tcnames) _log.debug("Filtering easyconfigs based on toolchain name '%s'..." % selected_tcname) ecs_and_files = [ x for x in ecs_and_files if x[0]['toolchain']['name'] == selected_tcname ] _log.debug("Filtered easyconfigs: %s" % [x[1] for x in ecs_and_files]) # TOOLCHAIN VERSION tcvers = unique([x[0]['toolchain']['version'] for x in ecs_and_files]) _log.debug("Found %d unique toolchain versions: %s" % (len(tcvers), tcvers)) tcver = specs.pop('toolchain_version', None) handled_params.append('toolchain_version') (tcver, selected_tcver) = pick_version(tcver, tcvers) _log.debug("Filtering easyconfigs based on toolchain version '%s'..." % selected_tcver) ecs_and_files = [ x for x in ecs_and_files if x[0]['toolchain']['version'] == selected_tcver ] _log.debug("Filtered easyconfigs: %s" % [x[1] for x in ecs_and_files]) # add full toolchain specification to specs if tcname and tcver: specs.update({'toolchain': {'name': tcname, 'version': tcver}}) handled_params.append('toolchain') else: if tcname: specs.update({'toolchain_name': tcname}) if tcver: specs.update({'toolchain_version': tcver}) # SOFTWARE VERSION vers = unique([x[0]['version'] for x in ecs_and_files]) _log.debug("Found %d unique software versions: %s" % (len(vers), vers)) ver = specs.pop('version', None) handled_params.append('version') (ver, selected_ver) = pick_version(ver, vers) if ver: specs.update({'version': ver}) _log.debug("Filtering easyconfigs based on software version '%s'..." % selected_ver) ecs_and_files = [ x for x in ecs_and_files if x[0]['version'] == selected_ver ] _log.debug("Filtered easyconfigs: %s" % [x[1] for x in ecs_and_files]) # go through parameters specified via --amend # always include versionprefix/suffix, because we might need it to generate a file name verpref = None versuff = None other_params = {'versionprefix': None, 'versionsuffix': None} for (param, val) in specs.items(): if not param in handled_params: other_params.update({param: val}) _log.debug( "Filtering based on other parameters (specified via --amend): %s" % other_params) for (param, val) in other_params.items(): if param in ecs_and_files[0][0]._config: vals = unique([x[0][param] for x in ecs_and_files]) else: vals = [] filter_ecs = False # try and select a value from the available ones, or fail if we can't if val in vals: # if the specified value is available, use it selected_val = val _log.debug("Specified %s is available, so using it: %s" % (param, selected_val)) filter_ecs = True elif val: # if a value is specified, use that, even if it's not available yet selected_val = val # promote value to list if deemed appropriate if vals and type(vals[0]) == list and not type(val) == list: _log.debug( "Promoting type of %s value to list, since original value was." % param) specs[param] = [val] _log.debug( "%s is specified, so using it (even though it's not available yet): %s" % (param, selected_val)) elif len(vals) == 1: # if only one value is available, use that selected_val = vals[0] _log.debug("Only one %s available ('%s'), so picking that" % (param, selected_val)) filter_ecs = True else: # otherwise, we fail, because we don't know how to pick between different fixes raise EasyBuildError( "No %s specified, and can't pick from available ones: %s", param, vals) if filter_ecs: _log.debug("Filtering easyconfigs based on %s '%s'..." % (param, selected_val)) ecs_and_files = [ x for x in ecs_and_files if x[0][param] == selected_val ] _log.debug("Filtered easyconfigs: %s" % [x[1] for x in ecs_and_files]) # keep track of versionprefix/suffix if param == "versionprefix": verpref = selected_val elif param == "versionsuffix": versuff = selected_val cnt = len(ecs_and_files) if not cnt == 1: fs = [x[1] for x in ecs_and_files] raise EasyBuildError( "Failed to select a single easyconfig from available ones, %s left: %s", cnt, fs) else: (selected_ec, selected_ec_file) = ecs_and_files[0] # check whether selected easyconfig matches requirements match = True for (key, val) in specs.items(): if key in selected_ec._config: # values must be equal to have a full match if selected_ec[key] != val: match = False else: # if we encounter a key that is not set in the selected easyconfig, we don't have a full match match = False # if it matches, no need to tweak if match: _log.info("Perfect match found: %s" % selected_ec_file) return (False, selected_ec_file) # GENERATE # if no file path was specified, generate a file name if fp is None: cfg = { 'version': ver, 'toolchain': { 'name': tcname, 'version': tcver }, 'versionprefix': verpref, 'versionsuffix': versuff, } installver = det_full_ec_version(cfg) fp = "%s-%s.eb" % (name, installver) # generate tweaked easyconfig file tweak_one(selected_ec_file, fp, specs) _log.info( "Generated easyconfig file %s, and using it to build the requested software." % fp) return (True, fp)
def name(self, ec): """Determine package name""" self.log.debug("Easyconfig dict passed to name() looks like: %s ", ec) return '%s-%s' % (ec['name'], det_full_ec_version(ec))
def resolve_dependencies(easyconfigs, modtool, retain_all_deps=False, raise_error_missing_ecs=True): """ Work through the list of easyconfigs to determine an optimal order :param easyconfigs: list of easyconfigs :param modtool: ModulesTool instance to use :param retain_all_deps: boolean indicating whether all dependencies must be retained, regardless of availability; retain all deps when True, check matching build option when False :param raise_error_missing_ecs: raise an error when one or more easyconfig files could not be found """ robot = build_option('robot_path') # retain all dependencies if specified by either the resp. build option or the dedicated named argument retain_all_deps = build_option('retain_all_deps') or retain_all_deps avail_modules = modtool.available() if retain_all_deps: # assume that no modules are available when forced, to retain all dependencies avail_modules = [] _log.info("Forcing all dependencies to be retained.") else: if len(avail_modules) == 0: _log.warning("No installed modules. Your MODULEPATH is probably incomplete: %s" % os.getenv('MODULEPATH')) ordered_ecs = [] # all available modules can be used for resolving dependencies except those that will be installed being_installed = [p['full_mod_name'] for p in easyconfigs] avail_modules = [m for m in avail_modules if m not in being_installed] _log.debug('easyconfigs before resolving deps: %s', easyconfigs) totally_missing, missing_easyconfigs = [], [] # resolve all dependencies, put a safeguard in place to avoid an infinite loop (shouldn't occur though) loopcnt = 0 maxloopcnt = 10000 while easyconfigs: # make sure this stops, we really don't want to get stuck in an infinite loop loopcnt += 1 if loopcnt > maxloopcnt: raise EasyBuildError("Maximum loop cnt %s reached, so quitting (easyconfigs: %s, missing_easyconfigs: %s)", maxloopcnt, easyconfigs, missing_easyconfigs) # first try resolving dependencies without using external dependencies last_processed_count = -1 while len(avail_modules) > last_processed_count: last_processed_count = len(avail_modules) res = find_resolved_modules(easyconfigs, avail_modules, modtool, retain_all_deps=retain_all_deps) resolved_ecs, easyconfigs, avail_modules = res ordered_ec_mod_names = [x['full_mod_name'] for x in ordered_ecs] for ec in resolved_ecs: # only add easyconfig if it's not included yet (based on module name) if not ec['full_mod_name'] in ordered_ec_mod_names: ordered_ecs.append(ec) # dependencies marked as external modules should be resolved via available modules at this point missing_external_modules = [d['full_mod_name'] for ec in easyconfigs for d in ec['dependencies'] if d.get('external_module', False)] if missing_external_modules: raise EasyBuildError("Missing modules for dependencies marked as external modules: %s", ', '.join(missing_external_modules)) # robot: look for existing dependencies, add them if robot and easyconfigs: # rely on EasyBuild module naming scheme when resolving dependencies, since we know that will # generate sensible module names that include the necessary information for the resolution to work # (name, version, toolchain, versionsuffix) being_installed = [EasyBuildMNS().det_full_module_name(p['ec']) for p in easyconfigs] additional = [] for entry in easyconfigs: # do not choose an entry that is being installed in the current run # if they depend, you probably want to rebuild them using the new dependency deps = entry['dependencies'] candidates = [d for d in deps if not EasyBuildMNS().det_full_module_name(d) in being_installed] if candidates: cand_dep = candidates[0] # find easyconfig, might not find any _log.debug("Looking for easyconfig for %s" % str(cand_dep)) # note: robot_find_easyconfig may return None path = robot_find_easyconfig(cand_dep['name'], det_full_ec_version(cand_dep)) if path is None: full_mod_name = ActiveMNS().det_full_module_name(cand_dep) # no easyconfig found + no module available => missing dependency if not modtool.exist([full_mod_name])[0]: if cand_dep not in totally_missing: totally_missing.append(cand_dep) # no easyconfig found for dependency, but module is available # => add to list of missing easyconfigs elif cand_dep not in missing_easyconfigs: _log.debug("Irresolvable dependency found (no easyconfig file): %s", cand_dep) missing_easyconfigs.append(cand_dep) # remove irresolvable dependency from list of dependencies so we can continue entry['dependencies'].remove(cand_dep) # add dummy entry for this dependency, so --dry-run for example can still report the dep additional.append({ 'dependencies': [], 'ec': None, 'full_mod_name': full_mod_name, 'spec': None, }) else: _log.info("Robot: resolving dependency %s with %s" % (cand_dep, path)) # build specs should not be passed down to resolved dependencies, # to avoid that e.g. --try-toolchain trickles down into the used toolchain itself hidden = cand_dep.get('hidden', False) processed_ecs = process_easyconfig(path, validate=not retain_all_deps, hidden=hidden) # ensure that selected easyconfig provides required dependency verify_easyconfig_filename(path, cand_dep, parsed_ec=processed_ecs) for ec in processed_ecs: if ec not in easyconfigs + additional: additional.append(ec) _log.debug("Added %s as dependency of %s" % (ec, entry)) else: mod_name = EasyBuildMNS().det_full_module_name(entry['ec']) _log.debug("No more candidate dependencies to resolve for %s" % mod_name) # add additional (new) easyconfigs to list of stuff to process easyconfigs.extend(additional) _log.debug("Unprocessed dependencies: %s", easyconfigs) elif not robot: # no use in continuing if robot is not enabled, dependencies won't be resolved anyway missing_deps = [dep for x in easyconfigs for dep in x['dependencies']] if missing_deps: raise_error_missing_deps(missing_deps, extra_msg="enable dependency resolution via --robot?") if totally_missing: raise_error_missing_deps(totally_missing, extra_msg="no easyconfig file or existing module found") if missing_easyconfigs: if raise_error_missing_ecs: raise_error_missing_deps(missing_easyconfigs, extra_msg="no easyconfig file found in robot search path") else: _log.warning("No easyconfig files found for: %s", missing_easyconfigs) _log.info("Dependency resolution complete, building as follows: %s", ordered_ecs) return ordered_ecs
def test_robot_find_minimal_toolchain_of_dependency(self): """Test robot_find_minimal_toolchain_of_dependency.""" # replace log.experimental with log.warning to allow experimental code easybuild.framework.easyconfig.tools._log.experimental = easybuild.framework.easyconfig.tools._log.warning test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') init_config(build_options={ 'valid_module_classes': module_classes(), 'robot_path': test_easyconfigs, }) # # First test that it can do basic resolution # gzip15 = { 'name': 'gzip', 'version': '1.5', 'versionsuffix': '', 'toolchain': {'name': 'goolf', 'version': '1.4.10'}, } get_toolchain_hierarchy.clear() new_gzip15_toolchain = robot_find_minimal_toolchain_of_dependency(gzip15, self.modtool) self.assertEqual(new_gzip15_toolchain, gzip15['toolchain']) # no easyconfig for gzip 1.4 with matching non-dummy (sub)toolchain gzip14 = { 'name': 'gzip', 'version': '1.4', 'versionsuffix': '', 'toolchain': {'name': 'goolf', 'version': '1.4.10'}, } get_toolchain_hierarchy.clear() self.assertEqual(robot_find_minimal_toolchain_of_dependency(gzip14, self.modtool), None) gzip14['toolchain'] = {'name': 'gompi', 'version': '1.4.10'} # # Second test also including dummy toolchain # init_config(build_options={ 'add_dummy_to_minimal_toolchains': True, 'valid_module_classes': module_classes(), 'robot_path': test_easyconfigs, }) # specify alternative parent toolchain gompi_1410 = {'name': 'gompi', 'version': '1.4.10'} get_toolchain_hierarchy.clear() new_gzip14_toolchain = robot_find_minimal_toolchain_of_dependency(gzip14, self.modtool, parent_tc=gompi_1410) self.assertTrue(new_gzip14_toolchain != gzip14['toolchain']) self.assertEqual(new_gzip14_toolchain, {'name': 'dummy', 'version': ''}) # default: use toolchain from dependency gzip14['toolchain'] = gompi_1410 get_toolchain_hierarchy.clear() new_gzip14_toolchain = robot_find_minimal_toolchain_of_dependency(gzip14, self.modtool) self.assertTrue(new_gzip14_toolchain != gzip14['toolchain']) self.assertEqual(new_gzip14_toolchain, {'name': 'dummy', 'version': ''}) # check reversed order (parent tc first) and skipping of parent tc itself dep = { 'name': 'SQLite', 'version': '3.8.10.2', 'toolchain': {'name': 'goolf', 'version': '1.4.10'}, } res = robot_find_minimal_toolchain_of_dependency(dep, self.modtool) self.assertEqual(res, {'name': 'GCC', 'version': '4.7.2'}) res = robot_find_minimal_toolchain_of_dependency(dep, self.modtool, parent_first=True) self.assertEqual(res, {'name': 'goolf', 'version': '1.4.10'}) # # Finally test if it can recognise existing modules and use those # barec = os.path.join(self.test_prefix, 'bar-1.2.3-goolf-1.4.10.eb') barec_txt = '\n'.join([ "easyblock = 'ConfigureMake'", "name = 'bar'", "version = '1.2.3'", "homepage = 'http://example.com'", "description = 'foo'", "toolchain = {'name': 'goolf', 'version': '1.4.10'}", # deliberately listing components of toolchain as dependencies without specifying subtoolchains, # to test resolving of dependencies with minimal toolchain # for each of these, we know test easyconfigs are available (which are required here) "dependencies = [", " ('OpenMPI', '1.6.4'),", # available with GCC/4.7.2 " ('OpenBLAS', '0.2.6', '-LAPACK-3.4.2'),", # available with gompi/1.4.10 " ('ScaLAPACK', '2.0.2', '-OpenBLAS-0.2.6-LAPACK-3.4.2'),", # available with gompi/1.4.10 " ('SQLite', '3.8.10.2'),", # available with goolf/1.4.10, gompi/1.4.10 and GCC/4.7.2 "]", ]) write_file(barec, barec_txt) # check without --minimal-toolchains init_config(build_options={ 'valid_module_classes': module_classes(), 'robot_path': test_easyconfigs, }) bar = EasyConfig(barec) expected_dep_versions = [ '1.6.4-GCC-4.7.2', '0.2.6-gompi-1.4.10-LAPACK-3.4.2', '2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2', '3.8.10.2-goolf-1.4.10', ] for dep, expected_dep_version in zip(bar.dependencies(), expected_dep_versions): self.assertEqual(det_full_ec_version(dep), expected_dep_version) # check with --minimal-toolchains enabled init_config(build_options={ 'minimal_toolchains': True, 'valid_module_classes': module_classes(), 'robot_path': test_easyconfigs, }) bar = EasyConfig(barec) # check that all bar dependencies have been processed as expected expected_dep_versions[-1] = '3.8.10.2-GCC-4.7.2' for dep, expected_dep_version in zip(bar.dependencies(), expected_dep_versions): self.assertEqual(det_full_ec_version(dep), expected_dep_version) # Add the gompi/1.4.10 version of SQLite as an available module module_parent = os.path.join(self.test_prefix, 'minimal_toolchain_modules') module_file = os.path.join(module_parent, 'SQLite', '3.8.10.2-gompi-1.4.10') module_txt = '\n'.join([ "#%Module", "set root /tmp/SQLite/3.8.10.2", "setenv EBROOTSQLITE $root", "setenv EBVERSIONSQLITE 3.8.10.2", "setenv EBDEVELSQLITE $root/easybuild/SQLite-3.8.10.2-easybuild-devel", ]) write_file(module_file, module_txt) os.environ['MODULEPATH'] = module_parent # Add the parent directory to the MODULEPATH invalidate_module_caches_for(module_parent) # Reinitialize the environment for the updated MODULEPATH and use_existing_modules init_config(build_options={ 'minimal_toolchains': True, 'use_existing_modules': True, 'valid_module_classes': module_classes(), 'robot_path': test_easyconfigs, }) # Check gompi is now being picked up bar = EasyConfig(barec) # Re-parse the parent easyconfig sqlite = bar.dependencies()[3] self.assertEqual(det_full_ec_version(sqlite), '3.8.10.2-gompi-1.4.10') # Add the goolf version as an available version and check that gets precedence over the gompi version module_file = os.path.join(module_parent, 'SQLite', '3.8.10.2-goolf-1.4.10') write_file(module_file, module_txt) invalidate_module_caches_for(module_parent) bar = EasyConfig(barec) # Re-parse the parent easyconfig sqlite = bar.dependencies()[3] self.assertEqual(det_full_ec_version(sqlite), '3.8.10.2-goolf-1.4.10')
def template_easyconfig_test(self, spec): """Tests for an individual easyconfig: parsing, instantiating easyblock, check patches, ...""" # set to False, so it's False in case of this test failing global single_tests_ok prev_single_tests_ok = single_tests_ok single_tests_ok = False # parse easyconfig ecs = process_easyconfig(spec) if len(ecs) == 1: ec = ecs[0]['ec'] # cache the parsed easyconfig, to avoid that it is parsed again self.parsed_easyconfigs.append(ecs[0]) else: self.assertTrue( False, "easyconfig %s does not contain blocks, yields only one parsed easyconfig" % spec) # check easyconfig file name expected_fn = '%s-%s.eb' % (ec['name'], det_full_ec_version(ec)) msg = "Filename '%s' of parsed easyconfig matches expected filename '%s'" % ( spec, expected_fn) self.assertEqual(os.path.basename(spec), expected_fn, msg) name, easyblock = fetch_parameters_from_easyconfig(ec.rawtxt, ['name', 'easyblock']) # make sure easyconfig file is in expected location expected_subdir = os.path.join('easybuild', 'easyconfigs', letter_dir_for(name), name) subdir = os.path.join(*spec.split(os.path.sep)[-5:-1]) fail_msg = "Easyconfig file %s not in expected subdirectory %s" % ( spec, expected_subdir) self.assertEqual(expected_subdir, subdir, fail_msg) # sanity check for software name, moduleclass self.assertEqual(ec['name'], name) self.assertTrue(ec['moduleclass'] in build_option('valid_module_classes')) # instantiate easyblock with easyconfig file app_class = get_easyblock_class(easyblock, name=name) # check that automagic fallback to ConfigureMake isn't done (deprecated behaviour) fn = os.path.basename(spec) error_msg = "%s relies on automagic fallback to ConfigureMake, should use easyblock = 'ConfigureMake' instead" % fn self.assertTrue(easyblock or app_class is not ConfigureMake, error_msg) app = app_class(ec) # more sanity checks self.assertTrue(name, app.name) self.assertTrue(ec['version'], app.version) # make sure that $root is not used, since it is not compatible with module files in Lua syntax res = re.findall('.*\$root.*', ec.rawtxt, re.M) error_msg = "Found use of '$root', not compatible with modules in Lua syntax, use '%%(installdir)s' instead: %s" self.assertFalse(res, error_msg % res) # make sure old GitHub urls for EasyBuild that include 'hpcugent' are no longer used old_urls = [ 'github.com/hpcugent/easybuild', 'hpcugent.github.com/easybuild', 'hpcugent.github.io/easybuild', ] for old_url in old_urls: self.assertFalse(old_url in ec.rawtxt, "Old URL '%s' not found in %s" % (old_url, spec)) # make sure binutils is included as a build dep if toolchain is GCCcore if ec['toolchain']['name'] == 'GCCcore': # with 'Tarball' easyblock: only unpacking, no building; Eigen is also just a tarball requires_binutils = ec['easyblock'] not in [ 'Tarball' ] and ec['name'] not in ['Eigen'] # let's also exclude the very special case where the system GCC is used as GCCcore, and only apply this # exception to the dependencies of binutils (since we should eventually build a new binutils with GCCcore) if ec['toolchain']['version'] == 'system': binutils_complete_dependencies = [ 'M4', 'Bison', 'flex', 'help2man', 'zlib', 'binutils' ] requires_binutils &= bool( ec['name'] not in binutils_complete_dependencies) # if no sources/extensions/components are specified, it's just a bundle (nothing is being compiled) requires_binutils &= bool(ec['sources'] or ec['exts_list'] or ec.get('components')) if requires_binutils: dep_names = [d['name'] for d in ec.builddependencies()] self.assertTrue( 'binutils' in dep_names, "binutils is a build dep in %s: %s" % (spec, dep_names)) # make sure all patch files are available specdir = os.path.dirname(spec) specfn = os.path.basename(spec) for patch in ec['patches']: if isinstance(patch, (tuple, list)): patch = patch[0] # only check actual patch files, not other files being copied via the patch functionality if patch.endswith('.patch'): patch_full = os.path.join(specdir, patch) msg = "Patch file %s is available for %s" % (patch_full, specfn) self.assertTrue(os.path.isfile(patch_full), msg) for ext in ec['exts_list']: if isinstance(ext, (tuple, list)) and len(ext) == 3: self.assertTrue(isinstance(ext[2], dict), "3rd element of extension spec is a dictionary") for ext_patch in ext[2].get('patches', []): if isinstance(ext_patch, (tuple, list)): ext_patch = ext_patch[0] # only check actual patch files, not other files being copied via the patch functionality if ext_patch.endswith('.patch'): ext_patch_full = os.path.join(specdir, ext_patch) msg = "Patch file %s is available for %s" % ( ext_patch_full, specfn) self.assertTrue(os.path.isfile(ext_patch_full), msg) # check whether all extra_options defined for used easyblock are defined extra_opts = app.extra_options() for key in extra_opts: self.assertTrue(key in app.cfg) app.close_log() os.remove(app.logfile) # dump the easyconfig file handle, test_ecfile = tempfile.mkstemp() os.close(handle) ec.dump(test_ecfile) dumped_ec = EasyConfigParser(test_ecfile).get_config_dict() os.remove(test_ecfile) # inject dummy values for templates that are only known at a later stage dummy_template_values = { 'builddir': '/dummy/builddir', 'installdir': '/dummy/installdir', } ec.template_values.update(dummy_template_values) ec_dict = ec.parser.get_config_dict() orig_toolchain = ec_dict['toolchain'] for key in ec_dict: # skip parameters for which value is equal to default value orig_val = ec_dict[key] if key in DEFAULT_CONFIG and orig_val == DEFAULT_CONFIG[key][0]: continue if key in extra_opts and orig_val == extra_opts[key][0]: continue if key not in DEFAULT_CONFIG and key not in extra_opts: continue orig_val = resolve_template(ec_dict[key], ec.template_values) dumped_val = resolve_template(dumped_ec[key], ec.template_values) # take into account that dumped value for *dependencies may include hard-coded subtoolchains # if no easyconfig was found for the dependency with the 'parent' toolchain, # if may get resolved using a subtoolchain, which is then hardcoded in the dumped easyconfig if key in DEPENDENCY_PARAMETERS: # number of dependencies should remain the same self.assertEqual(len(orig_val), len(dumped_val)) for orig_dep, dumped_dep in zip(orig_val, dumped_val): # name/version should always match self.assertEqual(orig_dep[:2], dumped_dep[:2]) # 3rd value is versionsuffix; if len(dumped_dep) >= 3: # if no versionsuffix was specified in original dep spec, then dumped value should be empty string if len(orig_dep) >= 3: self.assertEqual(dumped_dep[2], orig_dep[2]) else: self.assertEqual(dumped_dep[2], '') # 4th value is toolchain spec if len(dumped_dep) >= 4: if len(orig_dep) >= 4: self.assertEqual(dumped_dep[3], orig_dep[3]) else: # if a subtoolchain is specifed (only) in the dumped easyconfig, # it should *not* be the same as the parent toolchain self.assertNotEqual(dumped_dep[3], (orig_toolchain['name'], orig_toolchain['version'])) else: self.assertEqual(orig_val, dumped_val) # test passed, so set back to True single_tests_ok = True and prev_single_tests_ok
# add versionsuffix if Python is specified as a dependency if any(dep[0] == 'Python' for dep in cfg.get('dependencies', [])): if not_found_yet('versionsuffix'): cfg['versionsuffix'] = '-Python-%(pyver)s' # add empty sanity_check_paths if 'sanity_check_paths' not in cfg: cfg['sanity_check_paths'] = {'files': [], 'dirs': []} # enable use_pip & sanity_pip_check if cfg.get('easyblock') in ['PythonBundle', 'PythonPackage']: cfg.update({ 'use_pip': True, 'sanity_pip_check': True, }) # enable download_dep_fail if cfg.get('easyblock') == 'PythonPackage': cfg['download_dep_fail'] = True pprint.pprint(cfg) ec_raw = '\n'.join("%s = %s" % (key, quote_str(cfg[key])) for key in cfg) ec = EasyConfig(None, rawtxt=ec_raw) full_ec_ver = det_full_ec_version(ec) fn = os.path.join('%s-%s.eb' % (cfg['name'], full_ec_ver)) ec.dump(fn) info("Easyconfig file created: %s" % fn)
def find_minimally_resolved_modules(easyconfigs, avail_modules, existing_modules, retain_all_deps=False, use_existing_modules=True): """ Figure out which modules are resolved already, using minimal subtoolchains for dependencies. @param_easyconfigs: list of parsed easyconfigs @param avail_modules: list of available modules (used to check for resolved modules) @param existing_modules: list of existing modules (including non-available ones); used to determine minimal toolchain to use, only if use_existing_modules is True @param retain_all_deps: retain all dependencies, regardless of whether modules are available for them or not @param use_existing_modules: if a module is available with a particular (sub)toolchain, use it & stop searching """ _log.experimental("Using minimal toolchains when resolving dependencies") ordered_ecs = [] new_easyconfigs = [] modtool = modules_tool() # copy, we don't want to modify the origin list of available modules avail_modules = avail_modules[:] # Create a (temporary sub-)directory to store minimal easyconfigs minimal_ecs_dir = os.path.join(tempfile.gettempdir(), 'minimal-easyconfigs') mkdir(minimal_ecs_dir, parents=True) for easyconfig in easyconfigs: toolchain_hierarchy = get_toolchain_hierarchy( easyconfig['ec']['toolchain']) new_ec = easyconfig.copy() deps = [] for dep in easyconfig['dependencies']: dep_resolved = False orig_dep = dep new_dep = None full_mod_name = dep.get('full_mod_name', ActiveMNS().det_full_module_name(dep)) # treat external modules as resolved when retain_all_deps is enabled (e.g., under --dry-run), # since no corresponding easyconfig can be found for them if retain_all_deps and dep.get('external_module', False): _log.debug( "Treating dependency marked as external dependency as resolved: %s", dep) dep_resolved = True elif dep['toolchain'] != easyconfig['ec']['toolchain']: # in the case where the toolchain of a dependency is different to the parent toolchain we do nothing # we only find minimal dependencies if the dependency uses the same toolchain as the parent # that is, we respect explicitly defined toolchains for dependencies. dep_resolved = module_is_available(full_mod_name, modtool, avail_modules, dep['hidden']) else: # parent and dependency use same toolchain if use_existing_modules: # check whether a module using one of the (sub)toolchains is available for this dependency # if so, pick the minimal subtoolchain for which a module is available for toolchain in toolchain_hierarchy: cand_dep = copy.deepcopy(dep) cand_dep['toolchain'] = toolchain full_mod_name = ActiveMNS().det_full_module_name( cand_dep) cand_dep['full_mod_name'] = full_mod_name dep_resolved = module_is_available( full_mod_name, modtool, existing_modules, cand_dep['hidden']) if dep_resolved: new_dep = cand_dep _log.debug( "Module found for dep %s using toolchain %s: %s", dep, toolchain, full_mod_name) break if not dep_resolved: # if no module was found for this dependency with any of the (sub)modules, # or if EasyBuild was configured not to take existing modules into account first, # we find the minimal easyconfig and update the dependency res = robot_find_minimal_easyconfig_for_dependency(dep) if res is not None: new_dep, _ = res # now check for the existence of the module of the dep full_mod_name = ActiveMNS().det_full_module_name( new_dep) dep_resolved = module_is_available( full_mod_name, modtool, avail_modules, new_dep['hidden']) _log.debug( "Is replaced dep %s (module %s) resolved?: %s", new_dep, full_mod_name, dep_resolved) else: _log.debug( "Irresolvable minimal dependency found in robot search: %s" % orig_dep) # update the dependency in the parsed easyconfig if it was replaced if new_dep is not None: new_ec = deep_refresh_dependencies(new_ec, new_dep) _log.debug( "Updated easyconfig after replacing dep %s with %s: %s", orig_dep, new_dep, new_ec) dep = new_dep if not dep_resolved or (retain_all_deps and dep['full_mod_name'] not in avail_modules): # no module available (yet) => retain dependency as one to be resolved deps.append(dep) # update list of dependencies with only those unresolved new_ec['dependencies'] = deps if new_ec['dependencies']: # not all dependencies are resolved yet, so retain the easyconfig new_easyconfigs.append(new_ec) else: # if all dependencies have been resolved, add module for this easyconfig in the list of available modules avail_modules.append(new_ec['full_mod_name']) # dump easyconfig using minimal toolchain for dependencies # FIXME: only dump when something actually changed? newspec = '%s-%s.eb' % (new_ec['ec']['name'], det_full_ec_version(new_ec['ec'])) newspec = os.path.join(minimal_ecs_dir, newspec) _log.debug( "Attempting dumping minimal easyconfig to %s and adding it to final list", newspec) try: # only copy if the files are not the same file already (yes, it happens) if os.path.exists(newspec): _log.debug( "Not creating minimal easyconfig file %s since it already exists", newspec) else: _log.info( "Updating %s: %s is new minimal toolchain version", new_ec['spec'], newspec) new_ec['spec'] = newspec new_ec['ec'].dump(new_ec['spec']) ordered_ecs.append(new_ec) _log.debug("Adding easyconfig %s to final list" % new_ec['spec']) except (IOError, OSError) as err: raise EasyBuildError("Failed to create easyconfig %s: %s", newspec, err) return ordered_ecs, new_easyconfigs, avail_modules
def template_easyconfig_test(self, spec): """Tests for an individual easyconfig: parsing, instantiating easyblock, check patches, ...""" # set to False, so it's False in case of this test failing global single_tests_ok prev_single_tests_ok = single_tests_ok single_tests_ok = False # parse easyconfig ecs = process_easyconfig(spec) if len(ecs) == 1: ec = ecs[0]['ec'] else: self.assertTrue(False, "easyconfig %s does not contain blocks, yields only one parsed easyconfig" % spec) # check easyconfig file name expected_fn = '%s-%s.eb' % (ec['name'], det_full_ec_version(ec)) msg = "Filename '%s' of parsed easconfig matches expected filename '%s'" % (spec, expected_fn) self.assertEqual(os.path.basename(spec), expected_fn, msg) name, easyblock = fetch_parameters_from_easyconfig(ec.rawtxt, ['name', 'easyblock']) # sanity check for software name self.assertTrue(ec['name'], name) # instantiate easyblock with easyconfig file app_class = get_easyblock_class(easyblock, name=name) # check that automagic fallback to ConfigureMake isn't done (deprecated behaviour) fn = os.path.basename(spec) error_msg = "%s relies on automagic fallback to ConfigureMake, should use easyblock = 'ConfigureMake' instead" % fn self.assertTrue(easyblock or not app_class is ConfigureMake, error_msg) app = app_class(ec) # more sanity checks self.assertTrue(name, app.name) self.assertTrue(ec['version'], app.version) # make sure all patch files are available specdir = os.path.dirname(spec) specfn = os.path.basename(spec) for patch in ec['patches']: if isinstance(patch, (tuple, list)): patch = patch[0] # only check actual patch files, not other files being copied via the patch functionality if patch.endswith('.patch'): patch_full = os.path.join(specdir, patch) msg = "Patch file %s is available for %s" % (patch_full, specfn) self.assertTrue(os.path.isfile(patch_full), msg) ext_patches = [] for ext in ec['exts_list']: if isinstance(ext, (tuple, list)) and len(ext) == 3: self.assertTrue(isinstance(ext[2], dict), "3rd element of extension spec is a dictionary") for ext_patch in ext[2].get('patches', []): if isinstance(ext_patch, (tuple, list)): ext_patch = ext_patch[0] # only check actual patch files, not other files being copied via the patch functionality if ext_patch.endswith('.patch'): ext_patch_full = os.path.join(specdir, ext_patch) msg = "Patch file %s is available for %s" % (ext_patch_full, specfn) self.assertTrue(os.path.isfile(ext_patch_full), msg) # check whether all extra_options defined for used easyblock are defined for key in app.extra_options(): self.assertTrue(key in app.cfg) app.close_log() os.remove(app.logfile) # cache the parsed easyconfig, to avoid that it is parsed again self.parsed_easyconfigs.append(ecs[0]) # test passed, so set back to True single_tests_ok = True and prev_single_tests_ok
def resolve_dependencies(easyconfigs, modtool, retain_all_deps=False, raise_error_missing_ecs=True): """ Work through the list of easyconfigs to determine an optimal order :param easyconfigs: list of easyconfigs :param modtool: ModulesTool instance to use :param retain_all_deps: boolean indicating whether all dependencies must be retained, regardless of availability; retain all deps when True, check matching build option when False :param raise_error_missing_ecs: raise an error when one or more easyconfig files could not be found """ robot = build_option('robot_path') # retain all dependencies if specified by either the resp. build option or the dedicated named argument retain_all_deps = build_option('retain_all_deps') or retain_all_deps avail_modules = modtool.available() if retain_all_deps: # assume that no modules are available when forced, to retain all dependencies avail_modules = [] _log.info("Forcing all dependencies to be retained.") else: if len(avail_modules) == 0: _log.warning( "No installed modules. Your MODULEPATH is probably incomplete: %s" % os.getenv('MODULEPATH')) ordered_ecs = [] # all available modules can be used for resolving dependencies except those that will be installed being_installed = [p['full_mod_name'] for p in easyconfigs] avail_modules = [m for m in avail_modules if m not in being_installed] _log.debug('easyconfigs before resolving deps: %s', easyconfigs) totally_missing, missing_easyconfigs = [], [] # resolve all dependencies, put a safeguard in place to avoid an infinite loop (shouldn't occur though) loopcnt = 0 maxloopcnt = 10000 while easyconfigs: # make sure this stops, we really don't want to get stuck in an infinite loop loopcnt += 1 if loopcnt > maxloopcnt: raise EasyBuildError( "Maximum loop cnt %s reached, so quitting (easyconfigs: %s, missing_easyconfigs: %s)", maxloopcnt, easyconfigs, missing_easyconfigs) # first try resolving dependencies without using external dependencies last_processed_count = -1 while len(avail_modules) > last_processed_count: last_processed_count = len(avail_modules) res = find_resolved_modules(easyconfigs, avail_modules, modtool, retain_all_deps=retain_all_deps) resolved_ecs, easyconfigs, avail_modules = res ordered_ec_mod_names = [x['full_mod_name'] for x in ordered_ecs] for ec in resolved_ecs: # only add easyconfig if it's not included yet (based on module name) if not ec['full_mod_name'] in ordered_ec_mod_names: ordered_ecs.append(ec) # dependencies marked as external modules should be resolved via available modules at this point missing_external_modules = [ d['full_mod_name'] for ec in easyconfigs for d in ec['dependencies'] if d.get('external_module', False) ] if missing_external_modules: raise EasyBuildError( "Missing modules for dependencies marked as external modules: %s", ', '.join(missing_external_modules)) # robot: look for existing dependencies, add them if robot and easyconfigs: # rely on EasyBuild module naming scheme when resolving dependencies, since we know that will # generate sensible module names that include the necessary information for the resolution to work # (name, version, toolchain, versionsuffix) being_installed = [ EasyBuildMNS().det_full_module_name(p['ec']) for p in easyconfigs ] additional = [] for entry in easyconfigs: # do not choose an entry that is being installed in the current run # if they depend, you probably want to rebuild them using the new dependency deps = entry['dependencies'] candidates = [ d for d in deps if not EasyBuildMNS().det_full_module_name( d) in being_installed ] if candidates: cand_dep = candidates[0] # find easyconfig, might not find any _log.debug("Looking for easyconfig for %s" % str(cand_dep)) # note: robot_find_easyconfig may return None path = robot_find_easyconfig(cand_dep['name'], det_full_ec_version(cand_dep)) if path is None: full_mod_name = ActiveMNS().det_full_module_name( cand_dep) # no easyconfig found + no module available => missing dependency if not modtool.exist([full_mod_name])[0]: if cand_dep not in totally_missing: totally_missing.append(cand_dep) # no easyconfig found for dependency, but module is available # => add to list of missing easyconfigs elif cand_dep not in missing_easyconfigs: _log.debug( "Irresolvable dependency found (no easyconfig file): %s", cand_dep) missing_easyconfigs.append(cand_dep) # remove irresolvable dependency from list of dependencies so we can continue entry['dependencies'].remove(cand_dep) # add dummy entry for this dependency, so --dry-run for example can still report the dep additional.append({ 'dependencies': [], 'ec': None, 'full_mod_name': full_mod_name, 'spec': None, }) else: _log.info("Robot: resolving dependency %s with %s" % (cand_dep, path)) # build specs should not be passed down to resolved dependencies, # to avoid that e.g. --try-toolchain trickles down into the used toolchain itself hidden = cand_dep.get('hidden', False) processed_ecs = process_easyconfig( path, validate=not retain_all_deps, hidden=hidden) # ensure that selected easyconfig provides required dependency verify_easyconfig_filename(path, cand_dep, parsed_ec=processed_ecs) for ec in processed_ecs: if ec not in easyconfigs + additional: additional.append(ec) _log.debug("Added %s as dependency of %s" % (ec, entry)) else: mod_name = EasyBuildMNS().det_full_module_name(entry['ec']) _log.debug( "No more candidate dependencies to resolve for %s" % mod_name) # add additional (new) easyconfigs to list of stuff to process easyconfigs.extend(additional) _log.debug("Unprocessed dependencies: %s", easyconfigs) elif not robot: # no use in continuing if robot is not enabled, dependencies won't be resolved anyway missing_deps = [ dep for x in easyconfigs for dep in x['dependencies'] ] if missing_deps: raise_error_missing_deps( missing_deps, extra_msg="enable dependency resolution via --robot?") if totally_missing: raise_error_missing_deps( totally_missing, extra_msg="no easyconfig file or existing module found") if missing_easyconfigs: if raise_error_missing_ecs: raise_error_missing_deps( missing_easyconfigs, extra_msg="no easyconfig file found in robot search path") else: _log.warning("No easyconfig files found for: %s", missing_easyconfigs) _log.info("Dependency resolution complete, building as follows: %s", ordered_ecs) return ordered_ecs
def template_easyconfig_test(self, spec): """Tests for an individual easyconfig: parsing, instantiating easyblock, check patches, ...""" # set to False, so it's False in case of this test failing global single_tests_ok prev_single_tests_ok = single_tests_ok single_tests_ok = False # parse easyconfig ecs = process_easyconfig(spec) if len(ecs) == 1: ec = ecs[0]['ec'] else: self.assertTrue(False, "easyconfig %s does not contain blocks, yields only one parsed easyconfig" % spec) # check easyconfig file name expected_fn = '%s-%s.eb' % (ec['name'], det_full_ec_version(ec)) msg = "Filename '%s' of parsed easyconfig matches expected filename '%s'" % (spec, expected_fn) self.assertEqual(os.path.basename(spec), expected_fn, msg) name, easyblock = fetch_parameters_from_easyconfig(ec.rawtxt, ['name', 'easyblock']) # make sure easyconfig file is in expected location expected_subdir = os.path.join('easybuild', 'easyconfigs', letter_dir_for(name), name) subdir = os.path.join(*spec.split(os.path.sep)[-5:-1]) fail_msg = "Easyconfig file %s not in expected subdirectory %s" % (spec, expected_subdir) self.assertEqual(expected_subdir, subdir, fail_msg) # sanity check for software name, moduleclass self.assertEqual(ec['name'], name) self.assertTrue(ec['moduleclass'] in build_option('valid_module_classes')) # instantiate easyblock with easyconfig file app_class = get_easyblock_class(easyblock, name=name) # check that automagic fallback to ConfigureMake isn't done (deprecated behaviour) fn = os.path.basename(spec) error_msg = "%s relies on automagic fallback to ConfigureMake, should use easyblock = 'ConfigureMake' instead" % fn self.assertTrue(easyblock or not app_class is ConfigureMake, error_msg) app = app_class(ec) # more sanity checks self.assertTrue(name, app.name) self.assertTrue(ec['version'], app.version) # make sure that $root is not used, since it is not compatible with module files in Lua syntax res = re.findall('.*\$root.*', ec.rawtxt, re.M) error_msg = "Found use of '$root', not compatible with modules in Lua syntax, use '%%(installdir)s' instead: %s" self.assertFalse(res, error_msg % res) # make sure old GitHub urls for EasyBuild that include 'hpcugent' are no longer used old_urls = [ 'github.com/hpcugent/easybuild', 'hpcugent.github.com/easybuild', 'hpcugent.github.io/easybuild', ] for old_url in old_urls: self.assertFalse(old_url in ec.rawtxt, "Old URL '%s' not found in %s" % (old_url, spec)) # make sure binutils is included as a build dep if toolchain is GCCcore if ec['toolchain']['name'] == 'GCCcore': # with 'Tarball' easyblock: only unpacking, no building; Eigen is also just a tarball requires_binutils = ec['easyblock'] not in ['Tarball'] and ec['name'] not in ['Eigen'] # let's also exclude the very special case where the system GCC is used as GCCcore, and only apply this # exception to the dependencies of binutils (since we should eventually build a new binutils with GCCcore) if ec['toolchain']['version'] == 'system': binutils_complete_dependencies = ['M4', 'Bison', 'flex', 'help2man', 'zlib', 'binutils'] requires_binutils &= bool(ec['name'] not in binutils_complete_dependencies) # if no sources/extensions/components are specified, it's just a bundle (nothing is being compiled) requires_binutils &= bool(ec['sources'] or ec['exts_list'] or ec.get('components')) if requires_binutils: dep_names = [d['name'] for d in ec['builddependencies']] self.assertTrue('binutils' in dep_names, "binutils is a build dep in %s: %s" % (spec, dep_names)) # make sure all patch files are available specdir = os.path.dirname(spec) specfn = os.path.basename(spec) for patch in ec['patches']: if isinstance(patch, (tuple, list)): patch = patch[0] # only check actual patch files, not other files being copied via the patch functionality if patch.endswith('.patch'): patch_full = os.path.join(specdir, patch) msg = "Patch file %s is available for %s" % (patch_full, specfn) self.assertTrue(os.path.isfile(patch_full), msg) ext_patches = [] for ext in ec['exts_list']: if isinstance(ext, (tuple, list)) and len(ext) == 3: self.assertTrue(isinstance(ext[2], dict), "3rd element of extension spec is a dictionary") for ext_patch in ext[2].get('patches', []): if isinstance(ext_patch, (tuple, list)): ext_patch = ext_patch[0] # only check actual patch files, not other files being copied via the patch functionality if ext_patch.endswith('.patch'): ext_patch_full = os.path.join(specdir, ext_patch) msg = "Patch file %s is available for %s" % (ext_patch_full, specfn) self.assertTrue(os.path.isfile(ext_patch_full), msg) # check whether all extra_options defined for used easyblock are defined extra_opts = app.extra_options() for key in extra_opts: self.assertTrue(key in app.cfg) app.close_log() os.remove(app.logfile) # dump the easyconfig file handle, test_ecfile = tempfile.mkstemp() os.close(handle) ec.dump(test_ecfile) dumped_ec = EasyConfigParser(test_ecfile).get_config_dict() os.remove(test_ecfile) # inject dummy values for templates that are only known at a later stage dummy_template_values = { 'builddir': '/dummy/builddir', 'installdir': '/dummy/installdir', } ec.template_values.update(dummy_template_values) ec_dict = ec.parser.get_config_dict() orig_toolchain = ec_dict['toolchain'] for key in ec_dict: # skip parameters for which value is equal to default value orig_val = ec_dict[key] if key in DEFAULT_CONFIG and orig_val == DEFAULT_CONFIG[key][0]: continue if key in extra_opts and orig_val == extra_opts[key][0]: continue if key not in DEFAULT_CONFIG and key not in extra_opts: continue orig_val = resolve_template(ec_dict[key], ec.template_values) dumped_val = resolve_template(dumped_ec[key], ec.template_values) # take into account that dumped value for *dependencies may include hard-coded subtoolchains # if no easyconfig was found for the dependency with the 'parent' toolchain, # if may get resolved using a subtoolchain, which is then hardcoded in the dumped easyconfig if key in DEPENDENCY_PARAMETERS: # number of dependencies should remain the same self.assertEqual(len(orig_val), len(dumped_val)) for orig_dep, dumped_dep in zip(orig_val, dumped_val): # name/version should always match self.assertEqual(orig_dep[:2], dumped_dep[:2]) # 3rd value is versionsuffix; if len(dumped_dep) >= 3: # if no versionsuffix was specified in original dep spec, then dumped value should be empty string if len(orig_dep) >= 3: self.assertEqual(dumped_dep[2], orig_dep[2]) else: self.assertEqual(dumped_dep[2], '') # 4th value is toolchain spec if len(dumped_dep) >= 4: if len(orig_dep) >= 4: self.assertEqual(dumped_dep[3], orig_dep[3]) else: # if a subtoolchain is specifed (only) in the dumped easyconfig, # it should *not* be the same as the parent toolchain self.assertNotEqual(dumped_dep[3], (orig_toolchain['name'], orig_toolchain['version'])) else: self.assertEqual(orig_val, dumped_val) # cache the parsed easyconfig, to avoid that it is parsed again self.parsed_easyconfigs.append(ecs[0]) # test passed, so set back to True single_tests_ok = True and prev_single_tests_ok
def mk_key(spec): """Create key for dictionary with all dependencies.""" if 'ec' in spec: spec = spec['ec'] return (spec['name'], det_full_ec_version(spec))
def select_or_generate_ec(fp, paths, specs): """ Select or generate an easyconfig file with the given requirements, from existing easyconfig files. If easyconfig files are available for the specified software package, then this function will first try to determine which toolchain to use. * if a toolchain is given, it will use it (possible using a template easyconfig file as base); * if not, and only a single toolchain is available, is will assume it can use that toolchain * else, it fails -- EasyBuild doesn't select between multiple available toolchains Next, it will trim down the selected easyconfig files to a single one, based on the following requirements (in order of preference): * toolchain version * software version * other parameters (e.g. versionprefix, versionsuffix, etc.) If a complete match is found, it will return that easyconfig. Else, it will generate a new easyconfig file based on the selected 'best matching' easyconfig file. """ specs = copy.deepcopy(specs) # ensure that at least name is specified if not specs.get('name'): _log.error("Supplied 'specs' dictionary doesn't even contain a name of a software package?") name = specs['name'] handled_params = ['name'] # find ALL available easyconfig files for specified software cfg = { 'version': '*', 'toolchain': {'name': DUMMY_TOOLCHAIN_NAME, 'version': '*'}, 'versionprefix': '*', 'versionsuffix': '*', } installver = det_full_ec_version(cfg) ec_files = find_matching_easyconfigs(name, installver, paths) _log.debug("Unique ec_files: %s" % ec_files) # we need at least one config file to start from if len(ec_files) == 0: # look for a template file if no easyconfig for specified software name is available for path in paths: templ_file = os.path.join(path, "%s.eb" % EASYCONFIG_TEMPLATE) if os.path.isfile(templ_file): ec_files = [templ_file] break else: _log.debug("No template found at %s." % templ_file) if len(ec_files) == 0: _log.error("No easyconfig files found for software %s, and no templates available. I'm all out of ideas." % name) ecs_and_files = [(EasyConfig(f, validate=False), f) for f in ec_files] # TOOLCHAIN NAME # we can't rely on set, because we also need to be able to obtain a list of unique lists def unique(l): """Retain unique elements in a sorted list.""" l = sorted(l) if len(l) > 1: l2 = [l[0]] for x in l: if not x == l2[-1]: l2.append(x) return l2 else: return l # determine list of unique toolchain names tcnames = unique([x[0]['toolchain']['name'] for x in ecs_and_files]) _log.debug("Found %d unique toolchain names: %s" % (len(tcnames), tcnames)) # if a toolchain was selected, and we have no easyconfig files for it, try and use a template if specs.get('toolchain_name') and not specs['toolchain_name'] in tcnames: if EASYCONFIG_TEMPLATE in tcnames: _log.info("No easyconfig file for specified toolchain, but template is available.") else: _log.error("No easyconfig file for %s with toolchain %s, " \ "and no template available." % (name, specs['toolchain_name'])) tcname = specs.pop('toolchain_name', None) handled_params.append('toolchain_name') # trim down list according to selected toolchain if tcname in tcnames: # known toolchain, so only retain those selected_tcname = tcname else: if len(tcnames) == 1 and not tcnames[0] == EASYCONFIG_TEMPLATE: # only one (non-template) toolchain availble, so use that tcname = tcnames[0] selected_tcname = tcname elif len(tcnames) == 1 and tcnames[0] == EASYCONFIG_TEMPLATE: selected_tcname = tcnames[0] else: # fall-back: use template toolchain if a toolchain name was specified if tcname: selected_tcname = EASYCONFIG_TEMPLATE else: # if multiple toolchains are available, and none is specified, we quit # we can't just pick one, how would we prefer one over the other? _log.error("No toolchain name specified, and more than one available: %s." % tcnames) _log.debug("Filtering easyconfigs based on toolchain name '%s'..." % selected_tcname) ecs_and_files = [x for x in ecs_and_files if x[0]['toolchain']['name'] == selected_tcname] _log.debug("Filtered easyconfigs: %s" % [x[1] for x in ecs_and_files]) # TOOLCHAIN VERSION tcvers = unique([x[0]['toolchain']['version'] for x in ecs_and_files]) _log.debug("Found %d unique toolchain versions: %s" % (len(tcvers), tcvers)) tcver = specs.pop('toolchain_version', None) handled_params.append('toolchain_version') (tcver, selected_tcver) = pick_version(tcver, tcvers) _log.debug("Filtering easyconfigs based on toolchain version '%s'..." % selected_tcver) ecs_and_files = [x for x in ecs_and_files if x[0]['toolchain']['version'] == selected_tcver] _log.debug("Filtered easyconfigs: %s" % [x[1] for x in ecs_and_files]) # add full toolchain specification to specs if tcname and tcver: specs.update({'toolchain': {'name': tcname, 'version': tcver}}) handled_params.append('toolchain') else: if tcname: specs.update({'toolchain_name': tcname}) if tcver: specs.update({'toolchain_version': tcver}) # SOFTWARE VERSION vers = unique([x[0]['version'] for x in ecs_and_files]) _log.debug("Found %d unique software versions: %s" % (len(vers), vers)) ver = specs.pop('version', None) handled_params.append('version') (ver, selected_ver) = pick_version(ver, vers) if ver: specs.update({'version': ver}) _log.debug("Filtering easyconfigs based on software version '%s'..." % selected_ver) ecs_and_files = [x for x in ecs_and_files if x[0]['version'] == selected_ver] _log.debug("Filtered easyconfigs: %s" % [x[1] for x in ecs_and_files]) # go through parameters specified via --amend # always include versionprefix/suffix, because we might need it to generate a file name verpref = None versuff = None other_params = {'versionprefix': None, 'versionsuffix': None} for (param, val) in specs.items(): if not param in handled_params: other_params.update({param: val}) _log.debug("Filtering based on other parameters (specified via --amend): %s" % other_params) for (param, val) in other_params.items(): if param in ecs_and_files[0][0]._config: vals = unique([x[0][param] for x in ecs_and_files]) else: vals = [] filter_ecs = False # try and select a value from the available ones, or fail if we can't if val in vals: # if the specified value is available, use it selected_val = val _log.debug("Specified %s is available, so using it: %s" % (param, selected_val)) filter_ecs = True elif val: # if a value is specified, use that, even if it's not available yet selected_val = val # promote value to list if deemed appropriate if vals and type(vals[0]) == list and not type(val) == list: _log.debug("Promoting type of %s value to list, since original value was." % param) specs[param] = [val] _log.debug("%s is specified, so using it (even though it's not available yet): %s" % (param, selected_val)) elif len(vals) == 1: # if only one value is available, use that selected_val = vals[0] _log.debug("Only one %s available ('%s'), so picking that" % (param, selected_val)) filter_ecs = True else: # otherwise, we fail, because we don't know how to pick between different fixes _log.error("No %s specified, and can't pick from available %ses %s" % (param, param, vals)) if filter_ecs: _log.debug("Filtering easyconfigs based on %s '%s'..." % (param, selected_val)) ecs_and_files = [x for x in ecs_and_files if x[0][param] == selected_val] _log.debug("Filtered easyconfigs: %s" % [x[1] for x in ecs_and_files]) # keep track of versionprefix/suffix if param == "versionprefix": verpref = selected_val elif param == "versionsuffix": versuff = selected_val cnt = len(ecs_and_files) if not cnt == 1: fs = [x[1] for x in ecs_and_files] _log.error("Failed to select a single easyconfig from available ones, %s left: %s" % (cnt, fs)) else: (selected_ec, selected_ec_file) = ecs_and_files[0] # check whether selected easyconfig matches requirements match = True for (key, val) in specs.items(): if key in selected_ec._config: # values must be equal to have a full match if not selected_ec[key] == val: match = False else: # if we encounter a key that is not set in the selected easyconfig, we don't have a full match match = False # if it matches, no need to tweak if match: _log.info("Perfect match found: %s" % selected_ec_file) return (False, selected_ec_file) # GENERATE # if no file path was specified, generate a file name if fp is None: cfg = { 'version': ver, 'toolchain': {'name': tcname, 'version': tcver}, 'versionprefix': verpref, 'versionsuffix': versuff, } installver = det_full_ec_version(cfg) fp = "%s-%s.eb" % (name, installver) # generate tweaked easyconfig file tweak_one(selected_ec_file, fp, specs) _log.info("Generated easyconfig file %s, and using it to build the requested software." % fp) return (True, fp)
def resolve_dependencies(unprocessed, build_specs=None, retain_all_deps=False): """ Work through the list of easyconfigs to determine an optimal order @param unprocessed: list of easyconfigs @param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...) """ robot = build_option('robot_path') retain_all_deps = build_option('retain_all_deps') or retain_all_deps if retain_all_deps: # assume that no modules are available when forced, to retain all dependencies avail_modules = [] _log.info("Forcing all dependencies to be retained.") else: # Get a list of all available modules (format: [(name, installversion), ...]) avail_modules = modules_tool().available() if len(avail_modules) == 0: _log.warning("No installed modules. Your MODULEPATH is probably incomplete: %s" % os.getenv('MODULEPATH')) ordered_ecs = [] # all available modules can be used for resolving dependencies except those that will be installed being_installed = [p['module'] for p in unprocessed] avail_modules = [m for m in avail_modules if not m in being_installed] _log.debug('unprocessed before resolving deps: %s' % unprocessed) # resolve all dependencies, put a safeguard in place to avoid an infinite loop (shouldn't occur though) irresolvable = [] loopcnt = 0 maxloopcnt = 10000 while unprocessed: # make sure this stops, we really don't want to get stuck in an infinite loop loopcnt += 1 if loopcnt > maxloopcnt: tup = (maxloopcnt, unprocessed, irresolvable) msg = "Maximum loop cnt %s reached, so quitting (unprocessed: %s, irresolvable: %s)" % tup _log.error(msg) # first try resolving dependencies without using external dependencies last_processed_count = -1 while len(avail_modules) > last_processed_count: last_processed_count = len(avail_modules) more_ecs, unprocessed, avail_modules = find_resolved_modules(unprocessed, avail_modules) for ec in more_ecs: if not ec['module'] in [x['module'] for x in ordered_ecs]: ordered_ecs.append(ec) # robot: look for existing dependencies, add them if robot and unprocessed: being_installed = [det_full_module_name(p['ec'], eb_ns=True) for p in unprocessed] additional = [] for i, entry in enumerate(unprocessed): # do not choose an entry that is being installed in the current run # if they depend, you probably want to rebuild them using the new dependency deps = entry['dependencies'] candidates = [d for d in deps if not det_full_module_name(d, eb_ns=True) in being_installed] if len(candidates) > 0: cand_dep = candidates[0] # find easyconfig, might not find any _log.debug("Looking for easyconfig for %s" % str(cand_dep)) # note: robot_find_easyconfig may return None path = robot_find_easyconfig(robot, cand_dep['name'], det_full_ec_version(cand_dep)) if path is None: # no easyconfig found for dependency, add to list of irresolvable dependencies if cand_dep not in irresolvable: _log.debug("Irresolvable dependency found: %s" % cand_dep) irresolvable.append(cand_dep) # remove irresolvable dependency from list of dependencies so we can continue entry['dependencies'].remove(cand_dep) else: _log.info("Robot: resolving dependency %s with %s" % (cand_dep, path)) # build specs should not be passed down to resolved dependencies, # to avoid that e.g. --try-toolchain trickles down into the used toolchain itself processed_ecs = process_easyconfig(path, validate=not retain_all_deps) # ensure that selected easyconfig provides required dependency mods = [det_full_module_name(spec['ec']) for spec in processed_ecs] dep_mod_name = det_full_module_name(cand_dep) if not dep_mod_name in mods: tup = (path, dep_mod_name, mods) _log.error("easyconfig file %s does not contain module %s (mods: %s)" % tup) for ec in processed_ecs: if not ec in unprocessed + additional: additional.append(ec) _log.debug("Added %s as dependency of %s" % (ec, entry)) else: mod_name = det_full_module_name(entry['ec'], eb_ns=True) _log.debug("No more candidate dependencies to resolve for %s" % mod_name) # add additional (new) easyconfigs to list of stuff to process unprocessed.extend(additional) elif not robot: # no use in continuing if robot is not enabled, dependencies won't be resolved anyway irresolvable = [dep for x in unprocessed for dep in x['dependencies']] break if irresolvable: irresolvable_mod_deps = [(det_full_module_name(dep, eb_ns=True), dep) for dep in irresolvable] _log.error('Irresolvable dependencies encountered: %s' % irresolvable_mod_deps) _log.info("Dependency resolution complete, building as follows:\n%s" % ordered_ecs) return ordered_ecs
def template_easyconfig_test(self, spec): """Tests for an individual easyconfig: parsing, instantiating easyblock, check patches, ...""" # set to False, so it's False in case of this test failing global single_tests_ok prev_single_tests_ok = single_tests_ok single_tests_ok = False # parse easyconfig ecs = process_easyconfig(spec) if len(ecs) == 1: ec = ecs[0]['ec'] else: self.assertTrue(False, "easyconfig %s does not contain blocks, yields only one parsed easyconfig" % spec) # check easyconfig file name expected_fn = '%s-%s.eb' % (ec['name'], det_full_ec_version(ec)) msg = "Filename '%s' of parsed easyconfig matches expected filename '%s'" % (spec, expected_fn) self.assertEqual(os.path.basename(spec), expected_fn, msg) name, easyblock = fetch_parameters_from_easyconfig(ec.rawtxt, ['name', 'easyblock']) # make sure easyconfig file is in expected location expected_subdir = os.path.join('easybuild', 'easyconfigs', letter_dir_for(name), name) subdir = os.path.join(*spec.split(os.path.sep)[-5:-1]) fail_msg = "Easyconfig file %s not in expected subdirectory %s" % (spec, expected_subdir) self.assertEqual(expected_subdir, subdir, fail_msg) # sanity check for software name, moduleclass self.assertEqual(ec['name'], name) self.assertTrue(ec['moduleclass'] in build_option('valid_module_classes')) # instantiate easyblock with easyconfig file app_class = get_easyblock_class(easyblock, name=name) # check that automagic fallback to ConfigureMake isn't done (deprecated behaviour) fn = os.path.basename(spec) error_msg = "%s relies on automagic fallback to ConfigureMake, should use easyblock = 'ConfigureMake' instead" % fn self.assertTrue(easyblock or not app_class is ConfigureMake, error_msg) app = app_class(ec) # more sanity checks self.assertTrue(name, app.name) self.assertTrue(ec['version'], app.version) # make sure all patch files are available specdir = os.path.dirname(spec) specfn = os.path.basename(spec) for patch in ec['patches']: if isinstance(patch, (tuple, list)): patch = patch[0] # only check actual patch files, not other files being copied via the patch functionality if patch.endswith('.patch'): patch_full = os.path.join(specdir, patch) msg = "Patch file %s is available for %s" % (patch_full, specfn) self.assertTrue(os.path.isfile(patch_full), msg) ext_patches = [] for ext in ec['exts_list']: if isinstance(ext, (tuple, list)) and len(ext) == 3: self.assertTrue(isinstance(ext[2], dict), "3rd element of extension spec is a dictionary") for ext_patch in ext[2].get('patches', []): if isinstance(ext_patch, (tuple, list)): ext_patch = ext_patch[0] # only check actual patch files, not other files being copied via the patch functionality if ext_patch.endswith('.patch'): ext_patch_full = os.path.join(specdir, ext_patch) msg = "Patch file %s is available for %s" % (ext_patch_full, specfn) self.assertTrue(os.path.isfile(ext_patch_full), msg) # check whether all extra_options defined for used easyblock are defined extra_opts = app.extra_options() for key in extra_opts: self.assertTrue(key in app.cfg) app.close_log() os.remove(app.logfile) # dump the easyconfig file handle, test_ecfile = tempfile.mkstemp() os.close(handle) ec.dump(test_ecfile) dumped_ec = EasyConfigParser(test_ecfile).get_config_dict() os.remove(test_ecfile) # inject dummy values for templates that are only known at a later stage dummy_template_values = { 'builddir': '/dummy/builddir', 'installdir': '/dummy/installdir', } ec.template_values.update(dummy_template_values) ec_dict = ec.parser.get_config_dict() orig_toolchain = ec_dict['toolchain'] for key in ec_dict: # skip parameters for which value is equal to default value orig_val = ec_dict[key] if key in DEFAULT_CONFIG and orig_val == DEFAULT_CONFIG[key][0]: continue if key in extra_opts and orig_val == extra_opts[key][0]: continue if key not in DEFAULT_CONFIG and key not in extra_opts: continue orig_val = resolve_template(ec_dict[key], ec.template_values) dumped_val = resolve_template(dumped_ec[key], ec.template_values) # take into account that dumped value for *dependencies may include hard-coded subtoolchains # if no easyconfig was found for the dependency with the 'parent' toolchain, # if may get resolved using a subtoolchain, which is then hardcoded in the dumped easyconfig if key in DEPENDENCY_PARAMETERS: # number of dependencies should remain the same self.assertEqual(len(orig_val), len(dumped_val)) for orig_dep, dumped_dep in zip(orig_val, dumped_val): # name/version should always match self.assertEqual(orig_dep[:2], dumped_dep[:2]) # 3rd value is versionsuffix; if len(dumped_dep) >= 3: # if no versionsuffix was specified in original dep spec, then dumped value should be empty string if len(orig_dep) >= 3: self.assertEqual(dumped_dep[2], orig_dep[2]) else: self.assertEqual(dumped_dep[2], '') # 4th value is toolchain spec if len(dumped_dep) >= 4: if len(orig_dep) >= 4: self.assertEqual(dumped_dep[3], orig_dep[3]) else: # if a subtoolchain is specifed (only) in the dumped easyconfig, # it should *not* be the same as the parent toolchain self.assertNotEqual(dumped_dep[3], (orig_toolchain['name'], orig_toolchain['version'])) else: self.assertEqual(orig_val, dumped_val) # cache the parsed easyconfig, to avoid that it is parsed again self.parsed_easyconfigs.append(ecs[0]) # test passed, so set back to True single_tests_ok = True and prev_single_tests_ok
def find_minimally_resolved_modules(easyconfigs, avail_modules, existing_modules, retain_all_deps=False, use_existing_modules=True): """ Figure out which modules are resolved already, using minimal subtoolchains for dependencies. @param_easyconfigs: list of parsed easyconfigs @param avail_modules: list of available modules (used to check for resolved modules) @param existing_modules: list of existing modules (including non-available ones); used to determine minimal toolchain to use, only if use_existing_modules is True @param retain_all_deps: retain all dependencies, regardless of whether modules are available for them or not @param use_existing_modules: if a module is available with a particular (sub)toolchain, use it & stop searching """ _log.experimental("Using minimal toolchains when resolving dependencies") ordered_ecs = [] new_easyconfigs = [] modtool = modules_tool() # copy, we don't want to modify the origin list of available modules avail_modules = avail_modules[:] # Create a (temporary sub-)directory to store minimal easyconfigs minimal_ecs_dir = os.path.join(tempfile.gettempdir(), 'minimal-easyconfigs') mkdir(minimal_ecs_dir, parents=True) for easyconfig in easyconfigs: toolchain_hierarchy = get_toolchain_hierarchy(easyconfig['ec']['toolchain']) new_ec = easyconfig.copy() deps = [] for dep in easyconfig['dependencies']: dep_resolved = False orig_dep = dep new_dep = None full_mod_name = dep.get('full_mod_name', ActiveMNS().det_full_module_name(dep)) # treat external modules as resolved when retain_all_deps is enabled (e.g., under --dry-run), # since no corresponding easyconfig can be found for them if retain_all_deps and dep.get('external_module', False): _log.debug("Treating dependency marked as external dependency as resolved: %s", dep) dep_resolved = True elif dep['toolchain'] != easyconfig['ec']['toolchain']: # in the case where the toolchain of a dependency is different to the parent toolchain we do nothing # we only find minimal dependencies if the dependency uses the same toolchain as the parent # that is, we respect explicitly defined toolchains for dependencies. dep_resolved = module_is_available(full_mod_name, modtool, avail_modules, dep['hidden']) else: # parent and dependency use same toolchain if use_existing_modules: # check whether a module using one of the (sub)toolchains is available for this dependency # if so, pick the minimal subtoolchain for which a module is available for toolchain in toolchain_hierarchy: cand_dep = copy.deepcopy(dep) cand_dep['toolchain'] = toolchain full_mod_name = ActiveMNS().det_full_module_name(cand_dep) cand_dep['full_mod_name'] = full_mod_name dep_resolved = module_is_available(full_mod_name, modtool, existing_modules, cand_dep['hidden']) if dep_resolved: new_dep = cand_dep _log.debug("Module found for dep %s using toolchain %s: %s", dep, toolchain, full_mod_name) break if not dep_resolved: # if no module was found for this dependency with any of the (sub)modules, # or if EasyBuild was configured not to take existing modules into account first, # we find the minimal easyconfig and update the dependency res = robot_find_minimal_easyconfig_for_dependency(dep) if res is not None: new_dep, _ = res # now check for the existence of the module of the dep full_mod_name = ActiveMNS().det_full_module_name(new_dep) dep_resolved = module_is_available(full_mod_name, modtool, avail_modules, new_dep['hidden']) _log.debug("Is replaced dep %s (module %s) resolved?: %s", new_dep, full_mod_name, dep_resolved) else: _log.debug("Irresolvable minimal dependency found in robot search: %s" % orig_dep) # update the dependency in the parsed easyconfig if it was replaced if new_dep is not None: new_ec = deep_refresh_dependencies(new_ec, new_dep) _log.debug("Updated easyconfig after replacing dep %s with %s: %s", orig_dep, new_dep, new_ec) dep = new_dep if not dep_resolved or (retain_all_deps and dep['full_mod_name'] not in avail_modules): # no module available (yet) => retain dependency as one to be resolved deps.append(dep) # update list of dependencies with only those unresolved new_ec['dependencies'] = deps if new_ec['dependencies']: # not all dependencies are resolved yet, so retain the easyconfig new_easyconfigs.append(new_ec) else: # if all dependencies have been resolved, add module for this easyconfig in the list of available modules avail_modules.append(new_ec['full_mod_name']) # dump easyconfig using minimal toolchain for dependencies # FIXME: only dump when something actually changed? newspec = '%s-%s.eb' % (new_ec['ec']['name'], det_full_ec_version(new_ec['ec'])) newspec = os.path.join(minimal_ecs_dir, newspec) _log.debug("Attempting dumping minimal easyconfig to %s and adding it to final list", newspec) try: # only copy if the files are not the same file already (yes, it happens) if os.path.exists(newspec): _log.debug("Not creating minimal easyconfig file %s since it already exists", newspec) else: _log.info("Updating %s: %s is new minimal toolchain version", new_ec['spec'], newspec) new_ec['spec'] = newspec new_ec['ec'].dump(new_ec['spec']) ordered_ecs.append(new_ec) _log.debug("Adding easyconfig %s to final list" % new_ec['spec']) except (IOError, OSError) as err: raise EasyBuildError("Failed to create easyconfig %s: %s", newspec, err) return ordered_ecs, new_easyconfigs, avail_modules
def template_easyconfig_test(self, spec): """Tests for an individual easyconfig: parsing, instantiating easyblock, check patches, ...""" # set to False, so it's False in case of this test failing global single_tests_ok prev_single_tests_ok = single_tests_ok single_tests_ok = False # parse easyconfig ecs = process_easyconfig(spec) if len(ecs) == 1: ec = ecs[0]['ec'] else: self.assertTrue( False, "easyconfig %s does not contain blocks, yields only one parsed easyconfig" % spec) # check easyconfig file name expected_fn = '%s-%s.eb' % (ec['name'], det_full_ec_version(ec)) msg = "Filename '%s' of parsed easyconfig matches expected filename '%s'" % ( spec, expected_fn) self.assertEqual(os.path.basename(spec), expected_fn, msg) name, easyblock = fetch_parameters_from_easyconfig(ec.rawtxt, ['name', 'easyblock']) # make sure easyconfig file is in expected location expected_subdir = os.path.join('easybuild', 'easyconfigs', name.lower()[0], name) subdir = os.path.join(*spec.split(os.path.sep)[-5:-1]) fail_msg = "Easyconfig file %s not in expected subdirectory %s" % ( spec, expected_subdir) self.assertEqual(expected_subdir, subdir, fail_msg) # sanity check for software name self.assertTrue(ec['name'], name) # instantiate easyblock with easyconfig file app_class = get_easyblock_class(easyblock, name=name) # check that automagic fallback to ConfigureMake isn't done (deprecated behaviour) fn = os.path.basename(spec) error_msg = "%s relies on automagic fallback to ConfigureMake, should use easyblock = 'ConfigureMake' instead" % fn self.assertTrue(easyblock or not app_class is ConfigureMake, error_msg) app = app_class(ec) # more sanity checks self.assertTrue(name, app.name) self.assertTrue(ec['version'], app.version) # make sure all patch files are available specdir = os.path.dirname(spec) specfn = os.path.basename(spec) for patch in ec['patches']: if isinstance(patch, (tuple, list)): patch = patch[0] # only check actual patch files, not other files being copied via the patch functionality if patch.endswith('.patch'): patch_full = os.path.join(specdir, patch) msg = "Patch file %s is available for %s" % (patch_full, specfn) self.assertTrue(os.path.isfile(patch_full), msg) ext_patches = [] for ext in ec['exts_list']: if isinstance(ext, (tuple, list)) and len(ext) == 3: self.assertTrue(isinstance(ext[2], dict), "3rd element of extension spec is a dictionary") for ext_patch in ext[2].get('patches', []): if isinstance(ext_patch, (tuple, list)): ext_patch = ext_patch[0] # only check actual patch files, not other files being copied via the patch functionality if ext_patch.endswith('.patch'): ext_patch_full = os.path.join(specdir, ext_patch) msg = "Patch file %s is available for %s" % ( ext_patch_full, specfn) self.assertTrue(os.path.isfile(ext_patch_full), msg) # check whether all extra_options defined for used easyblock are defined for key in app.extra_options(): self.assertTrue(key in app.cfg) app.close_log() os.remove(app.logfile) # dump the easyconfig file handle, test_ecfile = tempfile.mkstemp() os.close(handle) ec.dump(test_ecfile) dumped_ec = EasyConfig(test_ecfile) os.remove(test_ecfile) # inject dummy values for templates that are only known at a later stage dummy_template_values = { 'builddir': '/dummy/builddir', 'installdir': '/dummy/installdir', } ec.template_values.update(dummy_template_values) dumped_ec.template_values.update(dummy_template_values) for key in sorted(ec._config): self.assertEqual(ec[key], dumped_ec[key]) # cache the parsed easyconfig, to avoid that it is parsed again self.parsed_easyconfigs.append(ecs[0]) # test passed, so set back to True single_tests_ok = True and prev_single_tests_ok
def resolve_dependencies(unprocessed, build_specs=None, retain_all_deps=False): """ Work through the list of easyconfigs to determine an optimal order @param unprocessed: list of easyconfigs @param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...) @param retain_all_deps: boolean indicating whether all dependencies must be retained, regardless of availability; retain all deps when True, check matching build option when False """ robot = build_option("robot_path") # retain all dependencies if specified by either the resp. build option or the dedicated named argument retain_all_deps = build_option("retain_all_deps") or retain_all_deps if retain_all_deps: # assume that no modules are available when forced, to retain all dependencies avail_modules = [] _log.info("Forcing all dependencies to be retained.") else: # Get a list of all available modules (format: [(name, installversion), ...]) avail_modules = modules_tool().available() if len(avail_modules) == 0: _log.warning("No installed modules. Your MODULEPATH is probably incomplete: %s" % os.getenv("MODULEPATH")) ordered_ecs = [] # all available modules can be used for resolving dependencies except those that will be installed being_installed = [p["full_mod_name"] for p in unprocessed] avail_modules = [m for m in avail_modules if not m in being_installed] _log.debug("unprocessed before resolving deps: %s" % unprocessed) # resolve all dependencies, put a safeguard in place to avoid an infinite loop (shouldn't occur though) irresolvable = [] loopcnt = 0 maxloopcnt = 10000 while unprocessed: # make sure this stops, we really don't want to get stuck in an infinite loop loopcnt += 1 if loopcnt > maxloopcnt: raise EasyBuildError( "Maximum loop cnt %s reached, so quitting (unprocessed: %s, irresolvable: %s)", maxloopcnt, unprocessed, irresolvable, ) # first try resolving dependencies without using external dependencies last_processed_count = -1 while len(avail_modules) > last_processed_count: last_processed_count = len(avail_modules) res = find_resolved_modules(unprocessed, avail_modules, retain_all_deps=retain_all_deps) more_ecs, unprocessed, avail_modules = res for ec in more_ecs: if not ec["full_mod_name"] in [x["full_mod_name"] for x in ordered_ecs]: ordered_ecs.append(ec) # dependencies marked as external modules should be resolved via available modules at this point missing_external_modules = [ d["full_mod_name"] for ec in unprocessed for d in ec["dependencies"] if d.get("external_module", False) ] if missing_external_modules: raise EasyBuildError( "Missing modules for one or more dependencies marked as external modules: %s", missing_external_modules ) # robot: look for existing dependencies, add them if robot and unprocessed: # rely on EasyBuild module naming scheme when resolving dependencies, since we know that will # generate sensible module names that include the necessary information for the resolution to work # (name, version, toolchain, versionsuffix) being_installed = [EasyBuildMNS().det_full_module_name(p["ec"]) for p in unprocessed] additional = [] for entry in unprocessed: # do not choose an entry that is being installed in the current run # if they depend, you probably want to rebuild them using the new dependency deps = entry["dependencies"] candidates = [d for d in deps if not EasyBuildMNS().det_full_module_name(d) in being_installed] if candidates: cand_dep = candidates[0] # find easyconfig, might not find any _log.debug("Looking for easyconfig for %s" % str(cand_dep)) # note: robot_find_easyconfig may return None path = robot_find_easyconfig(cand_dep["name"], det_full_ec_version(cand_dep)) if path is None: # no easyconfig found for dependency, add to list of irresolvable dependencies if cand_dep not in irresolvable: _log.debug("Irresolvable dependency found: %s" % cand_dep) irresolvable.append(cand_dep) # remove irresolvable dependency from list of dependencies so we can continue entry["dependencies"].remove(cand_dep) else: _log.info("Robot: resolving dependency %s with %s" % (cand_dep, path)) # build specs should not be passed down to resolved dependencies, # to avoid that e.g. --try-toolchain trickles down into the used toolchain itself hidden = cand_dep.get("hidden", False) processed_ecs = process_easyconfig(path, validate=not retain_all_deps, hidden=hidden) # ensure that selected easyconfig provides required dependency mods = [spec["ec"].full_mod_name for spec in processed_ecs] dep_mod_name = ActiveMNS().det_full_module_name(cand_dep) if not dep_mod_name in mods: raise EasyBuildError( "easyconfig file %s does not contain module %s (mods: %s)", path, dep_mod_name, mods ) for ec in processed_ecs: if not ec in unprocessed + additional: additional.append(ec) _log.debug("Added %s as dependency of %s" % (ec, entry)) else: mod_name = EasyBuildMNS().det_full_module_name(entry["ec"]) _log.debug("No more candidate dependencies to resolve for %s" % mod_name) # add additional (new) easyconfigs to list of stuff to process unprocessed.extend(additional) _log.debug("Unprocessed dependencies: %s", unprocessed) elif not robot: # no use in continuing if robot is not enabled, dependencies won't be resolved anyway irresolvable = [dep for x in unprocessed for dep in x["dependencies"]] break if irresolvable: _log.warning("Irresolvable dependencies (details): %s" % irresolvable) irresolvable_mods_eb = [EasyBuildMNS().det_full_module_name(dep) for dep in irresolvable] _log.warning("Irresolvable dependencies (EasyBuild module names): %s" % ", ".join(irresolvable_mods_eb)) irresolvable_mods = [ActiveMNS().det_full_module_name(dep) for dep in irresolvable] raise EasyBuildError("Irresolvable dependencies encountered: %s", ", ".join(irresolvable_mods)) _log.info("Dependency resolution complete, building as follows: %s" % ordered_ecs) return ordered_ecs
def resolve_dependencies(unprocessed, build_specs=None, retain_all_deps=False): """ Work through the list of easyconfigs to determine an optimal order @param unprocessed: list of easyconfigs @param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...) @param retain_all_deps: boolean indicating whether all dependencies must be retained, regardless of availability; retain all deps when True, check matching build option when False """ robot = build_option('robot_path') # retain all dependencies if specified by either the resp. build option or the dedicated named argument retain_all_deps = build_option('retain_all_deps') or retain_all_deps if retain_all_deps: # assume that no modules are available when forced, to retain all dependencies avail_modules = [] _log.info("Forcing all dependencies to be retained.") else: # Get a list of all available modules (format: [(name, installversion), ...]) avail_modules = modules_tool().available() if len(avail_modules) == 0: _log.warning( "No installed modules. Your MODULEPATH is probably incomplete: %s" % os.getenv('MODULEPATH')) ordered_ecs = [] # all available modules can be used for resolving dependencies except those that will be installed being_installed = [p['full_mod_name'] for p in unprocessed] avail_modules = [m for m in avail_modules if not m in being_installed] _log.debug('unprocessed before resolving deps: %s' % unprocessed) # resolve all dependencies, put a safeguard in place to avoid an infinite loop (shouldn't occur though) irresolvable = [] loopcnt = 0 maxloopcnt = 10000 while unprocessed: # make sure this stops, we really don't want to get stuck in an infinite loop loopcnt += 1 if loopcnt > maxloopcnt: tup = (maxloopcnt, unprocessed, irresolvable) msg = "Maximum loop cnt %s reached, so quitting (unprocessed: %s, irresolvable: %s)" % tup _log.error(msg) # first try resolving dependencies without using external dependencies last_processed_count = -1 while len(avail_modules) > last_processed_count: last_processed_count = len(avail_modules) res = find_resolved_modules(unprocessed, avail_modules, retain_all_deps=retain_all_deps) more_ecs, unprocessed, avail_modules = res for ec in more_ecs: if not ec['full_mod_name'] in [ x['full_mod_name'] for x in ordered_ecs ]: ordered_ecs.append(ec) # robot: look for existing dependencies, add them if robot and unprocessed: # rely on EasyBuild module naming scheme when resolving dependencies, since we know that will # generate sensible module names that include the necessary information for the resolution to work # (name, version, toolchain, versionsuffix) being_installed = [ EasyBuildMNS().det_full_module_name(p['ec']) for p in unprocessed ] additional = [] for entry in unprocessed: # do not choose an entry that is being installed in the current run # if they depend, you probably want to rebuild them using the new dependency deps = entry['dependencies'] candidates = [ d for d in deps if not EasyBuildMNS().det_full_module_name( d) in being_installed ] if candidates: cand_dep = candidates[0] # find easyconfig, might not find any _log.debug("Looking for easyconfig for %s" % str(cand_dep)) # note: robot_find_easyconfig may return None path = robot_find_easyconfig(cand_dep['name'], det_full_ec_version(cand_dep)) if path is None: # no easyconfig found for dependency, add to list of irresolvable dependencies if cand_dep not in irresolvable: _log.debug("Irresolvable dependency found: %s" % cand_dep) irresolvable.append(cand_dep) # remove irresolvable dependency from list of dependencies so we can continue entry['dependencies'].remove(cand_dep) else: _log.info("Robot: resolving dependency %s with %s" % (cand_dep, path)) # build specs should not be passed down to resolved dependencies, # to avoid that e.g. --try-toolchain trickles down into the used toolchain itself hidden = cand_dep.get('hidden', False) processed_ecs = process_easyconfig( path, validate=not retain_all_deps, hidden=hidden) # ensure that selected easyconfig provides required dependency mods = [ spec['ec'].full_mod_name for spec in processed_ecs ] dep_mod_name = ActiveMNS().det_full_module_name( cand_dep) if not dep_mod_name in mods: tup = (path, dep_mod_name, mods) _log.error( "easyconfig file %s does not contain module %s (mods: %s)" % tup) for ec in processed_ecs: if not ec in unprocessed + additional: additional.append(ec) _log.debug("Added %s as dependency of %s" % (ec, entry)) else: mod_name = EasyBuildMNS().det_full_module_name(entry['ec']) _log.debug( "No more candidate dependencies to resolve for %s" % mod_name) # add additional (new) easyconfigs to list of stuff to process unprocessed.extend(additional) elif not robot: # no use in continuing if robot is not enabled, dependencies won't be resolved anyway irresolvable = [ dep for x in unprocessed for dep in x['dependencies'] ] break if irresolvable: _log.warning("Irresolvable dependencies (details): %s" % irresolvable) irresolvable_mods_eb = [ EasyBuildMNS().det_full_module_name(dep) for dep in irresolvable ] _log.warning("Irresolvable dependencies (EasyBuild module names): %s" % ', '.join(irresolvable_mods_eb)) irresolvable_mods = [ ActiveMNS().det_full_module_name(dep) for dep in irresolvable ] _log.error('Irresolvable dependencies encountered: %s' % ', '.join(irresolvable_mods)) _log.info("Dependency resolution complete, building as follows:\n%s" % ordered_ecs) return ordered_ecs
def main(): """Main function.""" if len(sys.argv) == 3: ec = sys.argv[1] tc = sys.argv[2] else: error("Usage %s <easyconfig> <toolchain> [<name=version>]" % sys.argv[0]) tc_name, tc_ver = tc.split('/') print("Updating %s for %s toolchain version %s..." % (ec, tc_name, tc_ver)) set_up_configuration(silent=True) modtool = modules_tool() robot_path = build_option('robot_path') ec_path = det_easyconfig_paths([ec])[0] print("Found %s easyconfig file at %s" % (ec, ec_path)) parsed_ecs, _ = parse_easyconfigs([(ec_path, False)], validate=False) print("Resolving dependencies... ", end='') ecs = resolve_dependencies(parsed_ecs, modtool, retain_all_deps=True) print("found stack of %d easyconfigs" % len(ecs)) print("Filtering toolchain and its dependencies...") ec_tc = parsed_ecs[0]['ec']['toolchain'] ecs_to_remove = [{ 'name': ec_tc['name'], 'version': ec_tc['version'], 'toolchain': { 'name': SYSTEM_TOOLCHAIN_NAME } }] updated_ecs = {} # if GCCcore is used as toolchain, determine binutils version to use if tc_name == 'GCCcore': binutils_pattern = '^binutils.*-%s-%s.*.eb$' % (tc_name, tc_ver) _, res = search_file(robot_path, binutils_pattern) if res: if len(res) == 1: parsed_ecs, _ = parse_easyconfigs([(res[0], False)]) binutils_ec = parsed_ecs[0] tc = copy.copy(binutils_ec['ec']['toolchain']) ecs_to_remove.append({ 'name': 'binutils', 'version': binutils_ec['ec'].version, 'toolchain': tc }) else: error("Found more than one easyconfig matching '%s': %s" % (binutils_pattern, res)) else: error("No easyconfig file found for binutils using pattern '%s'" % binutils_pattern) while (ecs_to_remove): to_remove = ecs_to_remove.pop(0) print("Removing %(name)s/%(version)s (toolchain: %(toolchain)s)" % to_remove) for ec in ecs: if ec['ec'].name == to_remove['name'] and ec['ec'].version == to_remove['version'] and \ ec['ec']['toolchain']['name'] == to_remove['toolchain']['name']: ecs.remove(ec) ecs_to_remove.extend(dep for dep in ec['ec']['dependencies'] + ec['ec']['builddependencies']) updated_ecs[ec['full_mod_name']] = { 'builddependencies': [], 'dependencies': [], 'toolchain': copy.copy(ec['ec']['toolchain']), 'version': ec['ec'].version, } break ecs_to_write = [] for ec in ecs: ec_fn = os.path.basename(ec['spec']) print(term.bold("Determining version for %s..." % ec_fn)) full_mod_name = ec['full_mod_name'] ec_tc = copy.copy(ec['ec']['toolchain']) # update toolchain (unless it's SYSTEM) if ec_tc['name'] != SYSTEM_TOOLCHAIN_NAME: if ec_tc['name'] == tc_name: ec_tc['version'] = tc_ver else: error("Don't know how to update toolchain %s" % ec_tc['name']) # update (build) dependencies build_deps = [] for dep in ec['ec']['builddependencies']: new_dep_ver = updated_ecs[dep['full_mod_name']]['version'] build_deps.append((dep['name'], new_dep_ver)) deps = [] for dep in ec['ec']['dependencies']: new_dep_ver = updated_ecs[dep['full_mod_name']]['version'] deps.append((dep['name'], new_dep_ver)) # determine software version to use; # first, try searching for an existing easyconfig with specified toolchain; # if that fails, try to determine latest upstream version ec_pattern = '^%s.*-%s-%s.*.eb$' % (ec['ec'].name, tc_name, tc_ver) _, res = search_file(robot_path, ec_pattern) if res: if len(res) == 1: parsed_ecs, _ = parse_easyconfigs([(res[0], False)]) ec = parsed_ecs[0] new_version = ec['ec'].version print( term.green( "Found existing easyconfig, sticking to version %s" % new_version)) else: error("Multiple hits found using '%s': %s" % (res, ec_pattern)) else: new_version = update_version(ec['ec']) ecs_to_write.append(ec) if new_version is None: print( term.yellow( "No new version found for %s, using existing version" % full_mod_name)) new_version = ec['ec'].version updated_ecs[full_mod_name] = { 'builddependencies': build_deps, 'dependencies': deps, 'toolchain': ec_tc, 'version': new_version, } for ec in ecs_to_write: full_mod_name = ec['full_mod_name'] pprint.pprint(full_mod_name) ec = ec['ec'] ectxt = ec.rawtxt key_pattern = r'^%s\s*=.*' list_key_pattern = r'^%s\s*=\s*\[([^\]]|\n)*\s*\]' new_version = updated_ecs[full_mod_name]['version'] if ec.version != new_version: regex = re.compile(key_pattern % 'version', re.M) ectxt = regex.sub("version = '%s'" % new_version, ectxt) # if version got updated, also wipe the checksums regex = re.compile(list_key_pattern % 'checksums', re.M) ectxt = regex.sub("checksums = []", ectxt) # toolchain tc_str = "toolchain = {'name': '%(name)s', 'version': '%(version)s'}" % updated_ecs[ full_mod_name]['toolchain'] regex = re.compile(key_pattern % 'toolchain', re.M) ectxt = regex.sub(tc_str, ectxt) # dependencies for key in ('builddependencies', 'dependencies'): deps_str = '%s = [\n' % key for dep in updated_ecs[full_mod_name][key]: deps_str += ' ' + str(dep) + ',\n' deps_str += ']' regex = re.compile(list_key_pattern % key, re.M) ectxt = regex.sub(deps_str, ectxt) specs = { 'name': ec.name, 'toolchain': updated_ecs[full_mod_name]['toolchain'], 'version': new_version, 'versionsuffix': ec['versionsuffix'], } ec_fn = '%s-%s.eb' % (ec.name, det_full_ec_version(specs)) write_file(ec_fn, ectxt) print(term.green("%s written" % ec_fn))