def test_installversion(self): """test generation of install version""" ver = "3.14" verpref = "myprefix|" versuff = "|mysuffix" tcname = "GCC" tcver = "4.6.3" dummy = "dummy" installver = det_installversion(ver, tcname, tcver, verpref, versuff) self.assertEqual(installver, "%s%s-%s-%s%s" % (verpref, ver, tcname, tcver, versuff)) installver = det_installversion(ver, dummy, tcver, verpref, versuff) self.assertEqual(installver, "%s%s%s" % (verpref, ver, versuff))
def runTest(self): ver = "3.14" verpref = "myprefix|" versuff = "|mysuffix" tcname = "GCC" tcver = "4.6.3" dummy = "dummy" installver = det_installversion(ver, tcname, tcver, verpref, versuff) self.assertEqual(installver, "%s%s-%s-%s%s" % (verpref, ver, tcname, tcver, versuff)) installver = det_installversion(ver, dummy, tcver, verpref, versuff) self.assertEqual(installver, "%s%s%s" % (verpref, ver, versuff))
def test_legacy_installversion(self): """Test generation of install version (legacy).""" ver = "3.14" verpref = "myprefix|" versuff = "|mysuffix" tcname = "GCC" tcver = "4.6.3" dummy = "dummy" correct_installver = "%s%s-%s-%s%s" % (verpref, ver, tcname, tcver, versuff) installver = det_installversion(ver, tcname, tcver, verpref, versuff) self.assertEqual(installver, correct_installver) correct_installver = "%s%s%s" % (verpref, ver, versuff) installver = det_installversion(ver, dummy, tcver, verpref, versuff) self.assertEqual(installver, correct_installver)
def test_installversion(self): """test generation of install version""" ver = "3.14" verpref = "myprefix|" versuff = "|mysuffix" tcname = "GCC" tcver = "4.6.3" dummy = "dummy" installver = det_installversion(ver, tcname, tcver, verpref, versuff) self.assertEqual( installver, "%s%s-%s-%s%s" % (verpref, ver, tcname, tcver, versuff)) installver = det_installversion(ver, dummy, tcver, verpref, versuff) self.assertEqual(installver, "%s%s%s" % (verpref, ver, versuff))
def obtain_ec_for(specs, paths, fp): """ Obtain an easyconfig file to the given specifications. Either select between available ones, or use the best suited available one to generate a new easyconfig file. <paths> is a list of paths where easyconfig files can be found <fp> is the desired file name <log> is an EasyBuildLog instance """ # ensure that at least name is specified if not specs.get('name'): _log.error( "Supplied 'specs' dictionary doesn't even contain a name of a software package?" ) # collect paths to search in if not paths: _log.error( "No paths to look for easyconfig files, specify a path with --robot." ) # create glob patterns based on supplied info # figure out the install version installver = det_installversion(specs.get('version', '*'), specs.get('toolchain_name', '*'), specs.get('toolchain_version', '*'), specs.get('versionprefix', '*'), specs.get('versionsuffix', '*')) # find easyconfigs that match a pattern easyconfig_files = [] for path in paths: patterns = create_paths(path, specs['name'], installver) for pattern in patterns: easyconfig_files.extend(glob.glob(pattern)) cnt = len(easyconfig_files) _log.debug("List of obtained easyconfig files (%d): %s" % (cnt, easyconfig_files)) # select best easyconfig, or try to generate one that fits the requirements res = select_or_generate_ec(fp, paths, specs) if res: return res else: _log.error( "No easyconfig found for requested software, and also failed to generate one." )
def ec_filename_for(path): """ Return a suiting file name for the easyconfig file at <path>, as determined by its contents. """ ec = EasyConfig(path, validate=False) fn = "%s-%s.eb" % (ec['name'], det_installversion(ec['version'], ec['toolchain']['name'], ec['toolchain']['version'], ec['versionprefix'], ec['versionsuffix'])) return fn
def ec_filename_for(path): """ Return a suiting file name for the easyconfig file at <path>, as determined by its contents. """ ec = EasyConfig(path, validate=False) fn = "%s-%s.eb" % (ec['name'], det_installversion( ec['version'], ec['toolchain']['name'], ec['toolchain']['version'], ec['versionprefix'], ec['versionsuffix'])) return fn
def obtain_ec_for(specs, paths, fp): """ Obtain an easyconfig file to the given specifications. Either select between available ones, or use the best suited available one to generate a new easyconfig file. <paths> is a list of paths where easyconfig files can be found <fp> is the desired file name <log> is an EasyBuildLog instance """ # ensure that at least name is specified if not specs.get('name'): _log.error("Supplied 'specs' dictionary doesn't even contain a name of a software package?") # collect paths to search in if not paths: _log.error("No paths to look for easyconfig files, specify a path with --robot.") # create glob patterns based on supplied info # figure out the install version installver = det_installversion(specs.get('version', '*'), specs.get('toolchain_name', '*'), specs.get('toolchain_version', '*'), specs.get('versionprefix', '*'), specs.get('versionsuffix', '*')) # find easyconfigs that match a pattern easyconfig_files = [] for path in paths: patterns = create_paths(path, specs['name'], installver) for pattern in patterns: easyconfig_files.extend(glob.glob(pattern)) cnt = len(easyconfig_files) _log.debug("List of obtained easyconfig files (%d): %s" % (cnt, easyconfig_files)) # select best easyconfig, or try to generate one that fits the requirements res = select_or_generate_ec(fp, paths, specs) if res: return res else: _log.error("No easyconfig found for requested software, and also failed to generate one.")
def select_or_generate_ec(fp, paths, specs): """ Select or generate an easyconfig file with the given requirements, from existing easyconfig files. If easyconfig files are available for the specified software package, then this function will first try to determine which toolchain to use. * if a toolchain is given, it will use it (possible using a template easyconfig file as base); * if not, and only a single toolchain is available, is will assume it can use that toolchain * else, it fails -- EasyBuild doesn't select between multiple available toolchains Next, it will trim down the selected easyconfig files to a single one, based on the following requirements (in order of preference): * toolchain version * software version * other parameters (e.g. versionprefix, versionsuffix, etc.) If a complete match is found, it will return that easyconfig. Else, it will generate a new easyconfig file based on the selected 'best matching' easyconfig file. """ specs = copy.deepcopy(specs) # ensure that at least name is specified if not specs.get('name'): _log.error("Supplied 'specs' dictionary doesn't even contain a name of a software package?") name = specs['name'] handled_params = ['name'] # find ALL available easyconfig files for specified software ec_files = [] installver = det_installversion('*', 'dummy', '*', '*', '*') for path in paths: patterns = create_paths(path, name, installver) for pattern in patterns: ec_files.extend(glob.glob(pattern)) # we need at least one config file to start from if len(ec_files) == 0: # look for a template file if no easyconfig for specified software name is available for path in paths: templ_file = os.path.join(path, "TEMPLATE.eb") if os.path.isfile(templ_file): ec_files = [templ_file] break else: _log.debug("No template found at %s." % templ_file) if len(ec_files) == 0: _log.error("No easyconfig files found for software %s, and no templates available. I'm all out of ideas." % name) # we can't rely on set, because we also need to be able to obtain a list of unique lists def unique(l): """Retain unique elements in a sorted list.""" l = sorted(l) if len(l) > 1: l2 = [l[0]] for x in l: if not x == l2[-1]: l2.append(x) return l2 else: return l # filter unique ec_files = nub(ec_files) _log.debug("Unique ec_files: %s" % ec_files) ecs_and_files = [(EasyConfig(f, validate=False), f) for f in ec_files] # TOOLCHAIN NAME # determine list of unique toolchain names tcnames = unique([x[0]['toolchain']['name'] for x in ecs_and_files]) _log.debug("Found %d unique toolchain names: %s" % (len(tcnames), tcnames)) # if a toolchain was selected, and we have no easyconfig files for it, try and use a template if specs.get('toolchain_name') and not specs['toolchain_name'] in tcnames: if "TEMPLATE" in tcnames: _log.info("No easyconfig file for specified toolchain, but template is available.") else: _log.error("No easyconfig file for %s with toolchain %s, " \ "and no template available." % (name, specs['toolchain_name'])) tcname = specs.pop('toolchain_name', None) handled_params.append('toolchain_name') # trim down list according to selected toolchain if tcname in tcnames: # known toolchain, so only retain those selected_tcname = tcname else: if len(tcnames) == 1 and not tcnames[0] == "TEMPLATE": # only one (non-template) toolchain availble, so use that tcname = tcnames[0] selected_tcname = tcname elif len(tcnames) == 1 and tcnames[0] == "TEMPLATE": selected_tcname = tcnames[0] else: # fall-back: use template toolchain if a toolchain name was specified if tcname: selected_tcname = "TEMPLATE" else: # if multiple toolchains are available, and none is specified, we quit # we can't just pick one, how would we prefer one over the other? _log.error("No toolchain name specified, and more than one available: %s." % tcnames) _log.debug("Filtering easyconfigs based on toolchain name '%s'..." % selected_tcname) ecs_and_files = [x for x in ecs_and_files if x[0]['toolchain']['name'] == selected_tcname] _log.debug("Filtered easyconfigs: %s" % [x[1] for x in ecs_and_files]) # TOOLCHAIN VERSION tcvers = unique([x[0]['toolchain']['version'] for x in ecs_and_files]) _log.debug("Found %d unique toolchain versions: %s" % (len(tcvers), tcvers)) tcver = specs.pop('toolchain_version', None) handled_params.append('toolchain_version') (tcver, selected_tcver) = pick_version(tcver, tcvers) _log.debug("Filtering easyconfigs based on toolchain version '%s'..." % selected_tcver) ecs_and_files = [x for x in ecs_and_files if x[0]['toolchain']['version'] == selected_tcver] _log.debug("Filtered easyconfigs: %s" % [x[1] for x in ecs_and_files]) # add full toolchain specification to specs if tcname and tcver: specs.update({'toolchain': {'name': tcname, 'version': tcver}}) handled_params.append('toolchain') else: if tcname: specs.update({'toolchain_name': tcname}) if tcver: specs.update({'toolchain_version': tcver}) # SOFTWARE VERSION vers = unique([x[0]['version'] for x in ecs_and_files]) _log.debug("Found %d unique software versions: %s" % (len(vers), vers)) ver = specs.pop('version', None) handled_params.append('version') (ver, selected_ver) = pick_version(ver, vers) if ver: specs.update({'version': ver}) _log.debug("Filtering easyconfigs based on software version '%s'..." % selected_ver) ecs_and_files = [x for x in ecs_and_files if x[0]['version'] == selected_ver] _log.debug("Filtered easyconfigs: %s" % [x[1] for x in ecs_and_files]) # go through parameters specified via --amend # always include versionprefix/suffix, because we might need it to generate a file name verpref = None versuff = None other_params = {'versionprefix': None, 'versionsuffix': None} for (param, val) in specs.items(): if not param in handled_params: other_params.update({param: val}) _log.debug("Filtering based on other parameters (specified via --amend): %s" % other_params) for (param, val) in other_params.items(): if param in ecs_and_files[0][0]._config: vals = unique([x[0][param] for x in ecs_and_files]) else: vals = [] filter_ecs = False # try and select a value from the available ones, or fail if we can't if val in vals: # if the specified value is available, use it selected_val = val _log.debug("Specified %s is available, so using it: %s" % (param, selected_val)) filter_ecs = True elif val: # if a value is specified, use that, even if it's not available yet selected_val = val _log.debug("%s is specified, so using it (even though it's not available yet): %s" % (param, selected_val)) elif len(vals) == 1: # if only one value is available, use that selected_val = vals[0] _log.debug("Only one %s available ('%s'), so picking that" % (param, selected_val)) filter_ecs = True else: # otherwise, we fail, because we don't know how to pick between different fixes _log.error("No %s specified, and can't pick from available %ses %s" % (param, param, vals)) if filter_ecs: _log.debug("Filtering easyconfigs based on %s '%s'..." % (param, selected_val)) ecs_and_files = [x for x in ecs_and_files if x[0][param] == selected_val] _log.debug("Filtered easyconfigs: %s" % [x[1] for x in ecs_and_files]) # keep track of versionprefix/suffix if param == "versionprefix": verpref = selected_val elif param == "versionsuffix": versuff = selected_val cnt = len(ecs_and_files) if not cnt == 1: fs = [x[1] for x in ecs_and_files] _log.error("Failed to select a single easyconfig from available ones, %s left: %s" % (cnt, fs)) else: (selected_ec, selected_ec_file) = ecs_and_files[0] # check whether selected easyconfig matches requirements match = True for (key, val) in specs.items(): if key in selected_ec._config: # values must be equal to have a full match if not selected_ec[key] == val: match = False else: # if we encounter a key that is not set in the selected easyconfig, we don't have a full match match = False # if it matches, no need to tweak if match: _log.info("Perfect match found: %s" % selected_ec_file) return (False, selected_ec_file) # GENERATE # if no file path was specified, generate a file name if not fp: installver = det_installversion(ver, tcname, tcver, verpref, versuff) fp = "%s-%s.eb" % (name, installver) # generate tweaked easyconfig file tweak(selected_ec_file, fp, specs) _log.info("Generated easyconfig file %s, and using it to build the requested software." % fp) return (True, fp)
def select_or_generate_ec(fp, paths, specs): """ Select or generate an easyconfig file with the given requirements, from existing easyconfig files. If easyconfig files are available for the specified software package, then this function will first try to determine which toolchain to use. * if a toolchain is given, it will use it (possible using a template easyconfig file as base); * if not, and only a single toolchain is available, is will assume it can use that toolchain * else, it fails -- EasyBuild doesn't select between multiple available toolchains Next, it will trim down the selected easyconfig files to a single one, based on the following requirements (in order of preference): * toolchain version * software version * other parameters (e.g. versionprefix, versionsuffix, etc.) If a complete match is found, it will return that easyconfig. Else, it will generate a new easyconfig file based on the selected 'best matching' easyconfig file. """ specs = copy.deepcopy(specs) # ensure that at least name is specified if not specs.get('name'): _log.error( "Supplied 'specs' dictionary doesn't even contain a name of a software package?" ) name = specs['name'] handled_params = ['name'] # find ALL available easyconfig files for specified software ec_files = [] installver = det_installversion('*', 'dummy', '*', '*', '*') for path in paths: patterns = create_paths(path, name, installver) for pattern in patterns: ec_files.extend(glob.glob(pattern)) # we need at least one config file to start from if len(ec_files) == 0: # look for a template file if no easyconfig for specified software name is available for path in paths: templ_file = os.path.join(path, "TEMPLATE.eb") if os.path.isfile(templ_file): ec_files = [templ_file] break else: _log.debug("No template found at %s." % templ_file) if len(ec_files) == 0: _log.error( "No easyconfig files found for software %s, and no templates available. I'm all out of ideas." % name) # we can't rely on set, because we also need to be able to obtain a list of unique lists def unique(l): """Retain unique elements in a sorted list.""" l = sorted(l) if len(l) > 1: l2 = [l[0]] for x in l: if not x == l2[-1]: l2.append(x) return l2 else: return l # filter unique ec_files = nub(ec_files) _log.debug("Unique ec_files: %s" % ec_files) ecs_and_files = [(EasyConfig(f, validate=False), f) for f in ec_files] # TOOLCHAIN NAME # determine list of unique toolchain names tcnames = unique([x[0]['toolchain']['name'] for x in ecs_and_files]) _log.debug("Found %d unique toolchain names: %s" % (len(tcnames), tcnames)) # if a toolchain was selected, and we have no easyconfig files for it, try and use a template if specs.get('toolchain_name') and not specs['toolchain_name'] in tcnames: if "TEMPLATE" in tcnames: _log.info( "No easyconfig file for specified toolchain, but template is available." ) else: _log.error("No easyconfig file for %s with toolchain %s, " \ "and no template available." % (name, specs['toolchain_name'])) tcname = specs.pop('toolchain_name', None) handled_params.append('toolchain_name') # trim down list according to selected toolchain if tcname in tcnames: # known toolchain, so only retain those selected_tcname = tcname else: if len(tcnames) == 1 and not tcnames[0] == "TEMPLATE": # only one (non-template) toolchain availble, so use that tcname = tcnames[0] selected_tcname = tcname elif len(tcnames) == 1 and tcnames[0] == "TEMPLATE": selected_tcname = tcnames[0] else: # fall-back: use template toolchain if a toolchain name was specified if tcname: selected_tcname = "TEMPLATE" else: # if multiple toolchains are available, and none is specified, we quit # we can't just pick one, how would we prefer one over the other? _log.error( "No toolchain name specified, and more than one available: %s." % tcnames) _log.debug("Filtering easyconfigs based on toolchain name '%s'..." % selected_tcname) ecs_and_files = [ x for x in ecs_and_files if x[0]['toolchain']['name'] == selected_tcname ] _log.debug("Filtered easyconfigs: %s" % [x[1] for x in ecs_and_files]) # TOOLCHAIN VERSION tcvers = unique([x[0]['toolchain']['version'] for x in ecs_and_files]) _log.debug("Found %d unique toolchain versions: %s" % (len(tcvers), tcvers)) tcver = specs.pop('toolchain_version', None) handled_params.append('toolchain_version') (tcver, selected_tcver) = pick_version(tcver, tcvers) _log.debug("Filtering easyconfigs based on toolchain version '%s'..." % selected_tcver) ecs_and_files = [ x for x in ecs_and_files if x[0]['toolchain']['version'] == selected_tcver ] _log.debug("Filtered easyconfigs: %s" % [x[1] for x in ecs_and_files]) # add full toolchain specification to specs if tcname and tcver: specs.update({'toolchain': {'name': tcname, 'version': tcver}}) handled_params.append('toolchain') else: if tcname: specs.update({'toolchain_name': tcname}) if tcver: specs.update({'toolchain_version': tcver}) # SOFTWARE VERSION vers = unique([x[0]['version'] for x in ecs_and_files]) _log.debug("Found %d unique software versions: %s" % (len(vers), vers)) ver = specs.pop('version', None) handled_params.append('version') (ver, selected_ver) = pick_version(ver, vers) if ver: specs.update({'version': ver}) _log.debug("Filtering easyconfigs based on software version '%s'..." % selected_ver) ecs_and_files = [ x for x in ecs_and_files if x[0]['version'] == selected_ver ] _log.debug("Filtered easyconfigs: %s" % [x[1] for x in ecs_and_files]) # go through parameters specified via --amend # always include versionprefix/suffix, because we might need it to generate a file name verpref = None versuff = None other_params = {'versionprefix': None, 'versionsuffix': None} for (param, val) in specs.items(): if not param in handled_params: other_params.update({param: val}) _log.debug( "Filtering based on other parameters (specified via --amend): %s" % other_params) for (param, val) in other_params.items(): if param in ecs_and_files[0][0]._config: vals = unique([x[0][param] for x in ecs_and_files]) else: vals = [] filter_ecs = False # try and select a value from the available ones, or fail if we can't if val in vals: # if the specified value is available, use it selected_val = val _log.debug("Specified %s is available, so using it: %s" % (param, selected_val)) filter_ecs = True elif val: # if a value is specified, use that, even if it's not available yet selected_val = val _log.debug( "%s is specified, so using it (even though it's not available yet): %s" % (param, selected_val)) elif len(vals) == 1: # if only one value is available, use that selected_val = vals[0] _log.debug("Only one %s available ('%s'), so picking that" % (param, selected_val)) filter_ecs = True else: # otherwise, we fail, because we don't know how to pick between different fixes _log.error( "No %s specified, and can't pick from available %ses %s" % (param, param, vals)) if filter_ecs: _log.debug("Filtering easyconfigs based on %s '%s'..." % (param, selected_val)) ecs_and_files = [ x for x in ecs_and_files if x[0][param] == selected_val ] _log.debug("Filtered easyconfigs: %s" % [x[1] for x in ecs_and_files]) # keep track of versionprefix/suffix if param == "versionprefix": verpref = selected_val elif param == "versionsuffix": versuff = selected_val cnt = len(ecs_and_files) if not cnt == 1: fs = [x[1] for x in ecs_and_files] _log.error( "Failed to select a single easyconfig from available ones, %s left: %s" % (cnt, fs)) else: (selected_ec, selected_ec_file) = ecs_and_files[0] # check whether selected easyconfig matches requirements match = True for (key, val) in specs.items(): if key in selected_ec._config: # values must be equal to have a full match if not selected_ec[key] == val: match = False else: # if we encounter a key that is not set in the selected easyconfig, we don't have a full match match = False # if it matches, no need to tweak if match: _log.info("Perfect match found: %s" % selected_ec_file) return (False, selected_ec_file) # GENERATE # if no file path was specified, generate a file name if not fp: installver = det_installversion(ver, tcname, tcver, verpref, versuff) fp = "%s-%s.eb" % (name, installver) # generate tweaked easyconfig file tweak(selected_ec_file, fp, specs) _log.info( "Generated easyconfig file %s, and using it to build the requested software." % fp) return (True, fp)