def test_installversion(self):
"""test generation of install version"""
ver = "3.14"
verpref = "myprefix|"
versuff = "|mysuffix"
tcname = "GCC"
tcver = "4.6.3"
dummy = "dummy"
installver = det_installversion(ver, tcname, tcver, verpref, versuff)
self.assertEqual(installver, "%s%s-%s-%s%s" % (verpref, ver, tcname, tcver, versuff))
installver = det_installversion(ver, dummy, tcver, verpref, versuff)
self.assertEqual(installver, "%s%s%s" % (verpref, ver, versuff))
def runTest(self):
ver = "3.14"
verpref = "myprefix|"
versuff = "|mysuffix"
tcname = "GCC"
tcver = "4.6.3"
dummy = "dummy"
installver = det_installversion(ver, tcname, tcver, verpref, versuff)
self.assertEqual(installver, "%s%s-%s-%s%s" % (verpref, ver, tcname, tcver, versuff))
installver = det_installversion(ver, dummy, tcver, verpref, versuff)
self.assertEqual(installver, "%s%s%s" % (verpref, ver, versuff))
def test_legacy_installversion(self):
"""Test generation of install version (legacy)."""
ver = "3.14"
verpref = "myprefix|"
versuff = "|mysuffix"
tcname = "GCC"
tcver = "4.6.3"
dummy = "dummy"
correct_installver = "%s%s-%s-%s%s" % (verpref, ver, tcname, tcver, versuff)
installver = det_installversion(ver, tcname, tcver, verpref, versuff)
self.assertEqual(installver, correct_installver)
correct_installver = "%s%s%s" % (verpref, ver, versuff)
installver = det_installversion(ver, dummy, tcver, verpref, versuff)
self.assertEqual(installver, correct_installver)
def test_installversion(self):
"""test generation of install version"""
ver = "3.14"
verpref = "myprefix|"
versuff = "|mysuffix"
tcname = "GCC"
tcver = "4.6.3"
dummy = "dummy"
installver = det_installversion(ver, tcname, tcver, verpref, versuff)
self.assertEqual(
installver,
"%s%s-%s-%s%s" % (verpref, ver, tcname, tcver, versuff))
installver = det_installversion(ver, dummy, tcver, verpref, versuff)
self.assertEqual(installver, "%s%s%s" % (verpref, ver, versuff))
def obtain_ec_for(specs, paths, fp):
"""
Obtain an easyconfig file to the given specifications.
Either select between available ones, or use the best suited available one
to generate a new easyconfig file.
<paths> is a list of paths where easyconfig files can be found
<fp> is the desired file name
<log> is an EasyBuildLog instance
"""
# ensure that at least name is specified
if not specs.get('name'):
_log.error(
"Supplied 'specs' dictionary doesn't even contain a name of a software package?"
)
# collect paths to search in
if not paths:
_log.error(
"No paths to look for easyconfig files, specify a path with --robot."
)
# create glob patterns based on supplied info
# figure out the install version
installver = det_installversion(specs.get('version', '*'),
specs.get('toolchain_name', '*'),
specs.get('toolchain_version', '*'),
specs.get('versionprefix', '*'),
specs.get('versionsuffix', '*'))
# find easyconfigs that match a pattern
easyconfig_files = []
for path in paths:
patterns = create_paths(path, specs['name'], installver)
for pattern in patterns:
easyconfig_files.extend(glob.glob(pattern))
cnt = len(easyconfig_files)
_log.debug("List of obtained easyconfig files (%d): %s" %
(cnt, easyconfig_files))
# select best easyconfig, or try to generate one that fits the requirements
res = select_or_generate_ec(fp, paths, specs)
if res:
return res
else:
_log.error(
"No easyconfig found for requested software, and also failed to generate one."
)
def ec_filename_for(path):
"""
Return a suiting file name for the easyconfig file at <path>,
as determined by its contents.
"""
ec = EasyConfig(path, validate=False)
fn = "%s-%s.eb" % (ec['name'], det_installversion(ec['version'], ec['toolchain']['name'],
ec['toolchain']['version'], ec['versionprefix'],
ec['versionsuffix']))
return fn
def ec_filename_for(path):
"""
Return a suiting file name for the easyconfig file at <path>,
as determined by its contents.
"""
ec = EasyConfig(path, validate=False)
fn = "%s-%s.eb" % (ec['name'],
det_installversion(
ec['version'], ec['toolchain']['name'],
ec['toolchain']['version'], ec['versionprefix'],
ec['versionsuffix']))
return fn
def obtain_ec_for(specs, paths, fp):
"""
Obtain an easyconfig file to the given specifications.
Either select between available ones, or use the best suited available one
to generate a new easyconfig file.
<paths> is a list of paths where easyconfig files can be found
<fp> is the desired file name
<log> is an EasyBuildLog instance
"""
# ensure that at least name is specified
if not specs.get('name'):
_log.error("Supplied 'specs' dictionary doesn't even contain a name of a software package?")
# collect paths to search in
if not paths:
_log.error("No paths to look for easyconfig files, specify a path with --robot.")
# create glob patterns based on supplied info
# figure out the install version
installver = det_installversion(specs.get('version', '*'), specs.get('toolchain_name', '*'),
specs.get('toolchain_version', '*'), specs.get('versionprefix', '*'),
specs.get('versionsuffix', '*'))
# find easyconfigs that match a pattern
easyconfig_files = []
for path in paths:
patterns = create_paths(path, specs['name'], installver)
for pattern in patterns:
easyconfig_files.extend(glob.glob(pattern))
cnt = len(easyconfig_files)
_log.debug("List of obtained easyconfig files (%d): %s" % (cnt, easyconfig_files))
# select best easyconfig, or try to generate one that fits the requirements
res = select_or_generate_ec(fp, paths, specs)
if res:
return res
else:
_log.error("No easyconfig found for requested software, and also failed to generate one.")
def select_or_generate_ec(fp, paths, specs):
"""
Select or generate an easyconfig file with the given requirements, from existing easyconfig files.
If easyconfig files are available for the specified software package,
then this function will first try to determine which toolchain to use.
* if a toolchain is given, it will use it (possible using a template easyconfig file as base);
* if not, and only a single toolchain is available, is will assume it can use that toolchain
* else, it fails -- EasyBuild doesn't select between multiple available toolchains
Next, it will trim down the selected easyconfig files to a single one,
based on the following requirements (in order of preference):
* toolchain version
* software version
* other parameters (e.g. versionprefix, versionsuffix, etc.)
If a complete match is found, it will return that easyconfig.
Else, it will generate a new easyconfig file based on the selected 'best matching' easyconfig file.
"""
specs = copy.deepcopy(specs)
# ensure that at least name is specified
if not specs.get('name'):
_log.error("Supplied 'specs' dictionary doesn't even contain a name of a software package?")
name = specs['name']
handled_params = ['name']
# find ALL available easyconfig files for specified software
ec_files = []
installver = det_installversion('*', 'dummy', '*', '*', '*')
for path in paths:
patterns = create_paths(path, name, installver)
for pattern in patterns:
ec_files.extend(glob.glob(pattern))
# we need at least one config file to start from
if len(ec_files) == 0:
# look for a template file if no easyconfig for specified software name is available
for path in paths:
templ_file = os.path.join(path, "TEMPLATE.eb")
if os.path.isfile(templ_file):
ec_files = [templ_file]
break
else:
_log.debug("No template found at %s." % templ_file)
if len(ec_files) == 0:
_log.error("No easyconfig files found for software %s, and no templates available. I'm all out of ideas." % name)
# we can't rely on set, because we also need to be able to obtain a list of unique lists
def unique(l):
"""Retain unique elements in a sorted list."""
l = sorted(l)
if len(l) > 1:
l2 = [l[0]]
for x in l:
if not x == l2[-1]:
l2.append(x)
return l2
else:
return l
# filter unique
ec_files = nub(ec_files)
_log.debug("Unique ec_files: %s" % ec_files)
ecs_and_files = [(EasyConfig(f, validate=False), f) for f in ec_files]
# TOOLCHAIN NAME
# determine list of unique toolchain names
tcnames = unique([x[0]['toolchain']['name'] for x in ecs_and_files])
_log.debug("Found %d unique toolchain names: %s" % (len(tcnames), tcnames))
# if a toolchain was selected, and we have no easyconfig files for it, try and use a template
if specs.get('toolchain_name') and not specs['toolchain_name'] in tcnames:
if "TEMPLATE" in tcnames:
_log.info("No easyconfig file for specified toolchain, but template is available.")
else:
_log.error("No easyconfig file for %s with toolchain %s, " \
"and no template available." % (name, specs['toolchain_name']))
tcname = specs.pop('toolchain_name', None)
handled_params.append('toolchain_name')
# trim down list according to selected toolchain
if tcname in tcnames:
# known toolchain, so only retain those
selected_tcname = tcname
else:
if len(tcnames) == 1 and not tcnames[0] == "TEMPLATE":
# only one (non-template) toolchain availble, so use that
tcname = tcnames[0]
selected_tcname = tcname
elif len(tcnames) == 1 and tcnames[0] == "TEMPLATE":
selected_tcname = tcnames[0]
else:
# fall-back: use template toolchain if a toolchain name was specified
if tcname:
selected_tcname = "TEMPLATE"
else:
# if multiple toolchains are available, and none is specified, we quit
# we can't just pick one, how would we prefer one over the other?
_log.error("No toolchain name specified, and more than one available: %s." % tcnames)
_log.debug("Filtering easyconfigs based on toolchain name '%s'..." % selected_tcname)
ecs_and_files = [x for x in ecs_and_files if x[0]['toolchain']['name'] == selected_tcname]
_log.debug("Filtered easyconfigs: %s" % [x[1] for x in ecs_and_files])
# TOOLCHAIN VERSION
tcvers = unique([x[0]['toolchain']['version'] for x in ecs_and_files])
_log.debug("Found %d unique toolchain versions: %s" % (len(tcvers), tcvers))
tcver = specs.pop('toolchain_version', None)
handled_params.append('toolchain_version')
(tcver, selected_tcver) = pick_version(tcver, tcvers)
_log.debug("Filtering easyconfigs based on toolchain version '%s'..." % selected_tcver)
ecs_and_files = [x for x in ecs_and_files if x[0]['toolchain']['version'] == selected_tcver]
_log.debug("Filtered easyconfigs: %s" % [x[1] for x in ecs_and_files])
# add full toolchain specification to specs
if tcname and tcver:
specs.update({'toolchain': {'name': tcname, 'version': tcver}})
handled_params.append('toolchain')
else:
if tcname:
specs.update({'toolchain_name': tcname})
if tcver:
specs.update({'toolchain_version': tcver})
# SOFTWARE VERSION
vers = unique([x[0]['version'] for x in ecs_and_files])
_log.debug("Found %d unique software versions: %s" % (len(vers), vers))
ver = specs.pop('version', None)
handled_params.append('version')
(ver, selected_ver) = pick_version(ver, vers)
if ver:
specs.update({'version': ver})
_log.debug("Filtering easyconfigs based on software version '%s'..." % selected_ver)
ecs_and_files = [x for x in ecs_and_files if x[0]['version'] == selected_ver]
_log.debug("Filtered easyconfigs: %s" % [x[1] for x in ecs_and_files])
# go through parameters specified via --amend
# always include versionprefix/suffix, because we might need it to generate a file name
verpref = None
versuff = None
other_params = {'versionprefix': None, 'versionsuffix': None}
for (param, val) in specs.items():
if not param in handled_params:
other_params.update({param: val})
_log.debug("Filtering based on other parameters (specified via --amend): %s" % other_params)
for (param, val) in other_params.items():
if param in ecs_and_files[0][0]._config:
vals = unique([x[0][param] for x in ecs_and_files])
else:
vals = []
filter_ecs = False
# try and select a value from the available ones, or fail if we can't
if val in vals:
# if the specified value is available, use it
selected_val = val
_log.debug("Specified %s is available, so using it: %s" % (param, selected_val))
filter_ecs = True
elif val:
# if a value is specified, use that, even if it's not available yet
selected_val = val
_log.debug("%s is specified, so using it (even though it's not available yet): %s" % (param, selected_val))
elif len(vals) == 1:
# if only one value is available, use that
selected_val = vals[0]
_log.debug("Only one %s available ('%s'), so picking that" % (param, selected_val))
filter_ecs = True
else:
# otherwise, we fail, because we don't know how to pick between different fixes
_log.error("No %s specified, and can't pick from available %ses %s" % (param,
param,
vals))
if filter_ecs:
_log.debug("Filtering easyconfigs based on %s '%s'..." % (param, selected_val))
ecs_and_files = [x for x in ecs_and_files if x[0][param] == selected_val]
_log.debug("Filtered easyconfigs: %s" % [x[1] for x in ecs_and_files])
# keep track of versionprefix/suffix
if param == "versionprefix":
verpref = selected_val
elif param == "versionsuffix":
versuff = selected_val
cnt = len(ecs_and_files)
if not cnt == 1:
fs = [x[1] for x in ecs_and_files]
_log.error("Failed to select a single easyconfig from available ones, %s left: %s" % (cnt, fs))
else:
(selected_ec, selected_ec_file) = ecs_and_files[0]
# check whether selected easyconfig matches requirements
match = True
for (key, val) in specs.items():
if key in selected_ec._config:
# values must be equal to have a full match
if not selected_ec[key] == val:
match = False
else:
# if we encounter a key that is not set in the selected easyconfig, we don't have a full match
match = False
# if it matches, no need to tweak
if match:
_log.info("Perfect match found: %s" % selected_ec_file)
return (False, selected_ec_file)
# GENERATE
# if no file path was specified, generate a file name
if not fp:
installver = det_installversion(ver, tcname, tcver, verpref, versuff)
fp = "%s-%s.eb" % (name, installver)
# generate tweaked easyconfig file
tweak(selected_ec_file, fp, specs)
_log.info("Generated easyconfig file %s, and using it to build the requested software." % fp)
return (True, fp)
def select_or_generate_ec(fp, paths, specs):
"""
Select or generate an easyconfig file with the given requirements, from existing easyconfig files.
If easyconfig files are available for the specified software package,
then this function will first try to determine which toolchain to use.
* if a toolchain is given, it will use it (possible using a template easyconfig file as base);
* if not, and only a single toolchain is available, is will assume it can use that toolchain
* else, it fails -- EasyBuild doesn't select between multiple available toolchains
Next, it will trim down the selected easyconfig files to a single one,
based on the following requirements (in order of preference):
* toolchain version
* software version
* other parameters (e.g. versionprefix, versionsuffix, etc.)
If a complete match is found, it will return that easyconfig.
Else, it will generate a new easyconfig file based on the selected 'best matching' easyconfig file.
"""
specs = copy.deepcopy(specs)
# ensure that at least name is specified
if not specs.get('name'):
_log.error(
"Supplied 'specs' dictionary doesn't even contain a name of a software package?"
)
name = specs['name']
handled_params = ['name']
# find ALL available easyconfig files for specified software
ec_files = []
installver = det_installversion('*', 'dummy', '*', '*', '*')
for path in paths:
patterns = create_paths(path, name, installver)
for pattern in patterns:
ec_files.extend(glob.glob(pattern))
# we need at least one config file to start from
if len(ec_files) == 0:
# look for a template file if no easyconfig for specified software name is available
for path in paths:
templ_file = os.path.join(path, "TEMPLATE.eb")
if os.path.isfile(templ_file):
ec_files = [templ_file]
break
else:
_log.debug("No template found at %s." % templ_file)
if len(ec_files) == 0:
_log.error(
"No easyconfig files found for software %s, and no templates available. I'm all out of ideas."
% name)
# we can't rely on set, because we also need to be able to obtain a list of unique lists
def unique(l):
"""Retain unique elements in a sorted list."""
l = sorted(l)
if len(l) > 1:
l2 = [l[0]]
for x in l:
if not x == l2[-1]:
l2.append(x)
return l2
else:
return l
# filter unique
ec_files = nub(ec_files)
_log.debug("Unique ec_files: %s" % ec_files)
ecs_and_files = [(EasyConfig(f, validate=False), f) for f in ec_files]
# TOOLCHAIN NAME
# determine list of unique toolchain names
tcnames = unique([x[0]['toolchain']['name'] for x in ecs_and_files])
_log.debug("Found %d unique toolchain names: %s" % (len(tcnames), tcnames))
# if a toolchain was selected, and we have no easyconfig files for it, try and use a template
if specs.get('toolchain_name') and not specs['toolchain_name'] in tcnames:
if "TEMPLATE" in tcnames:
_log.info(
"No easyconfig file for specified toolchain, but template is available."
)
else:
_log.error("No easyconfig file for %s with toolchain %s, " \
"and no template available." % (name, specs['toolchain_name']))
tcname = specs.pop('toolchain_name', None)
handled_params.append('toolchain_name')
# trim down list according to selected toolchain
if tcname in tcnames:
# known toolchain, so only retain those
selected_tcname = tcname
else:
if len(tcnames) == 1 and not tcnames[0] == "TEMPLATE":
# only one (non-template) toolchain availble, so use that
tcname = tcnames[0]
selected_tcname = tcname
elif len(tcnames) == 1 and tcnames[0] == "TEMPLATE":
selected_tcname = tcnames[0]
else:
# fall-back: use template toolchain if a toolchain name was specified
if tcname:
selected_tcname = "TEMPLATE"
else:
# if multiple toolchains are available, and none is specified, we quit
# we can't just pick one, how would we prefer one over the other?
_log.error(
"No toolchain name specified, and more than one available: %s."
% tcnames)
_log.debug("Filtering easyconfigs based on toolchain name '%s'..." %
selected_tcname)
ecs_and_files = [
x for x in ecs_and_files
if x[0]['toolchain']['name'] == selected_tcname
]
_log.debug("Filtered easyconfigs: %s" % [x[1] for x in ecs_and_files])
# TOOLCHAIN VERSION
tcvers = unique([x[0]['toolchain']['version'] for x in ecs_and_files])
_log.debug("Found %d unique toolchain versions: %s" %
(len(tcvers), tcvers))
tcver = specs.pop('toolchain_version', None)
handled_params.append('toolchain_version')
(tcver, selected_tcver) = pick_version(tcver, tcvers)
_log.debug("Filtering easyconfigs based on toolchain version '%s'..." %
selected_tcver)
ecs_and_files = [
x for x in ecs_and_files
if x[0]['toolchain']['version'] == selected_tcver
]
_log.debug("Filtered easyconfigs: %s" % [x[1] for x in ecs_and_files])
# add full toolchain specification to specs
if tcname and tcver:
specs.update({'toolchain': {'name': tcname, 'version': tcver}})
handled_params.append('toolchain')
else:
if tcname:
specs.update({'toolchain_name': tcname})
if tcver:
specs.update({'toolchain_version': tcver})
# SOFTWARE VERSION
vers = unique([x[0]['version'] for x in ecs_and_files])
_log.debug("Found %d unique software versions: %s" % (len(vers), vers))
ver = specs.pop('version', None)
handled_params.append('version')
(ver, selected_ver) = pick_version(ver, vers)
if ver:
specs.update({'version': ver})
_log.debug("Filtering easyconfigs based on software version '%s'..." %
selected_ver)
ecs_and_files = [
x for x in ecs_and_files if x[0]['version'] == selected_ver
]
_log.debug("Filtered easyconfigs: %s" % [x[1] for x in ecs_and_files])
# go through parameters specified via --amend
# always include versionprefix/suffix, because we might need it to generate a file name
verpref = None
versuff = None
other_params = {'versionprefix': None, 'versionsuffix': None}
for (param, val) in specs.items():
if not param in handled_params:
other_params.update({param: val})
_log.debug(
"Filtering based on other parameters (specified via --amend): %s" %
other_params)
for (param, val) in other_params.items():
if param in ecs_and_files[0][0]._config:
vals = unique([x[0][param] for x in ecs_and_files])
else:
vals = []
filter_ecs = False
# try and select a value from the available ones, or fail if we can't
if val in vals:
# if the specified value is available, use it
selected_val = val
_log.debug("Specified %s is available, so using it: %s" %
(param, selected_val))
filter_ecs = True
elif val:
# if a value is specified, use that, even if it's not available yet
selected_val = val
_log.debug(
"%s is specified, so using it (even though it's not available yet): %s"
% (param, selected_val))
elif len(vals) == 1:
# if only one value is available, use that
selected_val = vals[0]
_log.debug("Only one %s available ('%s'), so picking that" %
(param, selected_val))
filter_ecs = True
else:
# otherwise, we fail, because we don't know how to pick between different fixes
_log.error(
"No %s specified, and can't pick from available %ses %s" %
(param, param, vals))
if filter_ecs:
_log.debug("Filtering easyconfigs based on %s '%s'..." %
(param, selected_val))
ecs_and_files = [
x for x in ecs_and_files if x[0][param] == selected_val
]
_log.debug("Filtered easyconfigs: %s" %
[x[1] for x in ecs_and_files])
# keep track of versionprefix/suffix
if param == "versionprefix":
verpref = selected_val
elif param == "versionsuffix":
versuff = selected_val
cnt = len(ecs_and_files)
if not cnt == 1:
fs = [x[1] for x in ecs_and_files]
_log.error(
"Failed to select a single easyconfig from available ones, %s left: %s"
% (cnt, fs))
else:
(selected_ec, selected_ec_file) = ecs_and_files[0]
# check whether selected easyconfig matches requirements
match = True
for (key, val) in specs.items():
if key in selected_ec._config:
# values must be equal to have a full match
if not selected_ec[key] == val:
match = False
else:
# if we encounter a key that is not set in the selected easyconfig, we don't have a full match
match = False
# if it matches, no need to tweak
if match:
_log.info("Perfect match found: %s" % selected_ec_file)
return (False, selected_ec_file)
# GENERATE
# if no file path was specified, generate a file name
if not fp:
installver = det_installversion(ver, tcname, tcver, verpref,
versuff)
fp = "%s-%s.eb" % (name, installver)
# generate tweaked easyconfig file
tweak(selected_ec_file, fp, specs)
_log.info(
"Generated easyconfig file %s, and using it to build the requested software."
% fp)
return (True, fp)
