def test_installversion(self):
"""Test generation of install version."""
ver = "3.14"
verpref = "myprefix|"
versuff = "|mysuffix"
tcname = "GCC"
tcver = "4.6.3"
dummy = "dummy"
correct_installver = "%s%s-%s-%s%s" % (verpref, ver, tcname, tcver, versuff)
cfg = {
'version': ver,
'toolchain': {'name': tcname, 'version': tcver},
'versionprefix': verpref,
'versionsuffix': versuff,
}
installver = det_full_ec_version(cfg)
self.assertEqual(installver, "%s%s-%s-%s%s" % (verpref, ver, tcname, tcver, versuff))
correct_installver = "%s%s%s" % (verpref, ver, versuff)
cfg = {
'version': ver,
'toolchain': {'name': dummy, 'version': tcver},
'versionprefix': verpref,
'versionsuffix': versuff,
}
installver = det_full_ec_version(cfg)
self.assertEqual(installver, correct_installver)
def test_dependency(self):
""" test all possible ways of specifying dependencies """
self.contents = '\n'.join([
'name = "pi"',
'version = "3.14"',
'homepage = "http://google.com"',
'description = "test easyconfig"',
'toolchain = {"name":"GCC", "version": "4.6.3"}',
'dependencies = [("first", "1.1"), {"name": "second", "version": "2.2"}]',
'builddependencies = [("first", "1.1"), {"name": "second", "version": "2.2"}]',
])
self.prep()
eb = EasyConfig(self.eb_file, valid_stops=self.all_stops)
# should include builddependencies
self.assertEqual(len(eb.dependencies()), 4)
self.assertEqual(len(eb.builddependencies()), 2)
first = eb.dependencies()[0]
second = eb.dependencies()[1]
self.assertEqual(first['name'], "first")
self.assertEqual(second['name'], "second")
self.assertEqual(first['version'], "1.1")
self.assertEqual(second['version'], "2.2")
self.assertEqual(det_full_ec_version(first), '1.1-GCC-4.6.3')
self.assertEqual(det_full_ec_version(second), '2.2-GCC-4.6.3')
# same tests for builddependencies
first = eb.builddependencies()[0]
second = eb.builddependencies()[1]
self.assertEqual(first['name'], "first")
self.assertEqual(second['name'], "second")
self.assertEqual(first['version'], "1.1")
self.assertEqual(second['version'], "2.2")
self.assertEqual(det_full_ec_version(first), '1.1-GCC-4.6.3')
self.assertEqual(det_full_ec_version(second), '2.2-GCC-4.6.3')
eb['dependencies'] = ["wrong type"]
self.assertErrorRegex(EasyBuildError,
"wrong type from unsupported type",
eb.dependencies)
eb['dependencies'] = [()]
self.assertErrorRegex(EasyBuildError, "without name", eb.dependencies)
eb['dependencies'] = [{'name': "test"}]
self.assertErrorRegex(EasyBuildError, "without version",
eb.dependencies)
def test_dependency(self):
""" test all possible ways of specifying dependencies """
self.contents = '\n'.join([
'easyblock = "ConfigureMake"',
'name = "pi"',
'version = "3.14"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
'toolchain = {"name":"GCC", "version": "4.6.3"}',
'dependencies = [("first", "1.1"), {"name": "second", "version": "2.2"}]',
'builddependencies = [("first", "1.1"), {"name": "second", "version": "2.2"}]',
])
self.prep()
eb = EasyConfig(self.eb_file)
# should include builddependencies
self.assertEqual(len(eb.dependencies()), 4)
self.assertEqual(len(eb.builddependencies()), 2)
first = eb.dependencies()[0]
second = eb.dependencies()[1]
self.assertEqual(first['name'], "first")
self.assertEqual(second['name'], "second")
self.assertEqual(first['version'], "1.1")
self.assertEqual(second['version'], "2.2")
self.assertEqual(det_full_ec_version(first), '1.1-GCC-4.6.3')
self.assertEqual(det_full_ec_version(second), '2.2-GCC-4.6.3')
# same tests for builddependencies
first = eb.builddependencies()[0]
second = eb.builddependencies()[1]
self.assertEqual(first['name'], "first")
self.assertEqual(second['name'], "second")
self.assertEqual(first['version'], "1.1")
self.assertEqual(second['version'], "2.2")
self.assertEqual(det_full_ec_version(first), '1.1-GCC-4.6.3')
self.assertEqual(det_full_ec_version(second), '2.2-GCC-4.6.3')
self.assertErrorRegex(EasyBuildError, "Dependency foo of unsupported type", eb._parse_dependency, "foo")
self.assertErrorRegex(EasyBuildError, "without name", eb._parse_dependency, ())
self.assertErrorRegex(EasyBuildError, "without version", eb._parse_dependency, {'name': 'test'})
err_msg = "Incorrect external dependency specification"
self.assertErrorRegex(EasyBuildError, err_msg, eb._parse_dependency, (EXTERNAL_MODULE_MARKER,))
self.assertErrorRegex(EasyBuildError, err_msg, eb._parse_dependency, ('foo', '1.2.3', EXTERNAL_MODULE_MARKER))
def test_dependency(self):
""" test all possible ways of specifying dependencies """
self.contents = '\n'.join([
'name = "pi"',
'version = "3.14"',
'homepage = "http://google.com"',
'description = "test easyconfig"',
'toolchain = {"name":"GCC", "version": "4.6.3"}',
'dependencies = [("first", "1.1"), {"name": "second", "version": "2.2"}]',
'builddependencies = [("first", "1.1"), {"name": "second", "version": "2.2"}]',
])
self.prep()
eb = EasyConfig(self.eb_file, valid_stops=self.all_stops)
# should include builddependencies
self.assertEqual(len(eb.dependencies()), 4)
self.assertEqual(len(eb.builddependencies()), 2)
first = eb.dependencies()[0]
second = eb.dependencies()[1]
self.assertEqual(first['name'], "first")
self.assertEqual(second['name'], "second")
self.assertEqual(first['version'], "1.1")
self.assertEqual(second['version'], "2.2")
self.assertEqual(det_full_ec_version(first), '1.1-GCC-4.6.3')
self.assertEqual(det_full_ec_version(second), '2.2-GCC-4.6.3')
# same tests for builddependencies
first = eb.builddependencies()[0]
second = eb.builddependencies()[1]
self.assertEqual(first['name'], "first")
self.assertEqual(second['name'], "second")
self.assertEqual(first['version'], "1.1")
self.assertEqual(second['version'], "2.2")
self.assertEqual(det_full_ec_version(first), '1.1-GCC-4.6.3')
self.assertEqual(det_full_ec_version(second), '2.2-GCC-4.6.3')
eb['dependencies'] = ["wrong type"]
self.assertErrorRegex(EasyBuildError, "wrong type from unsupported type", eb.dependencies)
eb['dependencies'] = [()]
self.assertErrorRegex(EasyBuildError, "without name", eb.dependencies)
eb['dependencies'] = [{'name': "test"}]
self.assertErrorRegex(EasyBuildError, "without version", eb.dependencies)
def test_validation(self):
""" test other validations beside mandatory variables """
self.contents = '\n'.join([
'name = "pi"',
'version = "3.14"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
'toolchain = {"name":"dummy", "version": "dummy"}',
'stop = "notvalid"',
])
self.prep()
ec = EasyConfig(self.eb_file, validate=False)
self.assertErrorRegex(EasyBuildError, r"\w* provided '\w*' is not valid", ec.validate)
ec['stop'] = 'patch'
# this should now not crash
ec.validate()
ec['osdependencies'] = ['non-existent-dep']
self.assertErrorRegex(EasyBuildError, "OS dependencies were not found", ec.validate)
# dummy toolchain, installversion == version
self.assertEqual(det_full_ec_version(ec), "3.14")
os.chmod(self.eb_file, 0000)
self.assertErrorRegex(EasyBuildError, "Permission denied", EasyConfig, self.eb_file)
os.chmod(self.eb_file, 0755)
self.contents += "\nsyntax_error'"
self.prep()
self.assertErrorRegex(EasyBuildError, "SyntaxError", EasyConfig, self.eb_file)
def robot_find_minimal_easyconfig_for_dependency(dep):
"""
Find an easyconfig with minimal toolchain for a dependency
"""
newdep = copy.deepcopy(dep)
toolchain_hierarchy = get_toolchain_hierarchy(dep['toolchain'])
res = None
# reversed search: start with subtoolchains first, i.e. first (dummy or) compiler-only toolchain, etc.
for toolchain in toolchain_hierarchy:
newdep['toolchain'] = toolchain
eb_file = robot_find_easyconfig(newdep['name'],
det_full_ec_version(newdep))
if eb_file is not None:
if newdep['toolchain'] != dep['toolchain']:
_log.info(
"Minimally resolving dependency %s using toolchain %s with %s",
dep, toolchain, eb_file)
res = (newdep, eb_file)
break
if res is None:
_log.debug("Irresolvable minimal dependency found: %s", dep)
return res
def det_toolchain_element_details(tc, elem):
"""
Determine details of a particular toolchain element, for a given Toolchain instance.
"""
# check for cached version first
tc_dict = tc.as_dict()
key = (tc_dict['name'], tc_dict['version'] + tc_dict['versionsuffix'], elem)
if key in _toolchain_details_cache:
_log.debug("Obtained details for '%s' in toolchain '%s' from cache" % (elem, tc_dict))
return _toolchain_details_cache[key]
# grab version from parsed easyconfig file for toolchain
eb_file = robot_find_easyconfig(tc_dict['name'], det_full_ec_version(tc_dict))
tc_ec = process_easyconfig(eb_file, parse_only=True)
if len(tc_ec) > 1:
_log.warning("More than one toolchain specification found for %s, only retaining first" % tc_dict)
_log.debug("Full list of toolchain specifications: %s" % tc_ec)
tc_ec = tc_ec[0]['ec']
tc_deps = tc_ec['dependencies']
tc_elem_details = None
for tc_dep in tc_deps:
if tc_dep['name'] == elem:
tc_elem_details = tc_dep
_log.debug("Found details for toolchain element %s: %s" % (elem, tc_elem_details))
break
if tc_elem_details is None:
# for compiler-only toolchains, toolchain and compilers are one-and-the-same
if tc_ec['name'] == elem:
tc_elem_details = tc_ec
else:
raise EasyBuildError("No toolchain element '%s' found for toolchain %s: %s", elem, tc.as_dict(), tc_ec)
_toolchain_details_cache[key] = tc_elem_details
_log.debug("Obtained details for '%s' in toolchain '%s', added to cache" % (elem, tc_dict))
return _toolchain_details_cache[key]
def det_toolchain_element_details(tc, elem):
"""
Determine details of a particular toolchain element, for a given Toolchain instance.
"""
# check for cached version first
tc_dict = tc.as_dict()
key = (tc_dict['name'], tc_dict['version'] + tc_dict['versionsuffix'], elem)
if key in _toolchain_details_cache:
_log.debug("Obtained details for '%s' in toolchain '%s' from cache" % (elem, tc_dict))
return _toolchain_details_cache[key]
# grab version from parsed easyconfig file for toolchain
eb_file = robot_find_easyconfig(tc_dict['name'], det_full_ec_version(tc_dict))
tc_ec = process_easyconfig(eb_file, parse_only=True)
if len(tc_ec) > 1:
_log.warning("More than one toolchain specification found for %s, only retaining first" % tc_dict)
_log.debug("Full list of toolchain specifications: %s" % tc_ec)
tc_ec = tc_ec[0]['ec']
tc_deps = tc_ec['dependencies']
tc_elem_details = None
for tc_dep in tc_deps:
if tc_dep['name'] == elem:
tc_elem_details = tc_dep
_log.debug("Found details for toolchain element %s: %s" % (elem, tc_elem_details))
break
if tc_elem_details is None:
# for compiler-only toolchains, toolchain and compilers are one-and-the-same
if tc_ec['name'] == elem:
tc_elem_details = tc_ec
else:
_log.error("No toolchain element '%s' found for toolchain %s: %s" % (elem, tc.as_dict(), tc_ec))
_toolchain_details_cache[key] = tc_elem_details
_log.debug("Obtained details for '%s' in toolchain '%s', added to cache" % (elem, tc_dict))
return _toolchain_details_cache[key]
def det_full_module_name(ec, eb_ns=False):
"""
Determine full module name following the currently active module naming scheme.
First try to pass 'parsed' easyconfig as supplied,
try and find a matching easyconfig file, parse it and supply it in case of a KeyError.
"""
try:
mod_name = _det_full_module_name(ec, eb_ns=eb_ns)
except KeyError, err:
_log.debug("KeyError '%s' when determining module name for %s, trying fallback procedure..." % (err, ec))
# for dependencies, only name/version/versionsuffix/toolchain easyconfig parameters are available;
# when a key error occurs, try and find an easyconfig file to parse via the robot,
# and retry with the parsed easyconfig file (which will contains a full set of keys)
robot = build_option('robot_path')
eb_file = robot_find_easyconfig(robot, ec['name'], det_full_ec_version(ec))
if eb_file is None:
_log.error("Failed to find an easyconfig file when determining module name for: %s" % ec)
else:
parsed_ec = process_easyconfig(eb_file)
if len(parsed_ec) > 1:
_log.warning("More than one parsed easyconfig obtained from %s, only retaining first" % eb_file)
try:
mod_name = _det_full_module_name(parsed_ec[0]['ec'], eb_ns=eb_ns)
except KeyError, err:
_log.error("A KeyError '%s' occured when determining a module name for %s." % parsed_ec['ec'])
def create_job(build_command,
easyconfig,
output_dir=None,
conn=None,
ppn=None):
"""
Creates a job, to build a *single* easyconfig
@param build_command: format string for command, full path to an easyconfig file will be substituted in it
@param easyconfig: easyconfig as processed by process_easyconfig
@param output_dir: optional output path; --regtest-output-dir will be used inside the job with this variable
@param conn: open connection to PBS server
@param ppn: ppn setting to use (# 'processors' (cores) per node to use)
returns the job
"""
if output_dir is None:
output_dir = 'easybuild-build'
# capture PYTHONPATH, MODULEPATH and all variables starting with EASYBUILD
easybuild_vars = {}
for name in os.environ:
if name.startswith("EASYBUILD"):
easybuild_vars[name] = os.environ[name]
others = ["PYTHONPATH", "MODULEPATH"]
for env_var in others:
if env_var in os.environ:
easybuild_vars[env_var] = os.environ[env_var]
_log.info("Dictionary of environment variables passed to job: %s" %
easybuild_vars)
# obtain unique name based on name/easyconfig version tuple
ec_tuple = (easyconfig['ec']['name'],
det_full_ec_version(easyconfig['ec']))
name = '-'.join(ec_tuple)
# create command based on build_command template
command = build_command % {
'spec': easyconfig['spec'],
'output_dir': os.path.join(os.path.abspath(output_dir), name),
}
# just use latest build stats
repo = init_repository(get_repository(), get_repositorypath())
buildstats = repo.get_buildstats(*ec_tuple)
resources = {}
if buildstats:
previous_time = buildstats[-1]['build_time']
resources['hours'] = int(math.ceil(previous_time * 2 / 60))
job = PbsJob(command,
name,
easybuild_vars,
resources=resources,
conn=conn,
ppn=ppn)
job.module = easyconfig['ec'].full_mod_name
return job
def det_full_module_name(self, ec):
"""
Determine full module name from given easyconfig, according to the EasyBuild module naming scheme.
:param ec: dict-like object with easyconfig parameter values (e.g. 'name', 'version', etc.)
:return: string with full module name <name>/<installversion>, e.g.: 'gzip/1.5-goolf-1.4.10'
"""
return os.path.join(ec['name'], det_full_ec_version(ec)).lower()
def obtain_ec_for(specs, paths, fp):
"""
Obtain an easyconfig file to the given specifications.
Either select between available ones, or use the best suited available one
to generate a new easyconfig file.
@param specs: list of available easyconfig files
@param paths: a list of paths where easyconfig files can be found
@param fp: the desired file name
"""
# ensure that at least name is specified
if not specs.get('name'):
_log.error(
"Supplied 'specs' dictionary doesn't even contain a name of a software package?"
)
# collect paths to search in
if not paths:
_log.error(
"No paths to look for easyconfig files, specify a path with --robot."
)
# create glob patterns based on supplied info
# figure out the install version
cfg = {
'version': specs.get('version', '*'),
'toolchain': {
'name': specs.get('toolchain_name', '*'),
'version': specs.get('toolchain_version', '*'),
},
'versionprefix': specs.get('versionprefix', '*'),
'versionsuffix': specs.get('versionsuffix', '*'),
}
installver = det_full_ec_version(cfg)
# find easyconfigs that match a pattern
easyconfig_files = []
for path in paths:
patterns = create_paths(path, specs['name'], installver)
for pattern in patterns:
easyconfig_files.extend(glob.glob(pattern))
cnt = len(easyconfig_files)
_log.debug("List of obtained easyconfig files (%d): %s" %
(cnt, easyconfig_files))
# select best easyconfig, or try to generate one that fits the requirements
res = select_or_generate_ec(fp, paths, specs)
if res:
return res
else:
_log.error(
"No easyconfig found for requested software, and also failed to generate one."
)
def create_job(job_backend,
build_command,
easyconfig,
output_dir='easybuild-build'):
"""
Creates a job to build a *single* easyconfig.
:param job_backend: A factory object for querying server parameters and creating actual job objects
:param build_command: format string for command, full path to an easyconfig file will be substituted in it
:param easyconfig: easyconfig as processed by process_easyconfig
:param output_dir: optional output path; --regtest-output-dir will be used inside the job with this variable
returns the job
"""
# capture PYTHONPATH, MODULEPATH and all variables starting with EASYBUILD
easybuild_vars = {}
for name in os.environ:
if name.startswith("EASYBUILD"):
easybuild_vars[name] = os.environ[name]
for env_var in ["PYTHONPATH", "MODULEPATH"]:
if env_var in os.environ:
easybuild_vars[env_var] = os.environ[env_var]
_log.info("Dictionary of environment variables passed to job: %s" %
easybuild_vars)
# obtain unique name based on name/easyconfig version tuple
ec_tuple = (easyconfig['ec']['name'],
det_full_ec_version(easyconfig['ec']))
name = '-'.join(ec_tuple)
# determine whether additional options need to be passed to the 'eb' command
add_opts = ''
if easyconfig['hidden']:
add_opts += ' --hidden'
# create command based on build_command template
command = build_command % {
'add_opts': add_opts,
'output_dir': os.path.join(os.path.abspath(output_dir), name),
'spec': easyconfig['spec'],
}
# just use latest build stats
repo = init_repository(get_repository(), get_repositorypath())
buildstats = repo.get_buildstats(*ec_tuple)
extra = {}
if buildstats:
previous_time = buildstats[-1]['build_time']
extra['hours'] = int(math.ceil(previous_time * 2 / 60))
if build_option('job_cores'):
extra['cores'] = build_option('job_cores')
job = job_backend.make_job(command, name, easybuild_vars, **extra)
job.module = easyconfig['ec'].full_mod_name
return job
def det_full_module_name(self, ec):
"""
Determine full module name from given easyconfig, according to the EasyBuild module naming scheme.
:param ec: dict-like object with easyconfig parameter values (e.g. 'name', 'version', etc.)
:return: string with full module name <name>/<installversion>, e.g.: 'gzip/1.5-goolf-1.4.10'
"""
return os.path.join(ec['name'], det_full_ec_version(ec))
def test_dependency(self):
""" test all possible ways of specifying dependencies """
self.contents = '\n'.join([
'easyblock = "ConfigureMake"',
'name = "pi"',
'version = "3.14"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
'toolchain = {"name":"GCC", "version": "4.6.3"}',
'dependencies = [("first", "1.1"), {"name": "second", "version": "2.2"}]',
'builddependencies = [("first", "1.1"), {"name": "second", "version": "2.2"}]',
])
self.prep()
eb = EasyConfig(self.eb_file)
# should include builddependencies
self.assertEqual(len(eb.dependencies()), 4)
self.assertEqual(len(eb.builddependencies()), 2)
first = eb.dependencies()[0]
second = eb.dependencies()[1]
self.assertEqual(first['name'], "first")
self.assertEqual(second['name'], "second")
self.assertEqual(first['version'], "1.1")
self.assertEqual(second['version'], "2.2")
self.assertEqual(det_full_ec_version(first), '1.1-GCC-4.6.3')
self.assertEqual(det_full_ec_version(second), '2.2-GCC-4.6.3')
# same tests for builddependencies
first = eb.builddependencies()[0]
second = eb.builddependencies()[1]
self.assertEqual(first['name'], "first")
self.assertEqual(second['name'], "second")
self.assertEqual(first['version'], "1.1")
self.assertEqual(second['version'], "2.2")
self.assertEqual(det_full_ec_version(first), '1.1-GCC-4.6.3')
self.assertEqual(det_full_ec_version(second), '2.2-GCC-4.6.3')
self.assertErrorRegex(EasyBuildError, "Dependency foo of unsupported type", eb._parse_dependency, "foo")
self.assertErrorRegex(EasyBuildError, "without name", eb._parse_dependency, ())
self.assertErrorRegex(EasyBuildError, "without version", eb._parse_dependency, {'name': 'test'})
def det_full_module_name(self, ec):
"""
Determine full module name from given easyconfig, according to the thematic module naming scheme.
:param ec: dict-like object with easyconfig parameter values (e.g. 'name', 'version', etc.)
:return: string representing full module name, e.g.: 'biology/ABySS/1.3.4-goolf-1.4.10'
"""
return os.path.join(ec['moduleclass'], ec['name'], det_full_ec_version(ec))
def det_full_module_name(self, ec):
"""
Determine full module name from given easyconfig, according to the thematic module naming scheme.
:param ec: dict-like object with easyconfig parameter values (e.g. 'name', 'version', etc.)
:return: string representing full module name, e.g.: 'biology/ABySS/1.3.4-goolf-1.4.10'
"""
return os.path.join(ec['moduleclass'], ec['name'],
det_full_ec_version(ec))
def ec_filename_for(path):
"""
Return a suiting file name for the easyconfig file at <path>,
as determined by its contents.
"""
ec = EasyConfig(path, validate=False)
fn = "%s-%s.eb" % (ec['name'], det_full_ec_version(ec))
return fn
def ec_filename_for(path):
"""
Return a suiting file name for the easyconfig file at <path>,
as determined by its contents.
"""
ec = EasyConfig(path, validate=False)
fn = "%s-%s.eb" % (ec['name'], det_full_ec_version(ec))
return fn
def ec_filename_for(path):
"""
Return a suiting file name for the easyconfig file at <path>,
as determined by its contents.
"""
ec = EasyConfig(path, build_options={"validate": False})
fn = "%s-%s.eb" % (ec["name"], det_full_ec_version(ec))
return fn
def check_ec_type(self, ec):
"""
Obtain a full parsed easyconfig file to pass to naming scheme methods if provided keys are insufficient.
"""
if not isinstance(ec, EasyConfig) and self.requires_full_easyconfig(ec.keys()):
self.log.debug("A parsed easyconfig is required by the module naming scheme, so finding one for %s" % ec)
# fetch/parse easyconfig file if deemed necessary
eb_file = robot_find_easyconfig(ec['name'], det_full_ec_version(ec))
if eb_file is not None:
parsed_ec = process_easyconfig(eb_file, parse_only=True, hidden=ec['hidden'])
if len(parsed_ec) > 1:
self.log.warning("More than one parsed easyconfig obtained from %s, only retaining first" % eb_file)
self.log.debug("Full list of parsed easyconfigs: %s" % parsed_ec)
ec = parsed_ec[0]['ec']
else:
tup = (ec['name'], det_full_ec_version(ec), ec)
self.log.error("Failed to find easyconfig file '%s-%s.eb' when determining module name for: %s" % tup)
return ec
def map_easyconfig_to_target_tc_hierarchy(ec_spec,
toolchain_mapping,
targetdir=None):
"""
Take an easyconfig spec, parse it, map it to a target toolchain and dump it out
:param ec_spec: Location of original easyconfig file
:param toolchain_mapping: Mapping between source toolchain and target toolchain
:param targetdir: Directory to dump the modified easyconfig file in
:return: Location of the modified easyconfig file
"""
# Fully parse the original easyconfig
parsed_ec = process_easyconfig(ec_spec, validate=False)[0]
# Replace the toolchain if the mapping exists
tc_name = parsed_ec['ec']['toolchain']['name']
if tc_name in toolchain_mapping:
new_toolchain = toolchain_mapping[tc_name]
_log.debug("Replacing parent toolchain %s with %s",
parsed_ec['ec']['toolchain'], new_toolchain)
parsed_ec['ec']['toolchain'] = new_toolchain
# Replace the toolchains of all the dependencies
for key in DEPENDENCY_PARAMETERS:
# loop over a *copy* of dependency dicts (with resolved templates);
# to update the original dep dict, we need to index with idx into self._config[key][0]...
for idx, dep in enumerate(parsed_ec['ec'][key]):
# reference to original dep dict, this is the one we should be updating
orig_dep = parsed_ec['ec']._config[key][0][idx]
# skip dependencies that are marked as external modules
if dep['external_module']:
continue
dep_tc_name = dep['toolchain']['name']
if dep_tc_name in toolchain_mapping:
orig_dep['toolchain'] = toolchain_mapping[dep_tc_name]
# Replace the binutils version (if necessary)
if 'binutils' in toolchain_mapping and (
dep['name'] == 'binutils' and dep_tc_name == GCCcore.NAME):
orig_dep.update(toolchain_mapping['binutils'])
# set module names
orig_dep['short_mod_name'] = ActiveMNS().det_short_module_name(
dep)
orig_dep['full_mod_name'] = ActiveMNS().det_full_module_name(
dep)
# Determine the name of the modified easyconfig and dump it to target_dir
ec_filename = '%s-%s.eb' % (parsed_ec['ec']['name'],
det_full_ec_version(parsed_ec['ec']))
tweaked_spec = os.path.join(targetdir or tempfile.gettempdir(),
ec_filename)
parsed_ec['ec'].dump(tweaked_spec, always_overwrite=False, backup=True)
_log.debug("Dumped easyconfig tweaked via --try-toolchain* to %s",
tweaked_spec)
return tweaked_spec
def create_job(job_backend, build_command, easyconfig, output_dir='easybuild-build'):
"""
Creates a job to build a *single* easyconfig.
:param job_backend: A factory object for querying server parameters and creating actual job objects
:param build_command: format string for command, full path to an easyconfig file will be substituted in it
:param easyconfig: easyconfig as processed by process_easyconfig
:param output_dir: optional output path; --regtest-output-dir will be used inside the job with this variable
returns the job
"""
# capture PYTHONPATH, MODULEPATH and all variables starting with EASYBUILD
easybuild_vars = {}
for name in os.environ:
if name.startswith("EASYBUILD"):
easybuild_vars[name] = os.environ[name]
for env_var in ["PYTHONPATH", "MODULEPATH"]:
if env_var in os.environ:
easybuild_vars[env_var] = os.environ[env_var]
_log.info("Dictionary of environment variables passed to job: %s" % easybuild_vars)
# obtain unique name based on name/easyconfig version tuple
ec_tuple = (easyconfig['ec']['name'], det_full_ec_version(easyconfig['ec']))
name = '-'.join(ec_tuple)
# determine whether additional options need to be passed to the 'eb' command
add_opts = ''
if easyconfig['hidden']:
add_opts += ' --hidden'
# create command based on build_command template
command = build_command % {
'add_opts': add_opts,
'output_dir': os.path.join(os.path.abspath(output_dir), name),
'spec': easyconfig['spec'],
}
# just use latest build stats
repo = init_repository(get_repository(), get_repositorypath())
buildstats = repo.get_buildstats(*ec_tuple)
extra = {}
if buildstats:
previous_time = buildstats[-1]['build_time']
extra['hours'] = int(math.ceil(previous_time * 2 / 60))
if build_option('job_cores'):
extra['cores'] = build_option('job_cores')
job = job_backend.make_job(command, name, easybuild_vars, **extra)
job.module = easyconfig['ec'].full_mod_name
return job
def det_installversion(version, toolchain_name, toolchain_version, prefix, suffix):
"""Deprecated 'det_installversion' function, to determine exact install version, based on supplied parameters."""
old_fn = 'framework.easyconfig.easyconfig.det_installversion'
_log.deprecated('Use module_generator.det_full_ec_version instead of %s' % old_fn, '2.0')
cfg = {
'version': version,
'toolchain': {'name': toolchain_name, 'version': toolchain_version},
'versionprefix': prefix,
'versionsuffix': suffix,
}
return det_full_ec_version(cfg)
def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir=None):
"""
Take an easyconfig spec, parse it, map it to a target toolchain and dump it out
:param ec_spec: Location of original easyconfig file
:param toolchain_mapping: Mapping between source toolchain and target toolchain
:param targetdir: Directory to dump the modified easyconfig file in
:return: Location of the modified easyconfig file
"""
# Fully parse the original easyconfig
parsed_ec = process_easyconfig(ec_spec, validate=False)[0]['ec']
# Replace the toolchain if the mapping exists
tc_name = parsed_ec['toolchain']['name']
if tc_name in toolchain_mapping:
new_toolchain = toolchain_mapping[tc_name]
_log.debug("Replacing parent toolchain %s with %s", parsed_ec['toolchain'], new_toolchain)
parsed_ec['toolchain'] = new_toolchain
# Replace the toolchains of all the dependencies
for key in DEPENDENCY_PARAMETERS:
# loop over a *copy* of dependency dicts (with resolved templates);
# to update the original dep dict, we need to get a reference with templating disabled...
val = parsed_ec[key]
orig_val = parsed_ec.get_ref(key)
if key in parsed_ec.iterate_options:
val = flatten(val)
orig_val = flatten(orig_val)
for idx, dep in enumerate(val):
# reference to original dep dict, this is the one we should be updating
orig_dep = orig_val[idx]
# skip dependencies that are marked as external modules
if dep['external_module']:
continue
dep_tc_name = dep['toolchain']['name']
if dep_tc_name in toolchain_mapping:
orig_dep['toolchain'] = toolchain_mapping[dep_tc_name]
# Replace the binutils version (if necessary)
if 'binutils' in toolchain_mapping and (dep['name'] == 'binutils' and dep_tc_name == GCCcore.NAME):
orig_dep.update(toolchain_mapping['binutils'])
# set module names
orig_dep['short_mod_name'] = ActiveMNS().det_short_module_name(dep)
orig_dep['full_mod_name'] = ActiveMNS().det_full_module_name(dep)
# Determine the name of the modified easyconfig and dump it to target_dir
ec_filename = '%s-%s.eb' % (parsed_ec['name'], det_full_ec_version(parsed_ec))
tweaked_spec = os.path.join(targetdir or tempfile.gettempdir(), ec_filename)
parsed_ec.dump(tweaked_spec, always_overwrite=False, backup=True)
_log.debug("Dumped easyconfig tweaked via --try-toolchain* to %s", tweaked_spec)
return tweaked_spec
def compose_ec_filenames(self):
"""Returns a list of all easyconfig names"""
ec_filenames = []
for sw in self.software_list:
full_ec_version = det_full_ec_version({
'toolchain': {'name': sw.toolchain_name, 'version': sw.toolchain_version},
'version': sw.version,
'versionsuffix': sw.versionsuffix,
})
ec_filename = '%s-%s.eb' % (sw.name, full_ec_version)
ec_filenames.append(ec_filename)
return ec_filenames
def obtain_ec_for(specs, paths, fp):
"""
Obtain an easyconfig file to the given specifications.
Either select between available ones, or use the best suited available one
to generate a new easyconfig file.
@param specs: list of available easyconfig files
@param paths: a list of paths where easyconfig files can be found
@param fp: the desired file name
"""
# ensure that at least name is specified
if not specs.get('name'):
_log.error("Supplied 'specs' dictionary doesn't even contain a name of a software package?")
# collect paths to search in
if not paths:
_log.error("No paths to look for easyconfig files, specify a path with --robot.")
# create glob patterns based on supplied info
# figure out the install version
cfg = {
'version': specs.get('version', '*'),
'toolchain': {
'name': specs.get('toolchain_name', '*'),
'version': specs.get('toolchain_version', '*'),
},
'versionprefix': specs.get('versionprefix', '*'),
'versionsuffix': specs.get('versionsuffix', '*'),
}
installver = det_full_ec_version(cfg)
# find easyconfigs that match a pattern
easyconfig_files = []
for path in paths:
patterns = create_paths(path, specs['name'], installver)
for pattern in patterns:
easyconfig_files.extend(glob.glob(pattern))
cnt = len(easyconfig_files)
_log.debug("List of obtained easyconfig files (%d): %s" % (cnt, easyconfig_files))
# select best easyconfig, or try to generate one that fits the requirements
res = select_or_generate_ec(fp, paths, specs)
if res:
return res
else:
_log.error("No easyconfig found for requested software, and also failed to generate one.")
def create_job(build_command, easyconfig, output_dir="", conn=None, ppn=None):
"""
Creates a job, to build a *single* easyconfig
build_command is a format string in which a full path to an eb file will be substituted
easyconfig should be in the format as processEasyConfig returns them
output_dir is an optional path. EASYBUILDTESTOUTPUT will be set inside the job with this variable
returns the job
"""
# create command based on build_command template
command = build_command % {'spec': easyconfig['spec']}
# capture PYTHONPATH, MODULEPATH and all variables starting with EASYBUILD
easybuild_vars = {}
for name in os.environ:
if name.startswith("EASYBUILD"):
easybuild_vars[name] = os.environ[name]
others = ["PYTHONPATH", "MODULEPATH"]
for env_var in others:
if env_var in os.environ:
easybuild_vars[env_var] = os.environ[env_var]
_log.info("Dictionary of environment variables passed to job: %s" %
easybuild_vars)
# obtain unique name based on name/easyconfig version tuple
ec_tuple = (easyconfig['ec']['name'],
det_full_ec_version(easyconfig['ec']))
name = '-'.join(ec_tuple)
var = config.oldstyle_environment_variables['test_output_path']
easybuild_vars[var] = os.path.join(os.path.abspath(output_dir), name)
# just use latest build stats
repo = init_repository(get_repository(), get_repositorypath())
buildstats = repo.get_buildstats(*ec_tuple)
resources = {}
if buildstats:
previous_time = buildstats[-1]['build_time']
resources['hours'] = int(math.ceil(previous_time * 2 / 60))
job = PbsJob(command,
name,
easybuild_vars,
resources=resources,
conn=conn,
ppn=ppn)
job.module = det_full_module_name(easyconfig['ec'])
return job
def create_job(build_command, easyconfig, output_dir=None, conn=None, ppn=None):
"""
Creates a job, to build a *single* easyconfig
@param build_command: format string for command, full path to an easyconfig file will be substituted in it
@param easyconfig: easyconfig as processed by process_easyconfig
@param output_dir: optional output path; --regtest-output-dir will be used inside the job with this variable
@param conn: open connection to PBS server
@param ppn: ppn setting to use (# 'processors' (cores) per node to use)
returns the job
"""
if output_dir is None:
output_dir = 'easybuild-build'
# capture PYTHONPATH, MODULEPATH and all variables starting with EASYBUILD
easybuild_vars = {}
for name in os.environ:
if name.startswith("EASYBUILD"):
easybuild_vars[name] = os.environ[name]
others = ["PYTHONPATH", "MODULEPATH"]
for env_var in others:
if env_var in os.environ:
easybuild_vars[env_var] = os.environ[env_var]
_log.info("Dictionary of environment variables passed to job: %s" % easybuild_vars)
# obtain unique name based on name/easyconfig version tuple
ec_tuple = (easyconfig['ec']['name'], det_full_ec_version(easyconfig['ec']))
name = '-'.join(ec_tuple)
# create command based on build_command template
command = build_command % {
'spec': easyconfig['spec'],
'output_dir': os.path.join(os.path.abspath(output_dir), name),
}
# just use latest build stats
repo = init_repository(get_repository(), get_repositorypath())
buildstats = repo.get_buildstats(*ec_tuple)
resources = {}
if buildstats:
previous_time = buildstats[-1]['build_time']
resources['hours'] = int(math.ceil(previous_time * 2 / 60))
job = PbsJob(command, name, easybuild_vars, resources=resources, conn=conn, ppn=ppn)
job.module = easyconfig['ec'].full_mod_name
return job
def det_full_module_name(self, ec):
"""
Determine full module name from given easyconfig, according to the EasyBuild module naming scheme.
:param ec: dict-like object with easyconfig parameter values (e.g. 'name', 'version', etc.)
:return: string with lowercase of full module name <name>/<installversion>, e.g.: 'gzip/1.5-goolf-1.4.10
"""
# fetch required values
name = ec['name']
fversion = det_full_ec_version(ec)
#version = ec['version']
#tc_name = ec['toolchain']['name']
#tc_version = ec['toolchain']['version']
# compose module name by lowercasing and stitching parts together
return os.path.join(name.lower(), fversion.lower())
def create_job(build_command, easyconfig, output_dir="", conn=None, ppn=None):
"""
Creates a job, to build a *single* easyconfig
build_command is a format string in which a full path to an eb file will be substituted
easyconfig should be in the format as processEasyConfig returns them
output_dir is an optional path. EASYBUILDTESTOUTPUT will be set inside the job with this variable
returns the job
"""
# create command based on build_command template
command = build_command % {'spec': easyconfig['spec']}
# capture PYTHONPATH, MODULEPATH and all variables starting with EASYBUILD
easybuild_vars = {}
for name in os.environ:
if name.startswith("EASYBUILD"):
easybuild_vars[name] = os.environ[name]
others = ["PYTHONPATH", "MODULEPATH"]
for env_var in others:
if env_var in os.environ:
easybuild_vars[env_var] = os.environ[env_var]
_log.info("Dictionary of environment variables passed to job: %s" % easybuild_vars)
# obtain unique name based on name/easyconfig version tuple
ec_tuple = (easyconfig['ec']['name'], det_full_ec_version(easyconfig['ec']))
name = '-'.join(ec_tuple)
var = config.oldstyle_environment_variables['test_output_path']
easybuild_vars[var] = os.path.join(os.path.abspath(output_dir), name)
# just use latest build stats
repo = init_repository(get_repository(), get_repositorypath())
buildstats = repo.get_buildstats(*ec_tuple)
resources = {}
if buildstats:
previous_time = buildstats[-1]['build_time']
resources['hours'] = int(math.ceil(previous_time * 2 / 60))
job = PbsJob(command, name, easybuild_vars, resources=resources, conn=conn, ppn=ppn)
job.module = det_full_module_name(easyconfig['ec'])
return job
def check_ec_type(self, ec):
"""
Query module naming scheme using specified method and argument.
Obtain and pass a full parsed easyconfig file if provided keys are insufficient.
"""
if not isinstance(ec, EasyConfig) and self.requires_full_easyconfig(ec.keys()):
self.log.debug("A parsed easyconfig is required by the module naming scheme, so finding one for %s" % ec)
# fetch/parse easyconfig file if deemed necessary
eb_file = robot_find_easyconfig(ec['name'], det_full_ec_version(ec))
if eb_file is not None:
parsed_ec = process_easyconfig(eb_file, parse_only=True)
if len(parsed_ec) > 1:
self.log.warning("More than one parsed easyconfig obtained from %s, only retaining first" % eb_file)
self.log.debug("Full list of parsed easyconfigs: %s" % parsed_ec)
ec = parsed_ec[0]['ec']
else:
self.log.error("Failed to find an easyconfig file when determining module name for: %s" % ec)
return ec
def robot_find_minimal_easyconfig_for_dependency(dep):
"""
Find an easyconfig with minimal toolchain for a dependency
"""
newdep = copy.deepcopy(dep)
toolchain_hierarchy = get_toolchain_hierarchy(dep['toolchain'])
res = None
# reversed search: start with subtoolchains first, i.e. first (dummy or) compiler-only toolchain, etc.
for toolchain in toolchain_hierarchy:
newdep['toolchain'] = toolchain
eb_file = robot_find_easyconfig(newdep['name'], det_full_ec_version(newdep))
if eb_file is not None:
if newdep['toolchain'] != dep['toolchain']:
_log.info("Minimally resolving dependency %s using toolchain %s with %s", dep, toolchain, eb_file)
res = (newdep, eb_file)
break
if res is None:
_log.debug("Irresolvable minimal dependency found: %s", dep)
return res
def det_full_module_name(ec, eb_ns=False):
"""
Determine full module name following the currently active module naming scheme.
First try to pass 'parsed' easyconfig as supplied,
try and find a matching easyconfig file, parse it and supply it in case of a KeyError.
"""
try:
mod_name = _det_full_module_name(ec, eb_ns=eb_ns)
except KeyError, err:
_log.debug(
"KeyError '%s' when determining module name for %s, trying fallback procedure..."
% (err, ec))
# for dependencies, only name/version/versionsuffix/toolchain easyconfig parameters are available;
# when a key error occurs, try and find an easyconfig file to parse via the robot,
# and retry with the parsed easyconfig file (which will contains a full set of keys)
robot = build_option('robot_path')
eb_file = robot_find_easyconfig(robot, ec['name'],
det_full_ec_version(ec))
if eb_file is None:
_log.error(
"Failed to find an easyconfig file when determining module name for: %s"
% ec)
else:
parsed_ec = process_easyconfig(eb_file)
if len(parsed_ec) > 1:
_log.warning(
"More than one parsed easyconfig obtained from %s, only retaining first"
% eb_file)
try:
mod_name = _det_full_module_name(parsed_ec[0]['ec'],
eb_ns=eb_ns)
except KeyError, err:
_log.error(
"A KeyError '%s' occured when determining a module name for %s."
% parsed_ec['ec'])
def mk_key(spec):
"""Create key for dictionary with all dependencies."""
if "ec" in spec:
spec = spec["ec"]
return (spec["name"], det_full_ec_version(spec))
def select_or_generate_ec(fp, paths, specs):
"""
Select or generate an easyconfig file with the given requirements, from existing easyconfig files.
If easyconfig files are available for the specified software package,
then this function will first try to determine which toolchain to use.
* if a toolchain is given, it will use it (possible using a template easyconfig file as base);
* if not, and only a single toolchain is available, is will assume it can use that toolchain
* else, it fails -- EasyBuild doesn't select between multiple available toolchains
Next, it will trim down the selected easyconfig files to a single one,
based on the following requirements (in order of preference):
* toolchain version
* software version
* other parameters (e.g. versionprefix, versionsuffix, etc.)
If a complete match is found, it will return that easyconfig.
Else, it will generate a new easyconfig file based on the selected 'best matching' easyconfig file.
"""
specs = copy.deepcopy(specs)
# ensure that at least name is specified
if not specs.get('name'):
raise EasyBuildError(
"Supplied 'specs' dictionary doesn't even contain a name of a software package?"
)
name = specs['name']
handled_params = ['name']
# find ALL available easyconfig files for specified software
cfg = {
'version': '*',
'toolchain': {
'name': DUMMY_TOOLCHAIN_NAME,
'version': '*'
},
'versionprefix': '*',
'versionsuffix': '*',
}
installver = det_full_ec_version(cfg)
ec_files = find_matching_easyconfigs(name, installver, paths)
_log.debug("Unique ec_files: %s" % ec_files)
# we need at least one config file to start from
if len(ec_files) == 0:
# look for a template file if no easyconfig for specified software name is available
for path in paths:
templ_file = os.path.join(path, "%s.eb" % EASYCONFIG_TEMPLATE)
if os.path.isfile(templ_file):
ec_files = [templ_file]
break
else:
_log.debug("No template found at %s." % templ_file)
if len(ec_files) == 0:
raise EasyBuildError(
"No easyconfig files found for software %s, and no templates available. "
"I'm all out of ideas.", name)
ecs_and_files = [(EasyConfig(f, validate=False), f) for f in ec_files]
# TOOLCHAIN NAME
# we can't rely on set, because we also need to be able to obtain a list of unique lists
def unique(l):
"""Retain unique elements in a sorted list."""
l = sorted(l)
if len(l) > 1:
l2 = [l[0]]
for x in l:
if not x == l2[-1]:
l2.append(x)
return l2
else:
return l
# determine list of unique toolchain names
tcnames = unique([x[0]['toolchain']['name'] for x in ecs_and_files])
_log.debug("Found %d unique toolchain names: %s" % (len(tcnames), tcnames))
# if a toolchain was selected, and we have no easyconfig files for it, try and use a template
if specs.get('toolchain_name') and not specs['toolchain_name'] in tcnames:
if EASYCONFIG_TEMPLATE in tcnames:
_log.info(
"No easyconfig file for specified toolchain, but template is available."
)
else:
raise EasyBuildError(
"No easyconfig file for %s with toolchain %s, and no template available.",
name, specs['toolchain_name'])
tcname = specs.pop('toolchain_name', None)
handled_params.append('toolchain_name')
# trim down list according to selected toolchain
if tcname in tcnames:
# known toolchain, so only retain those
selected_tcname = tcname
else:
if len(tcnames) == 1 and not tcnames[0] == EASYCONFIG_TEMPLATE:
# only one (non-template) toolchain availble, so use that
tcname = tcnames[0]
selected_tcname = tcname
elif len(tcnames) == 1 and tcnames[0] == EASYCONFIG_TEMPLATE:
selected_tcname = tcnames[0]
else:
# fall-back: use template toolchain if a toolchain name was specified
if tcname:
selected_tcname = EASYCONFIG_TEMPLATE
else:
# if multiple toolchains are available, and none is specified, we quit
# we can't just pick one, how would we prefer one over the other?
raise EasyBuildError(
"No toolchain name specified, and more than one available: %s.",
tcnames)
_log.debug("Filtering easyconfigs based on toolchain name '%s'..." %
selected_tcname)
ecs_and_files = [
x for x in ecs_and_files
if x[0]['toolchain']['name'] == selected_tcname
]
_log.debug("Filtered easyconfigs: %s" % [x[1] for x in ecs_and_files])
# TOOLCHAIN VERSION
tcvers = unique([x[0]['toolchain']['version'] for x in ecs_and_files])
_log.debug("Found %d unique toolchain versions: %s" %
(len(tcvers), tcvers))
tcver = specs.pop('toolchain_version', None)
handled_params.append('toolchain_version')
(tcver, selected_tcver) = pick_version(tcver, tcvers)
_log.debug("Filtering easyconfigs based on toolchain version '%s'..." %
selected_tcver)
ecs_and_files = [
x for x in ecs_and_files
if x[0]['toolchain']['version'] == selected_tcver
]
_log.debug("Filtered easyconfigs: %s" % [x[1] for x in ecs_and_files])
# add full toolchain specification to specs
if tcname and tcver:
specs.update({'toolchain': {'name': tcname, 'version': tcver}})
handled_params.append('toolchain')
else:
if tcname:
specs.update({'toolchain_name': tcname})
if tcver:
specs.update({'toolchain_version': tcver})
# SOFTWARE VERSION
vers = unique([x[0]['version'] for x in ecs_and_files])
_log.debug("Found %d unique software versions: %s" % (len(vers), vers))
ver = specs.pop('version', None)
handled_params.append('version')
(ver, selected_ver) = pick_version(ver, vers)
if ver:
specs.update({'version': ver})
_log.debug("Filtering easyconfigs based on software version '%s'..." %
selected_ver)
ecs_and_files = [
x for x in ecs_and_files if x[0]['version'] == selected_ver
]
_log.debug("Filtered easyconfigs: %s" % [x[1] for x in ecs_and_files])
# go through parameters specified via --amend
# always include versionprefix/suffix, because we might need it to generate a file name
verpref = None
versuff = None
other_params = {'versionprefix': None, 'versionsuffix': None}
for (param, val) in specs.items():
if not param in handled_params:
other_params.update({param: val})
_log.debug(
"Filtering based on other parameters (specified via --amend): %s" %
other_params)
for (param, val) in other_params.items():
if param in ecs_and_files[0][0]._config:
vals = unique([x[0][param] for x in ecs_and_files])
else:
vals = []
filter_ecs = False
# try and select a value from the available ones, or fail if we can't
if val in vals:
# if the specified value is available, use it
selected_val = val
_log.debug("Specified %s is available, so using it: %s" %
(param, selected_val))
filter_ecs = True
elif val:
# if a value is specified, use that, even if it's not available yet
selected_val = val
# promote value to list if deemed appropriate
if vals and type(vals[0]) == list and not type(val) == list:
_log.debug(
"Promoting type of %s value to list, since original value was."
% param)
specs[param] = [val]
_log.debug(
"%s is specified, so using it (even though it's not available yet): %s"
% (param, selected_val))
elif len(vals) == 1:
# if only one value is available, use that
selected_val = vals[0]
_log.debug("Only one %s available ('%s'), so picking that" %
(param, selected_val))
filter_ecs = True
else:
# otherwise, we fail, because we don't know how to pick between different fixes
raise EasyBuildError(
"No %s specified, and can't pick from available ones: %s",
param, vals)
if filter_ecs:
_log.debug("Filtering easyconfigs based on %s '%s'..." %
(param, selected_val))
ecs_and_files = [
x for x in ecs_and_files if x[0][param] == selected_val
]
_log.debug("Filtered easyconfigs: %s" %
[x[1] for x in ecs_and_files])
# keep track of versionprefix/suffix
if param == "versionprefix":
verpref = selected_val
elif param == "versionsuffix":
versuff = selected_val
cnt = len(ecs_and_files)
if not cnt == 1:
fs = [x[1] for x in ecs_and_files]
raise EasyBuildError(
"Failed to select a single easyconfig from available ones, %s left: %s",
cnt, fs)
else:
(selected_ec, selected_ec_file) = ecs_and_files[0]
# check whether selected easyconfig matches requirements
match = True
for (key, val) in specs.items():
if key in selected_ec._config:
# values must be equal to have a full match
if selected_ec[key] != val:
match = False
else:
# if we encounter a key that is not set in the selected easyconfig, we don't have a full match
match = False
# if it matches, no need to tweak
if match:
_log.info("Perfect match found: %s" % selected_ec_file)
return (False, selected_ec_file)
# GENERATE
# if no file path was specified, generate a file name
if fp is None:
cfg = {
'version': ver,
'toolchain': {
'name': tcname,
'version': tcver
},
'versionprefix': verpref,
'versionsuffix': versuff,
}
installver = det_full_ec_version(cfg)
fp = "%s-%s.eb" % (name, installver)
# generate tweaked easyconfig file
tweak_one(selected_ec_file, fp, specs)
_log.info(
"Generated easyconfig file %s, and using it to build the requested software."
% fp)
return (True, fp)
def name(self, ec):
"""Determine package name"""
self.log.debug("Easyconfig dict passed to name() looks like: %s ", ec)
return '%s-%s' % (ec['name'], det_full_ec_version(ec))
def resolve_dependencies(easyconfigs, modtool, retain_all_deps=False, raise_error_missing_ecs=True):
"""
Work through the list of easyconfigs to determine an optimal order
:param easyconfigs: list of easyconfigs
:param modtool: ModulesTool instance to use
:param retain_all_deps: boolean indicating whether all dependencies must be retained, regardless of availability;
retain all deps when True, check matching build option when False
:param raise_error_missing_ecs: raise an error when one or more easyconfig files could not be found
"""
robot = build_option('robot_path')
# retain all dependencies if specified by either the resp. build option or the dedicated named argument
retain_all_deps = build_option('retain_all_deps') or retain_all_deps
avail_modules = modtool.available()
if retain_all_deps:
# assume that no modules are available when forced, to retain all dependencies
avail_modules = []
_log.info("Forcing all dependencies to be retained.")
else:
if len(avail_modules) == 0:
_log.warning("No installed modules. Your MODULEPATH is probably incomplete: %s" % os.getenv('MODULEPATH'))
ordered_ecs = []
# all available modules can be used for resolving dependencies except those that will be installed
being_installed = [p['full_mod_name'] for p in easyconfigs]
avail_modules = [m for m in avail_modules if m not in being_installed]
_log.debug('easyconfigs before resolving deps: %s', easyconfigs)
totally_missing, missing_easyconfigs = [], []
# resolve all dependencies, put a safeguard in place to avoid an infinite loop (shouldn't occur though)
loopcnt = 0
maxloopcnt = 10000
while easyconfigs:
# make sure this stops, we really don't want to get stuck in an infinite loop
loopcnt += 1
if loopcnt > maxloopcnt:
raise EasyBuildError("Maximum loop cnt %s reached, so quitting (easyconfigs: %s, missing_easyconfigs: %s)",
maxloopcnt, easyconfigs, missing_easyconfigs)
# first try resolving dependencies without using external dependencies
last_processed_count = -1
while len(avail_modules) > last_processed_count:
last_processed_count = len(avail_modules)
res = find_resolved_modules(easyconfigs, avail_modules, modtool, retain_all_deps=retain_all_deps)
resolved_ecs, easyconfigs, avail_modules = res
ordered_ec_mod_names = [x['full_mod_name'] for x in ordered_ecs]
for ec in resolved_ecs:
# only add easyconfig if it's not included yet (based on module name)
if not ec['full_mod_name'] in ordered_ec_mod_names:
ordered_ecs.append(ec)
# dependencies marked as external modules should be resolved via available modules at this point
missing_external_modules = [d['full_mod_name'] for ec in easyconfigs for d in ec['dependencies']
if d.get('external_module', False)]
if missing_external_modules:
raise EasyBuildError("Missing modules for dependencies marked as external modules: %s",
', '.join(missing_external_modules))
# robot: look for existing dependencies, add them
if robot and easyconfigs:
# rely on EasyBuild module naming scheme when resolving dependencies, since we know that will
# generate sensible module names that include the necessary information for the resolution to work
# (name, version, toolchain, versionsuffix)
being_installed = [EasyBuildMNS().det_full_module_name(p['ec']) for p in easyconfigs]
additional = []
for entry in easyconfigs:
# do not choose an entry that is being installed in the current run
# if they depend, you probably want to rebuild them using the new dependency
deps = entry['dependencies']
candidates = [d for d in deps if not EasyBuildMNS().det_full_module_name(d) in being_installed]
if candidates:
cand_dep = candidates[0]
# find easyconfig, might not find any
_log.debug("Looking for easyconfig for %s" % str(cand_dep))
# note: robot_find_easyconfig may return None
path = robot_find_easyconfig(cand_dep['name'], det_full_ec_version(cand_dep))
if path is None:
full_mod_name = ActiveMNS().det_full_module_name(cand_dep)
# no easyconfig found + no module available => missing dependency
if not modtool.exist([full_mod_name])[0]:
if cand_dep not in totally_missing:
totally_missing.append(cand_dep)
# no easyconfig found for dependency, but module is available
# => add to list of missing easyconfigs
elif cand_dep not in missing_easyconfigs:
_log.debug("Irresolvable dependency found (no easyconfig file): %s", cand_dep)
missing_easyconfigs.append(cand_dep)
# remove irresolvable dependency from list of dependencies so we can continue
entry['dependencies'].remove(cand_dep)
# add dummy entry for this dependency, so --dry-run for example can still report the dep
additional.append({
'dependencies': [],
'ec': None,
'full_mod_name': full_mod_name,
'spec': None,
})
else:
_log.info("Robot: resolving dependency %s with %s" % (cand_dep, path))
# build specs should not be passed down to resolved dependencies,
# to avoid that e.g. --try-toolchain trickles down into the used toolchain itself
hidden = cand_dep.get('hidden', False)
processed_ecs = process_easyconfig(path, validate=not retain_all_deps, hidden=hidden)
# ensure that selected easyconfig provides required dependency
verify_easyconfig_filename(path, cand_dep, parsed_ec=processed_ecs)
for ec in processed_ecs:
if ec not in easyconfigs + additional:
additional.append(ec)
_log.debug("Added %s as dependency of %s" % (ec, entry))
else:
mod_name = EasyBuildMNS().det_full_module_name(entry['ec'])
_log.debug("No more candidate dependencies to resolve for %s" % mod_name)
# add additional (new) easyconfigs to list of stuff to process
easyconfigs.extend(additional)
_log.debug("Unprocessed dependencies: %s", easyconfigs)
elif not robot:
# no use in continuing if robot is not enabled, dependencies won't be resolved anyway
missing_deps = [dep for x in easyconfigs for dep in x['dependencies']]
if missing_deps:
raise_error_missing_deps(missing_deps, extra_msg="enable dependency resolution via --robot?")
if totally_missing:
raise_error_missing_deps(totally_missing, extra_msg="no easyconfig file or existing module found")
if missing_easyconfigs:
if raise_error_missing_ecs:
raise_error_missing_deps(missing_easyconfigs, extra_msg="no easyconfig file found in robot search path")
else:
_log.warning("No easyconfig files found for: %s", missing_easyconfigs)
_log.info("Dependency resolution complete, building as follows: %s", ordered_ecs)
return ordered_ecs
def test_robot_find_minimal_toolchain_of_dependency(self):
"""Test robot_find_minimal_toolchain_of_dependency."""
# replace log.experimental with log.warning to allow experimental code
easybuild.framework.easyconfig.tools._log.experimental = easybuild.framework.easyconfig.tools._log.warning
test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
init_config(build_options={
'valid_module_classes': module_classes(),
'robot_path': test_easyconfigs,
})
#
# First test that it can do basic resolution
#
gzip15 = {
'name': 'gzip',
'version': '1.5',
'versionsuffix': '',
'toolchain': {'name': 'goolf', 'version': '1.4.10'},
}
get_toolchain_hierarchy.clear()
new_gzip15_toolchain = robot_find_minimal_toolchain_of_dependency(gzip15, self.modtool)
self.assertEqual(new_gzip15_toolchain, gzip15['toolchain'])
# no easyconfig for gzip 1.4 with matching non-dummy (sub)toolchain
gzip14 = {
'name': 'gzip',
'version': '1.4',
'versionsuffix': '',
'toolchain': {'name': 'goolf', 'version': '1.4.10'},
}
get_toolchain_hierarchy.clear()
self.assertEqual(robot_find_minimal_toolchain_of_dependency(gzip14, self.modtool), None)
gzip14['toolchain'] = {'name': 'gompi', 'version': '1.4.10'}
#
# Second test also including dummy toolchain
#
init_config(build_options={
'add_dummy_to_minimal_toolchains': True,
'valid_module_classes': module_classes(),
'robot_path': test_easyconfigs,
})
# specify alternative parent toolchain
gompi_1410 = {'name': 'gompi', 'version': '1.4.10'}
get_toolchain_hierarchy.clear()
new_gzip14_toolchain = robot_find_minimal_toolchain_of_dependency(gzip14, self.modtool, parent_tc=gompi_1410)
self.assertTrue(new_gzip14_toolchain != gzip14['toolchain'])
self.assertEqual(new_gzip14_toolchain, {'name': 'dummy', 'version': ''})
# default: use toolchain from dependency
gzip14['toolchain'] = gompi_1410
get_toolchain_hierarchy.clear()
new_gzip14_toolchain = robot_find_minimal_toolchain_of_dependency(gzip14, self.modtool)
self.assertTrue(new_gzip14_toolchain != gzip14['toolchain'])
self.assertEqual(new_gzip14_toolchain, {'name': 'dummy', 'version': ''})
# check reversed order (parent tc first) and skipping of parent tc itself
dep = {
'name': 'SQLite',
'version': '3.8.10.2',
'toolchain': {'name': 'goolf', 'version': '1.4.10'},
}
res = robot_find_minimal_toolchain_of_dependency(dep, self.modtool)
self.assertEqual(res, {'name': 'GCC', 'version': '4.7.2'})
res = robot_find_minimal_toolchain_of_dependency(dep, self.modtool, parent_first=True)
self.assertEqual(res, {'name': 'goolf', 'version': '1.4.10'})
#
# Finally test if it can recognise existing modules and use those
#
barec = os.path.join(self.test_prefix, 'bar-1.2.3-goolf-1.4.10.eb')
barec_txt = '\n'.join([
"easyblock = 'ConfigureMake'",
"name = 'bar'",
"version = '1.2.3'",
"homepage = 'http://example.com'",
"description = 'foo'",
"toolchain = {'name': 'goolf', 'version': '1.4.10'}",
# deliberately listing components of toolchain as dependencies without specifying subtoolchains,
# to test resolving of dependencies with minimal toolchain
# for each of these, we know test easyconfigs are available (which are required here)
"dependencies = [",
" ('OpenMPI', '1.6.4'),", # available with GCC/4.7.2
" ('OpenBLAS', '0.2.6', '-LAPACK-3.4.2'),", # available with gompi/1.4.10
" ('ScaLAPACK', '2.0.2', '-OpenBLAS-0.2.6-LAPACK-3.4.2'),", # available with gompi/1.4.10
" ('SQLite', '3.8.10.2'),", # available with goolf/1.4.10, gompi/1.4.10 and GCC/4.7.2
"]",
])
write_file(barec, barec_txt)
# check without --minimal-toolchains
init_config(build_options={
'valid_module_classes': module_classes(),
'robot_path': test_easyconfigs,
})
bar = EasyConfig(barec)
expected_dep_versions = [
'1.6.4-GCC-4.7.2',
'0.2.6-gompi-1.4.10-LAPACK-3.4.2',
'2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2',
'3.8.10.2-goolf-1.4.10',
]
for dep, expected_dep_version in zip(bar.dependencies(), expected_dep_versions):
self.assertEqual(det_full_ec_version(dep), expected_dep_version)
# check with --minimal-toolchains enabled
init_config(build_options={
'minimal_toolchains': True,
'valid_module_classes': module_classes(),
'robot_path': test_easyconfigs,
})
bar = EasyConfig(barec)
# check that all bar dependencies have been processed as expected
expected_dep_versions[-1] = '3.8.10.2-GCC-4.7.2'
for dep, expected_dep_version in zip(bar.dependencies(), expected_dep_versions):
self.assertEqual(det_full_ec_version(dep), expected_dep_version)
# Add the gompi/1.4.10 version of SQLite as an available module
module_parent = os.path.join(self.test_prefix, 'minimal_toolchain_modules')
module_file = os.path.join(module_parent, 'SQLite', '3.8.10.2-gompi-1.4.10')
module_txt = '\n'.join([
"#%Module",
"set root /tmp/SQLite/3.8.10.2",
"setenv EBROOTSQLITE $root",
"setenv EBVERSIONSQLITE 3.8.10.2",
"setenv EBDEVELSQLITE $root/easybuild/SQLite-3.8.10.2-easybuild-devel",
])
write_file(module_file, module_txt)
os.environ['MODULEPATH'] = module_parent # Add the parent directory to the MODULEPATH
invalidate_module_caches_for(module_parent)
# Reinitialize the environment for the updated MODULEPATH and use_existing_modules
init_config(build_options={
'minimal_toolchains': True,
'use_existing_modules': True,
'valid_module_classes': module_classes(),
'robot_path': test_easyconfigs,
})
# Check gompi is now being picked up
bar = EasyConfig(barec) # Re-parse the parent easyconfig
sqlite = bar.dependencies()[3]
self.assertEqual(det_full_ec_version(sqlite), '3.8.10.2-gompi-1.4.10')
# Add the goolf version as an available version and check that gets precedence over the gompi version
module_file = os.path.join(module_parent, 'SQLite', '3.8.10.2-goolf-1.4.10')
write_file(module_file, module_txt)
invalidate_module_caches_for(module_parent)
bar = EasyConfig(barec) # Re-parse the parent easyconfig
sqlite = bar.dependencies()[3]
self.assertEqual(det_full_ec_version(sqlite), '3.8.10.2-goolf-1.4.10')
def template_easyconfig_test(self, spec):
"""Tests for an individual easyconfig: parsing, instantiating easyblock, check patches, ..."""
# set to False, so it's False in case of this test failing
global single_tests_ok
prev_single_tests_ok = single_tests_ok
single_tests_ok = False
# parse easyconfig
ecs = process_easyconfig(spec)
if len(ecs) == 1:
ec = ecs[0]['ec']
# cache the parsed easyconfig, to avoid that it is parsed again
self.parsed_easyconfigs.append(ecs[0])
else:
self.assertTrue(
False,
"easyconfig %s does not contain blocks, yields only one parsed easyconfig"
% spec)
# check easyconfig file name
expected_fn = '%s-%s.eb' % (ec['name'], det_full_ec_version(ec))
msg = "Filename '%s' of parsed easyconfig matches expected filename '%s'" % (
spec, expected_fn)
self.assertEqual(os.path.basename(spec), expected_fn, msg)
name, easyblock = fetch_parameters_from_easyconfig(ec.rawtxt,
['name', 'easyblock'])
# make sure easyconfig file is in expected location
expected_subdir = os.path.join('easybuild', 'easyconfigs',
letter_dir_for(name), name)
subdir = os.path.join(*spec.split(os.path.sep)[-5:-1])
fail_msg = "Easyconfig file %s not in expected subdirectory %s" % (
spec, expected_subdir)
self.assertEqual(expected_subdir, subdir, fail_msg)
# sanity check for software name, moduleclass
self.assertEqual(ec['name'], name)
self.assertTrue(ec['moduleclass'] in build_option('valid_module_classes'))
# instantiate easyblock with easyconfig file
app_class = get_easyblock_class(easyblock, name=name)
# check that automagic fallback to ConfigureMake isn't done (deprecated behaviour)
fn = os.path.basename(spec)
error_msg = "%s relies on automagic fallback to ConfigureMake, should use easyblock = 'ConfigureMake' instead" % fn
self.assertTrue(easyblock or app_class is not ConfigureMake, error_msg)
app = app_class(ec)
# more sanity checks
self.assertTrue(name, app.name)
self.assertTrue(ec['version'], app.version)
# make sure that $root is not used, since it is not compatible with module files in Lua syntax
res = re.findall('.*\$root.*', ec.rawtxt, re.M)
error_msg = "Found use of '$root', not compatible with modules in Lua syntax, use '%%(installdir)s' instead: %s"
self.assertFalse(res, error_msg % res)
# make sure old GitHub urls for EasyBuild that include 'hpcugent' are no longer used
old_urls = [
'github.com/hpcugent/easybuild',
'hpcugent.github.com/easybuild',
'hpcugent.github.io/easybuild',
]
for old_url in old_urls:
self.assertFalse(old_url in ec.rawtxt,
"Old URL '%s' not found in %s" % (old_url, spec))
# make sure binutils is included as a build dep if toolchain is GCCcore
if ec['toolchain']['name'] == 'GCCcore':
# with 'Tarball' easyblock: only unpacking, no building; Eigen is also just a tarball
requires_binutils = ec['easyblock'] not in [
'Tarball'
] and ec['name'] not in ['Eigen']
# let's also exclude the very special case where the system GCC is used as GCCcore, and only apply this
# exception to the dependencies of binutils (since we should eventually build a new binutils with GCCcore)
if ec['toolchain']['version'] == 'system':
binutils_complete_dependencies = [
'M4', 'Bison', 'flex', 'help2man', 'zlib', 'binutils'
]
requires_binutils &= bool(
ec['name'] not in binutils_complete_dependencies)
# if no sources/extensions/components are specified, it's just a bundle (nothing is being compiled)
requires_binutils &= bool(ec['sources'] or ec['exts_list']
or ec.get('components'))
if requires_binutils:
dep_names = [d['name'] for d in ec.builddependencies()]
self.assertTrue(
'binutils' in dep_names,
"binutils is a build dep in %s: %s" % (spec, dep_names))
# make sure all patch files are available
specdir = os.path.dirname(spec)
specfn = os.path.basename(spec)
for patch in ec['patches']:
if isinstance(patch, (tuple, list)):
patch = patch[0]
# only check actual patch files, not other files being copied via the patch functionality
if patch.endswith('.patch'):
patch_full = os.path.join(specdir, patch)
msg = "Patch file %s is available for %s" % (patch_full, specfn)
self.assertTrue(os.path.isfile(patch_full), msg)
for ext in ec['exts_list']:
if isinstance(ext, (tuple, list)) and len(ext) == 3:
self.assertTrue(isinstance(ext[2], dict),
"3rd element of extension spec is a dictionary")
for ext_patch in ext[2].get('patches', []):
if isinstance(ext_patch, (tuple, list)):
ext_patch = ext_patch[0]
# only check actual patch files, not other files being copied via the patch functionality
if ext_patch.endswith('.patch'):
ext_patch_full = os.path.join(specdir, ext_patch)
msg = "Patch file %s is available for %s" % (
ext_patch_full, specfn)
self.assertTrue(os.path.isfile(ext_patch_full), msg)
# check whether all extra_options defined for used easyblock are defined
extra_opts = app.extra_options()
for key in extra_opts:
self.assertTrue(key in app.cfg)
app.close_log()
os.remove(app.logfile)
# dump the easyconfig file
handle, test_ecfile = tempfile.mkstemp()
os.close(handle)
ec.dump(test_ecfile)
dumped_ec = EasyConfigParser(test_ecfile).get_config_dict()
os.remove(test_ecfile)
# inject dummy values for templates that are only known at a later stage
dummy_template_values = {
'builddir': '/dummy/builddir',
'installdir': '/dummy/installdir',
}
ec.template_values.update(dummy_template_values)
ec_dict = ec.parser.get_config_dict()
orig_toolchain = ec_dict['toolchain']
for key in ec_dict:
# skip parameters for which value is equal to default value
orig_val = ec_dict[key]
if key in DEFAULT_CONFIG and orig_val == DEFAULT_CONFIG[key][0]:
continue
if key in extra_opts and orig_val == extra_opts[key][0]:
continue
if key not in DEFAULT_CONFIG and key not in extra_opts:
continue
orig_val = resolve_template(ec_dict[key], ec.template_values)
dumped_val = resolve_template(dumped_ec[key], ec.template_values)
# take into account that dumped value for *dependencies may include hard-coded subtoolchains
# if no easyconfig was found for the dependency with the 'parent' toolchain,
# if may get resolved using a subtoolchain, which is then hardcoded in the dumped easyconfig
if key in DEPENDENCY_PARAMETERS:
# number of dependencies should remain the same
self.assertEqual(len(orig_val), len(dumped_val))
for orig_dep, dumped_dep in zip(orig_val, dumped_val):
# name/version should always match
self.assertEqual(orig_dep[:2], dumped_dep[:2])
# 3rd value is versionsuffix;
if len(dumped_dep) >= 3:
# if no versionsuffix was specified in original dep spec, then dumped value should be empty string
if len(orig_dep) >= 3:
self.assertEqual(dumped_dep[2], orig_dep[2])
else:
self.assertEqual(dumped_dep[2], '')
# 4th value is toolchain spec
if len(dumped_dep) >= 4:
if len(orig_dep) >= 4:
self.assertEqual(dumped_dep[3], orig_dep[3])
else:
# if a subtoolchain is specifed (only) in the dumped easyconfig,
# it should *not* be the same as the parent toolchain
self.assertNotEqual(dumped_dep[3],
(orig_toolchain['name'],
orig_toolchain['version']))
else:
self.assertEqual(orig_val, dumped_val)
# test passed, so set back to True
single_tests_ok = True and prev_single_tests_ok
# add versionsuffix if Python is specified as a dependency
if any(dep[0] == 'Python' for dep in cfg.get('dependencies', [])):
if not_found_yet('versionsuffix'):
cfg['versionsuffix'] = '-Python-%(pyver)s'
# add empty sanity_check_paths
if 'sanity_check_paths' not in cfg:
cfg['sanity_check_paths'] = {'files': [], 'dirs': []}
# enable use_pip & sanity_pip_check
if cfg.get('easyblock') in ['PythonBundle', 'PythonPackage']:
cfg.update({
'use_pip': True,
'sanity_pip_check': True,
})
# enable download_dep_fail
if cfg.get('easyblock') == 'PythonPackage':
cfg['download_dep_fail'] = True
pprint.pprint(cfg)
ec_raw = '\n'.join("%s = %s" % (key, quote_str(cfg[key])) for key in cfg)
ec = EasyConfig(None, rawtxt=ec_raw)
full_ec_ver = det_full_ec_version(ec)
fn = os.path.join('%s-%s.eb' % (cfg['name'], full_ec_ver))
ec.dump(fn)
info("Easyconfig file created: %s" % fn)
def find_minimally_resolved_modules(easyconfigs,
avail_modules,
existing_modules,
retain_all_deps=False,
use_existing_modules=True):
"""
Figure out which modules are resolved already, using minimal subtoolchains for dependencies.
@param_easyconfigs: list of parsed easyconfigs
@param avail_modules: list of available modules (used to check for resolved modules)
@param existing_modules: list of existing modules (including non-available ones); used to determine
minimal toolchain to use, only if use_existing_modules is True
@param retain_all_deps: retain all dependencies, regardless of whether modules are available for them or not
@param use_existing_modules: if a module is available with a particular (sub)toolchain, use it & stop searching
"""
_log.experimental("Using minimal toolchains when resolving dependencies")
ordered_ecs = []
new_easyconfigs = []
modtool = modules_tool()
# copy, we don't want to modify the origin list of available modules
avail_modules = avail_modules[:]
# Create a (temporary sub-)directory to store minimal easyconfigs
minimal_ecs_dir = os.path.join(tempfile.gettempdir(),
'minimal-easyconfigs')
mkdir(minimal_ecs_dir, parents=True)
for easyconfig in easyconfigs:
toolchain_hierarchy = get_toolchain_hierarchy(
easyconfig['ec']['toolchain'])
new_ec = easyconfig.copy()
deps = []
for dep in easyconfig['dependencies']:
dep_resolved = False
orig_dep = dep
new_dep = None
full_mod_name = dep.get('full_mod_name',
ActiveMNS().det_full_module_name(dep))
# treat external modules as resolved when retain_all_deps is enabled (e.g., under --dry-run),
# since no corresponding easyconfig can be found for them
if retain_all_deps and dep.get('external_module', False):
_log.debug(
"Treating dependency marked as external dependency as resolved: %s",
dep)
dep_resolved = True
elif dep['toolchain'] != easyconfig['ec']['toolchain']:
# in the case where the toolchain of a dependency is different to the parent toolchain we do nothing
# we only find minimal dependencies if the dependency uses the same toolchain as the parent
# that is, we respect explicitly defined toolchains for dependencies.
dep_resolved = module_is_available(full_mod_name, modtool,
avail_modules,
dep['hidden'])
else: # parent and dependency use same toolchain
if use_existing_modules:
# check whether a module using one of the (sub)toolchains is available for this dependency
# if so, pick the minimal subtoolchain for which a module is available
for toolchain in toolchain_hierarchy:
cand_dep = copy.deepcopy(dep)
cand_dep['toolchain'] = toolchain
full_mod_name = ActiveMNS().det_full_module_name(
cand_dep)
cand_dep['full_mod_name'] = full_mod_name
dep_resolved = module_is_available(
full_mod_name, modtool, existing_modules,
cand_dep['hidden'])
if dep_resolved:
new_dep = cand_dep
_log.debug(
"Module found for dep %s using toolchain %s: %s",
dep, toolchain, full_mod_name)
break
if not dep_resolved:
# if no module was found for this dependency with any of the (sub)modules,
# or if EasyBuild was configured not to take existing modules into account first,
# we find the minimal easyconfig and update the dependency
res = robot_find_minimal_easyconfig_for_dependency(dep)
if res is not None:
new_dep, _ = res
# now check for the existence of the module of the dep
full_mod_name = ActiveMNS().det_full_module_name(
new_dep)
dep_resolved = module_is_available(
full_mod_name, modtool, avail_modules,
new_dep['hidden'])
_log.debug(
"Is replaced dep %s (module %s) resolved?: %s",
new_dep, full_mod_name, dep_resolved)
else:
_log.debug(
"Irresolvable minimal dependency found in robot search: %s"
% orig_dep)
# update the dependency in the parsed easyconfig if it was replaced
if new_dep is not None:
new_ec = deep_refresh_dependencies(new_ec, new_dep)
_log.debug(
"Updated easyconfig after replacing dep %s with %s: %s",
orig_dep, new_dep, new_ec)
dep = new_dep
if not dep_resolved or (retain_all_deps and dep['full_mod_name']
not in avail_modules):
# no module available (yet) => retain dependency as one to be resolved
deps.append(dep)
# update list of dependencies with only those unresolved
new_ec['dependencies'] = deps
if new_ec['dependencies']:
# not all dependencies are resolved yet, so retain the easyconfig
new_easyconfigs.append(new_ec)
else:
# if all dependencies have been resolved, add module for this easyconfig in the list of available modules
avail_modules.append(new_ec['full_mod_name'])
# dump easyconfig using minimal toolchain for dependencies
# FIXME: only dump when something actually changed?
newspec = '%s-%s.eb' % (new_ec['ec']['name'],
det_full_ec_version(new_ec['ec']))
newspec = os.path.join(minimal_ecs_dir, newspec)
_log.debug(
"Attempting dumping minimal easyconfig to %s and adding it to final list",
newspec)
try:
# only copy if the files are not the same file already (yes, it happens)
if os.path.exists(newspec):
_log.debug(
"Not creating minimal easyconfig file %s since it already exists",
newspec)
else:
_log.info(
"Updating %s: %s is new minimal toolchain version",
new_ec['spec'], newspec)
new_ec['spec'] = newspec
new_ec['ec'].dump(new_ec['spec'])
ordered_ecs.append(new_ec)
_log.debug("Adding easyconfig %s to final list" %
new_ec['spec'])
except (IOError, OSError) as err:
raise EasyBuildError("Failed to create easyconfig %s: %s",
newspec, err)
return ordered_ecs, new_easyconfigs, avail_modules
def template_easyconfig_test(self, spec):
"""Tests for an individual easyconfig: parsing, instantiating easyblock, check patches, ..."""
# set to False, so it's False in case of this test failing
global single_tests_ok
prev_single_tests_ok = single_tests_ok
single_tests_ok = False
# parse easyconfig
ecs = process_easyconfig(spec)
if len(ecs) == 1:
ec = ecs[0]['ec']
else:
self.assertTrue(False, "easyconfig %s does not contain blocks, yields only one parsed easyconfig" % spec)
# check easyconfig file name
expected_fn = '%s-%s.eb' % (ec['name'], det_full_ec_version(ec))
msg = "Filename '%s' of parsed easconfig matches expected filename '%s'" % (spec, expected_fn)
self.assertEqual(os.path.basename(spec), expected_fn, msg)
name, easyblock = fetch_parameters_from_easyconfig(ec.rawtxt, ['name', 'easyblock'])
# sanity check for software name
self.assertTrue(ec['name'], name)
# instantiate easyblock with easyconfig file
app_class = get_easyblock_class(easyblock, name=name)
# check that automagic fallback to ConfigureMake isn't done (deprecated behaviour)
fn = os.path.basename(spec)
error_msg = "%s relies on automagic fallback to ConfigureMake, should use easyblock = 'ConfigureMake' instead" % fn
self.assertTrue(easyblock or not app_class is ConfigureMake, error_msg)
app = app_class(ec)
# more sanity checks
self.assertTrue(name, app.name)
self.assertTrue(ec['version'], app.version)
# make sure all patch files are available
specdir = os.path.dirname(spec)
specfn = os.path.basename(spec)
for patch in ec['patches']:
if isinstance(patch, (tuple, list)):
patch = patch[0]
# only check actual patch files, not other files being copied via the patch functionality
if patch.endswith('.patch'):
patch_full = os.path.join(specdir, patch)
msg = "Patch file %s is available for %s" % (patch_full, specfn)
self.assertTrue(os.path.isfile(patch_full), msg)
ext_patches = []
for ext in ec['exts_list']:
if isinstance(ext, (tuple, list)) and len(ext) == 3:
self.assertTrue(isinstance(ext[2], dict), "3rd element of extension spec is a dictionary")
for ext_patch in ext[2].get('patches', []):
if isinstance(ext_patch, (tuple, list)):
ext_patch = ext_patch[0]
# only check actual patch files, not other files being copied via the patch functionality
if ext_patch.endswith('.patch'):
ext_patch_full = os.path.join(specdir, ext_patch)
msg = "Patch file %s is available for %s" % (ext_patch_full, specfn)
self.assertTrue(os.path.isfile(ext_patch_full), msg)
# check whether all extra_options defined for used easyblock are defined
for key in app.extra_options():
self.assertTrue(key in app.cfg)
app.close_log()
os.remove(app.logfile)
# cache the parsed easyconfig, to avoid that it is parsed again
self.parsed_easyconfigs.append(ecs[0])
# test passed, so set back to True
single_tests_ok = True and prev_single_tests_ok
def resolve_dependencies(easyconfigs,
modtool,
retain_all_deps=False,
raise_error_missing_ecs=True):
"""
Work through the list of easyconfigs to determine an optimal order
:param easyconfigs: list of easyconfigs
:param modtool: ModulesTool instance to use
:param retain_all_deps: boolean indicating whether all dependencies must be retained, regardless of availability;
retain all deps when True, check matching build option when False
:param raise_error_missing_ecs: raise an error when one or more easyconfig files could not be found
"""
robot = build_option('robot_path')
# retain all dependencies if specified by either the resp. build option or the dedicated named argument
retain_all_deps = build_option('retain_all_deps') or retain_all_deps
avail_modules = modtool.available()
if retain_all_deps:
# assume that no modules are available when forced, to retain all dependencies
avail_modules = []
_log.info("Forcing all dependencies to be retained.")
else:
if len(avail_modules) == 0:
_log.warning(
"No installed modules. Your MODULEPATH is probably incomplete: %s"
% os.getenv('MODULEPATH'))
ordered_ecs = []
# all available modules can be used for resolving dependencies except those that will be installed
being_installed = [p['full_mod_name'] for p in easyconfigs]
avail_modules = [m for m in avail_modules if m not in being_installed]
_log.debug('easyconfigs before resolving deps: %s', easyconfigs)
totally_missing, missing_easyconfigs = [], []
# resolve all dependencies, put a safeguard in place to avoid an infinite loop (shouldn't occur though)
loopcnt = 0
maxloopcnt = 10000
while easyconfigs:
# make sure this stops, we really don't want to get stuck in an infinite loop
loopcnt += 1
if loopcnt > maxloopcnt:
raise EasyBuildError(
"Maximum loop cnt %s reached, so quitting (easyconfigs: %s, missing_easyconfigs: %s)",
maxloopcnt, easyconfigs, missing_easyconfigs)
# first try resolving dependencies without using external dependencies
last_processed_count = -1
while len(avail_modules) > last_processed_count:
last_processed_count = len(avail_modules)
res = find_resolved_modules(easyconfigs,
avail_modules,
modtool,
retain_all_deps=retain_all_deps)
resolved_ecs, easyconfigs, avail_modules = res
ordered_ec_mod_names = [x['full_mod_name'] for x in ordered_ecs]
for ec in resolved_ecs:
# only add easyconfig if it's not included yet (based on module name)
if not ec['full_mod_name'] in ordered_ec_mod_names:
ordered_ecs.append(ec)
# dependencies marked as external modules should be resolved via available modules at this point
missing_external_modules = [
d['full_mod_name'] for ec in easyconfigs
for d in ec['dependencies'] if d.get('external_module', False)
]
if missing_external_modules:
raise EasyBuildError(
"Missing modules for dependencies marked as external modules: %s",
', '.join(missing_external_modules))
# robot: look for existing dependencies, add them
if robot and easyconfigs:
# rely on EasyBuild module naming scheme when resolving dependencies, since we know that will
# generate sensible module names that include the necessary information for the resolution to work
# (name, version, toolchain, versionsuffix)
being_installed = [
EasyBuildMNS().det_full_module_name(p['ec'])
for p in easyconfigs
]
additional = []
for entry in easyconfigs:
# do not choose an entry that is being installed in the current run
# if they depend, you probably want to rebuild them using the new dependency
deps = entry['dependencies']
candidates = [
d for d in deps if not EasyBuildMNS().det_full_module_name(
d) in being_installed
]
if candidates:
cand_dep = candidates[0]
# find easyconfig, might not find any
_log.debug("Looking for easyconfig for %s" % str(cand_dep))
# note: robot_find_easyconfig may return None
path = robot_find_easyconfig(cand_dep['name'],
det_full_ec_version(cand_dep))
if path is None:
full_mod_name = ActiveMNS().det_full_module_name(
cand_dep)
# no easyconfig found + no module available => missing dependency
if not modtool.exist([full_mod_name])[0]:
if cand_dep not in totally_missing:
totally_missing.append(cand_dep)
# no easyconfig found for dependency, but module is available
# => add to list of missing easyconfigs
elif cand_dep not in missing_easyconfigs:
_log.debug(
"Irresolvable dependency found (no easyconfig file): %s",
cand_dep)
missing_easyconfigs.append(cand_dep)
# remove irresolvable dependency from list of dependencies so we can continue
entry['dependencies'].remove(cand_dep)
# add dummy entry for this dependency, so --dry-run for example can still report the dep
additional.append({
'dependencies': [],
'ec': None,
'full_mod_name': full_mod_name,
'spec': None,
})
else:
_log.info("Robot: resolving dependency %s with %s" %
(cand_dep, path))
# build specs should not be passed down to resolved dependencies,
# to avoid that e.g. --try-toolchain trickles down into the used toolchain itself
hidden = cand_dep.get('hidden', False)
processed_ecs = process_easyconfig(
path, validate=not retain_all_deps, hidden=hidden)
# ensure that selected easyconfig provides required dependency
verify_easyconfig_filename(path,
cand_dep,
parsed_ec=processed_ecs)
for ec in processed_ecs:
if ec not in easyconfigs + additional:
additional.append(ec)
_log.debug("Added %s as dependency of %s" %
(ec, entry))
else:
mod_name = EasyBuildMNS().det_full_module_name(entry['ec'])
_log.debug(
"No more candidate dependencies to resolve for %s" %
mod_name)
# add additional (new) easyconfigs to list of stuff to process
easyconfigs.extend(additional)
_log.debug("Unprocessed dependencies: %s", easyconfigs)
elif not robot:
# no use in continuing if robot is not enabled, dependencies won't be resolved anyway
missing_deps = [
dep for x in easyconfigs for dep in x['dependencies']
]
if missing_deps:
raise_error_missing_deps(
missing_deps,
extra_msg="enable dependency resolution via --robot?")
if totally_missing:
raise_error_missing_deps(
totally_missing,
extra_msg="no easyconfig file or existing module found")
if missing_easyconfigs:
if raise_error_missing_ecs:
raise_error_missing_deps(
missing_easyconfigs,
extra_msg="no easyconfig file found in robot search path")
else:
_log.warning("No easyconfig files found for: %s",
missing_easyconfigs)
_log.info("Dependency resolution complete, building as follows: %s",
ordered_ecs)
return ordered_ecs
def template_easyconfig_test(self, spec):
"""Tests for an individual easyconfig: parsing, instantiating easyblock, check patches, ..."""
# set to False, so it's False in case of this test failing
global single_tests_ok
prev_single_tests_ok = single_tests_ok
single_tests_ok = False
# parse easyconfig
ecs = process_easyconfig(spec)
if len(ecs) == 1:
ec = ecs[0]['ec']
else:
self.assertTrue(False, "easyconfig %s does not contain blocks, yields only one parsed easyconfig" % spec)
# check easyconfig file name
expected_fn = '%s-%s.eb' % (ec['name'], det_full_ec_version(ec))
msg = "Filename '%s' of parsed easyconfig matches expected filename '%s'" % (spec, expected_fn)
self.assertEqual(os.path.basename(spec), expected_fn, msg)
name, easyblock = fetch_parameters_from_easyconfig(ec.rawtxt, ['name', 'easyblock'])
# make sure easyconfig file is in expected location
expected_subdir = os.path.join('easybuild', 'easyconfigs', letter_dir_for(name), name)
subdir = os.path.join(*spec.split(os.path.sep)[-5:-1])
fail_msg = "Easyconfig file %s not in expected subdirectory %s" % (spec, expected_subdir)
self.assertEqual(expected_subdir, subdir, fail_msg)
# sanity check for software name, moduleclass
self.assertEqual(ec['name'], name)
self.assertTrue(ec['moduleclass'] in build_option('valid_module_classes'))
# instantiate easyblock with easyconfig file
app_class = get_easyblock_class(easyblock, name=name)
# check that automagic fallback to ConfigureMake isn't done (deprecated behaviour)
fn = os.path.basename(spec)
error_msg = "%s relies on automagic fallback to ConfigureMake, should use easyblock = 'ConfigureMake' instead" % fn
self.assertTrue(easyblock or not app_class is ConfigureMake, error_msg)
app = app_class(ec)
# more sanity checks
self.assertTrue(name, app.name)
self.assertTrue(ec['version'], app.version)
# make sure that $root is not used, since it is not compatible with module files in Lua syntax
res = re.findall('.*\$root.*', ec.rawtxt, re.M)
error_msg = "Found use of '$root', not compatible with modules in Lua syntax, use '%%(installdir)s' instead: %s"
self.assertFalse(res, error_msg % res)
# make sure old GitHub urls for EasyBuild that include 'hpcugent' are no longer used
old_urls = [
'github.com/hpcugent/easybuild',
'hpcugent.github.com/easybuild',
'hpcugent.github.io/easybuild',
]
for old_url in old_urls:
self.assertFalse(old_url in ec.rawtxt, "Old URL '%s' not found in %s" % (old_url, spec))
# make sure binutils is included as a build dep if toolchain is GCCcore
if ec['toolchain']['name'] == 'GCCcore':
# with 'Tarball' easyblock: only unpacking, no building; Eigen is also just a tarball
requires_binutils = ec['easyblock'] not in ['Tarball'] and ec['name'] not in ['Eigen']
# let's also exclude the very special case where the system GCC is used as GCCcore, and only apply this
# exception to the dependencies of binutils (since we should eventually build a new binutils with GCCcore)
if ec['toolchain']['version'] == 'system':
binutils_complete_dependencies = ['M4', 'Bison', 'flex', 'help2man', 'zlib', 'binutils']
requires_binutils &= bool(ec['name'] not in binutils_complete_dependencies)
# if no sources/extensions/components are specified, it's just a bundle (nothing is being compiled)
requires_binutils &= bool(ec['sources'] or ec['exts_list'] or ec.get('components'))
if requires_binutils:
dep_names = [d['name'] for d in ec['builddependencies']]
self.assertTrue('binutils' in dep_names, "binutils is a build dep in %s: %s" % (spec, dep_names))
# make sure all patch files are available
specdir = os.path.dirname(spec)
specfn = os.path.basename(spec)
for patch in ec['patches']:
if isinstance(patch, (tuple, list)):
patch = patch[0]
# only check actual patch files, not other files being copied via the patch functionality
if patch.endswith('.patch'):
patch_full = os.path.join(specdir, patch)
msg = "Patch file %s is available for %s" % (patch_full, specfn)
self.assertTrue(os.path.isfile(patch_full), msg)
ext_patches = []
for ext in ec['exts_list']:
if isinstance(ext, (tuple, list)) and len(ext) == 3:
self.assertTrue(isinstance(ext[2], dict), "3rd element of extension spec is a dictionary")
for ext_patch in ext[2].get('patches', []):
if isinstance(ext_patch, (tuple, list)):
ext_patch = ext_patch[0]
# only check actual patch files, not other files being copied via the patch functionality
if ext_patch.endswith('.patch'):
ext_patch_full = os.path.join(specdir, ext_patch)
msg = "Patch file %s is available for %s" % (ext_patch_full, specfn)
self.assertTrue(os.path.isfile(ext_patch_full), msg)
# check whether all extra_options defined for used easyblock are defined
extra_opts = app.extra_options()
for key in extra_opts:
self.assertTrue(key in app.cfg)
app.close_log()
os.remove(app.logfile)
# dump the easyconfig file
handle, test_ecfile = tempfile.mkstemp()
os.close(handle)
ec.dump(test_ecfile)
dumped_ec = EasyConfigParser(test_ecfile).get_config_dict()
os.remove(test_ecfile)
# inject dummy values for templates that are only known at a later stage
dummy_template_values = {
'builddir': '/dummy/builddir',
'installdir': '/dummy/installdir',
}
ec.template_values.update(dummy_template_values)
ec_dict = ec.parser.get_config_dict()
orig_toolchain = ec_dict['toolchain']
for key in ec_dict:
# skip parameters for which value is equal to default value
orig_val = ec_dict[key]
if key in DEFAULT_CONFIG and orig_val == DEFAULT_CONFIG[key][0]:
continue
if key in extra_opts and orig_val == extra_opts[key][0]:
continue
if key not in DEFAULT_CONFIG and key not in extra_opts:
continue
orig_val = resolve_template(ec_dict[key], ec.template_values)
dumped_val = resolve_template(dumped_ec[key], ec.template_values)
# take into account that dumped value for *dependencies may include hard-coded subtoolchains
# if no easyconfig was found for the dependency with the 'parent' toolchain,
# if may get resolved using a subtoolchain, which is then hardcoded in the dumped easyconfig
if key in DEPENDENCY_PARAMETERS:
# number of dependencies should remain the same
self.assertEqual(len(orig_val), len(dumped_val))
for orig_dep, dumped_dep in zip(orig_val, dumped_val):
# name/version should always match
self.assertEqual(orig_dep[:2], dumped_dep[:2])
# 3rd value is versionsuffix;
if len(dumped_dep) >= 3:
# if no versionsuffix was specified in original dep spec, then dumped value should be empty string
if len(orig_dep) >= 3:
self.assertEqual(dumped_dep[2], orig_dep[2])
else:
self.assertEqual(dumped_dep[2], '')
# 4th value is toolchain spec
if len(dumped_dep) >= 4:
if len(orig_dep) >= 4:
self.assertEqual(dumped_dep[3], orig_dep[3])
else:
# if a subtoolchain is specifed (only) in the dumped easyconfig,
# it should *not* be the same as the parent toolchain
self.assertNotEqual(dumped_dep[3], (orig_toolchain['name'], orig_toolchain['version']))
else:
self.assertEqual(orig_val, dumped_val)
# cache the parsed easyconfig, to avoid that it is parsed again
self.parsed_easyconfigs.append(ecs[0])
# test passed, so set back to True
single_tests_ok = True and prev_single_tests_ok
def mk_key(spec):
"""Create key for dictionary with all dependencies."""
if 'ec' in spec:
spec = spec['ec']
return (spec['name'], det_full_ec_version(spec))
def select_or_generate_ec(fp, paths, specs):
"""
Select or generate an easyconfig file with the given requirements, from existing easyconfig files.
If easyconfig files are available for the specified software package,
then this function will first try to determine which toolchain to use.
* if a toolchain is given, it will use it (possible using a template easyconfig file as base);
* if not, and only a single toolchain is available, is will assume it can use that toolchain
* else, it fails -- EasyBuild doesn't select between multiple available toolchains
Next, it will trim down the selected easyconfig files to a single one,
based on the following requirements (in order of preference):
* toolchain version
* software version
* other parameters (e.g. versionprefix, versionsuffix, etc.)
If a complete match is found, it will return that easyconfig.
Else, it will generate a new easyconfig file based on the selected 'best matching' easyconfig file.
"""
specs = copy.deepcopy(specs)
# ensure that at least name is specified
if not specs.get('name'):
_log.error("Supplied 'specs' dictionary doesn't even contain a name of a software package?")
name = specs['name']
handled_params = ['name']
# find ALL available easyconfig files for specified software
cfg = {
'version': '*',
'toolchain': {'name': DUMMY_TOOLCHAIN_NAME, 'version': '*'},
'versionprefix': '*',
'versionsuffix': '*',
}
installver = det_full_ec_version(cfg)
ec_files = find_matching_easyconfigs(name, installver, paths)
_log.debug("Unique ec_files: %s" % ec_files)
# we need at least one config file to start from
if len(ec_files) == 0:
# look for a template file if no easyconfig for specified software name is available
for path in paths:
templ_file = os.path.join(path, "%s.eb" % EASYCONFIG_TEMPLATE)
if os.path.isfile(templ_file):
ec_files = [templ_file]
break
else:
_log.debug("No template found at %s." % templ_file)
if len(ec_files) == 0:
_log.error("No easyconfig files found for software %s, and no templates available. I'm all out of ideas." % name)
ecs_and_files = [(EasyConfig(f, validate=False), f) for f in ec_files]
# TOOLCHAIN NAME
# we can't rely on set, because we also need to be able to obtain a list of unique lists
def unique(l):
"""Retain unique elements in a sorted list."""
l = sorted(l)
if len(l) > 1:
l2 = [l[0]]
for x in l:
if not x == l2[-1]:
l2.append(x)
return l2
else:
return l
# determine list of unique toolchain names
tcnames = unique([x[0]['toolchain']['name'] for x in ecs_and_files])
_log.debug("Found %d unique toolchain names: %s" % (len(tcnames), tcnames))
# if a toolchain was selected, and we have no easyconfig files for it, try and use a template
if specs.get('toolchain_name') and not specs['toolchain_name'] in tcnames:
if EASYCONFIG_TEMPLATE in tcnames:
_log.info("No easyconfig file for specified toolchain, but template is available.")
else:
_log.error("No easyconfig file for %s with toolchain %s, " \
"and no template available." % (name, specs['toolchain_name']))
tcname = specs.pop('toolchain_name', None)
handled_params.append('toolchain_name')
# trim down list according to selected toolchain
if tcname in tcnames:
# known toolchain, so only retain those
selected_tcname = tcname
else:
if len(tcnames) == 1 and not tcnames[0] == EASYCONFIG_TEMPLATE:
# only one (non-template) toolchain availble, so use that
tcname = tcnames[0]
selected_tcname = tcname
elif len(tcnames) == 1 and tcnames[0] == EASYCONFIG_TEMPLATE:
selected_tcname = tcnames[0]
else:
# fall-back: use template toolchain if a toolchain name was specified
if tcname:
selected_tcname = EASYCONFIG_TEMPLATE
else:
# if multiple toolchains are available, and none is specified, we quit
# we can't just pick one, how would we prefer one over the other?
_log.error("No toolchain name specified, and more than one available: %s." % tcnames)
_log.debug("Filtering easyconfigs based on toolchain name '%s'..." % selected_tcname)
ecs_and_files = [x for x in ecs_and_files if x[0]['toolchain']['name'] == selected_tcname]
_log.debug("Filtered easyconfigs: %s" % [x[1] for x in ecs_and_files])
# TOOLCHAIN VERSION
tcvers = unique([x[0]['toolchain']['version'] for x in ecs_and_files])
_log.debug("Found %d unique toolchain versions: %s" % (len(tcvers), tcvers))
tcver = specs.pop('toolchain_version', None)
handled_params.append('toolchain_version')
(tcver, selected_tcver) = pick_version(tcver, tcvers)
_log.debug("Filtering easyconfigs based on toolchain version '%s'..." % selected_tcver)
ecs_and_files = [x for x in ecs_and_files if x[0]['toolchain']['version'] == selected_tcver]
_log.debug("Filtered easyconfigs: %s" % [x[1] for x in ecs_and_files])
# add full toolchain specification to specs
if tcname and tcver:
specs.update({'toolchain': {'name': tcname, 'version': tcver}})
handled_params.append('toolchain')
else:
if tcname:
specs.update({'toolchain_name': tcname})
if tcver:
specs.update({'toolchain_version': tcver})
# SOFTWARE VERSION
vers = unique([x[0]['version'] for x in ecs_and_files])
_log.debug("Found %d unique software versions: %s" % (len(vers), vers))
ver = specs.pop('version', None)
handled_params.append('version')
(ver, selected_ver) = pick_version(ver, vers)
if ver:
specs.update({'version': ver})
_log.debug("Filtering easyconfigs based on software version '%s'..." % selected_ver)
ecs_and_files = [x for x in ecs_and_files if x[0]['version'] == selected_ver]
_log.debug("Filtered easyconfigs: %s" % [x[1] for x in ecs_and_files])
# go through parameters specified via --amend
# always include versionprefix/suffix, because we might need it to generate a file name
verpref = None
versuff = None
other_params = {'versionprefix': None, 'versionsuffix': None}
for (param, val) in specs.items():
if not param in handled_params:
other_params.update({param: val})
_log.debug("Filtering based on other parameters (specified via --amend): %s" % other_params)
for (param, val) in other_params.items():
if param in ecs_and_files[0][0]._config:
vals = unique([x[0][param] for x in ecs_and_files])
else:
vals = []
filter_ecs = False
# try and select a value from the available ones, or fail if we can't
if val in vals:
# if the specified value is available, use it
selected_val = val
_log.debug("Specified %s is available, so using it: %s" % (param, selected_val))
filter_ecs = True
elif val:
# if a value is specified, use that, even if it's not available yet
selected_val = val
# promote value to list if deemed appropriate
if vals and type(vals[0]) == list and not type(val) == list:
_log.debug("Promoting type of %s value to list, since original value was." % param)
specs[param] = [val]
_log.debug("%s is specified, so using it (even though it's not available yet): %s" % (param, selected_val))
elif len(vals) == 1:
# if only one value is available, use that
selected_val = vals[0]
_log.debug("Only one %s available ('%s'), so picking that" % (param, selected_val))
filter_ecs = True
else:
# otherwise, we fail, because we don't know how to pick between different fixes
_log.error("No %s specified, and can't pick from available %ses %s" % (param,
param,
vals))
if filter_ecs:
_log.debug("Filtering easyconfigs based on %s '%s'..." % (param, selected_val))
ecs_and_files = [x for x in ecs_and_files if x[0][param] == selected_val]
_log.debug("Filtered easyconfigs: %s" % [x[1] for x in ecs_and_files])
# keep track of versionprefix/suffix
if param == "versionprefix":
verpref = selected_val
elif param == "versionsuffix":
versuff = selected_val
cnt = len(ecs_and_files)
if not cnt == 1:
fs = [x[1] for x in ecs_and_files]
_log.error("Failed to select a single easyconfig from available ones, %s left: %s" % (cnt, fs))
else:
(selected_ec, selected_ec_file) = ecs_and_files[0]
# check whether selected easyconfig matches requirements
match = True
for (key, val) in specs.items():
if key in selected_ec._config:
# values must be equal to have a full match
if not selected_ec[key] == val:
match = False
else:
# if we encounter a key that is not set in the selected easyconfig, we don't have a full match
match = False
# if it matches, no need to tweak
if match:
_log.info("Perfect match found: %s" % selected_ec_file)
return (False, selected_ec_file)
# GENERATE
# if no file path was specified, generate a file name
if fp is None:
cfg = {
'version': ver,
'toolchain': {'name': tcname, 'version': tcver},
'versionprefix': verpref,
'versionsuffix': versuff,
}
installver = det_full_ec_version(cfg)
fp = "%s-%s.eb" % (name, installver)
# generate tweaked easyconfig file
tweak_one(selected_ec_file, fp, specs)
_log.info("Generated easyconfig file %s, and using it to build the requested software." % fp)
return (True, fp)
def resolve_dependencies(unprocessed, build_specs=None, retain_all_deps=False):
"""
Work through the list of easyconfigs to determine an optimal order
@param unprocessed: list of easyconfigs
@param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...)
"""
robot = build_option('robot_path')
retain_all_deps = build_option('retain_all_deps') or retain_all_deps
if retain_all_deps:
# assume that no modules are available when forced, to retain all dependencies
avail_modules = []
_log.info("Forcing all dependencies to be retained.")
else:
# Get a list of all available modules (format: [(name, installversion), ...])
avail_modules = modules_tool().available()
if len(avail_modules) == 0:
_log.warning("No installed modules. Your MODULEPATH is probably incomplete: %s" % os.getenv('MODULEPATH'))
ordered_ecs = []
# all available modules can be used for resolving dependencies except those that will be installed
being_installed = [p['module'] for p in unprocessed]
avail_modules = [m for m in avail_modules if not m in being_installed]
_log.debug('unprocessed before resolving deps: %s' % unprocessed)
# resolve all dependencies, put a safeguard in place to avoid an infinite loop (shouldn't occur though)
irresolvable = []
loopcnt = 0
maxloopcnt = 10000
while unprocessed:
# make sure this stops, we really don't want to get stuck in an infinite loop
loopcnt += 1
if loopcnt > maxloopcnt:
tup = (maxloopcnt, unprocessed, irresolvable)
msg = "Maximum loop cnt %s reached, so quitting (unprocessed: %s, irresolvable: %s)" % tup
_log.error(msg)
# first try resolving dependencies without using external dependencies
last_processed_count = -1
while len(avail_modules) > last_processed_count:
last_processed_count = len(avail_modules)
more_ecs, unprocessed, avail_modules = find_resolved_modules(unprocessed, avail_modules)
for ec in more_ecs:
if not ec['module'] in [x['module'] for x in ordered_ecs]:
ordered_ecs.append(ec)
# robot: look for existing dependencies, add them
if robot and unprocessed:
being_installed = [det_full_module_name(p['ec'], eb_ns=True) for p in unprocessed]
additional = []
for i, entry in enumerate(unprocessed):
# do not choose an entry that is being installed in the current run
# if they depend, you probably want to rebuild them using the new dependency
deps = entry['dependencies']
candidates = [d for d in deps if not det_full_module_name(d, eb_ns=True) in being_installed]
if len(candidates) > 0:
cand_dep = candidates[0]
# find easyconfig, might not find any
_log.debug("Looking for easyconfig for %s" % str(cand_dep))
# note: robot_find_easyconfig may return None
path = robot_find_easyconfig(robot, cand_dep['name'], det_full_ec_version(cand_dep))
if path is None:
# no easyconfig found for dependency, add to list of irresolvable dependencies
if cand_dep not in irresolvable:
_log.debug("Irresolvable dependency found: %s" % cand_dep)
irresolvable.append(cand_dep)
# remove irresolvable dependency from list of dependencies so we can continue
entry['dependencies'].remove(cand_dep)
else:
_log.info("Robot: resolving dependency %s with %s" % (cand_dep, path))
# build specs should not be passed down to resolved dependencies,
# to avoid that e.g. --try-toolchain trickles down into the used toolchain itself
processed_ecs = process_easyconfig(path, validate=not retain_all_deps)
# ensure that selected easyconfig provides required dependency
mods = [det_full_module_name(spec['ec']) for spec in processed_ecs]
dep_mod_name = det_full_module_name(cand_dep)
if not dep_mod_name in mods:
tup = (path, dep_mod_name, mods)
_log.error("easyconfig file %s does not contain module %s (mods: %s)" % tup)
for ec in processed_ecs:
if not ec in unprocessed + additional:
additional.append(ec)
_log.debug("Added %s as dependency of %s" % (ec, entry))
else:
mod_name = det_full_module_name(entry['ec'], eb_ns=True)
_log.debug("No more candidate dependencies to resolve for %s" % mod_name)
# add additional (new) easyconfigs to list of stuff to process
unprocessed.extend(additional)
elif not robot:
# no use in continuing if robot is not enabled, dependencies won't be resolved anyway
irresolvable = [dep for x in unprocessed for dep in x['dependencies']]
break
if irresolvable:
irresolvable_mod_deps = [(det_full_module_name(dep, eb_ns=True), dep) for dep in irresolvable]
_log.error('Irresolvable dependencies encountered: %s' % irresolvable_mod_deps)
_log.info("Dependency resolution complete, building as follows:\n%s" % ordered_ecs)
return ordered_ecs
def template_easyconfig_test(self, spec):
"""Tests for an individual easyconfig: parsing, instantiating easyblock, check patches, ..."""
# set to False, so it's False in case of this test failing
global single_tests_ok
prev_single_tests_ok = single_tests_ok
single_tests_ok = False
# parse easyconfig
ecs = process_easyconfig(spec)
if len(ecs) == 1:
ec = ecs[0]['ec']
else:
self.assertTrue(False, "easyconfig %s does not contain blocks, yields only one parsed easyconfig" % spec)
# check easyconfig file name
expected_fn = '%s-%s.eb' % (ec['name'], det_full_ec_version(ec))
msg = "Filename '%s' of parsed easyconfig matches expected filename '%s'" % (spec, expected_fn)
self.assertEqual(os.path.basename(spec), expected_fn, msg)
name, easyblock = fetch_parameters_from_easyconfig(ec.rawtxt, ['name', 'easyblock'])
# make sure easyconfig file is in expected location
expected_subdir = os.path.join('easybuild', 'easyconfigs', letter_dir_for(name), name)
subdir = os.path.join(*spec.split(os.path.sep)[-5:-1])
fail_msg = "Easyconfig file %s not in expected subdirectory %s" % (spec, expected_subdir)
self.assertEqual(expected_subdir, subdir, fail_msg)
# sanity check for software name, moduleclass
self.assertEqual(ec['name'], name)
self.assertTrue(ec['moduleclass'] in build_option('valid_module_classes'))
# instantiate easyblock with easyconfig file
app_class = get_easyblock_class(easyblock, name=name)
# check that automagic fallback to ConfigureMake isn't done (deprecated behaviour)
fn = os.path.basename(spec)
error_msg = "%s relies on automagic fallback to ConfigureMake, should use easyblock = 'ConfigureMake' instead" % fn
self.assertTrue(easyblock or not app_class is ConfigureMake, error_msg)
app = app_class(ec)
# more sanity checks
self.assertTrue(name, app.name)
self.assertTrue(ec['version'], app.version)
# make sure all patch files are available
specdir = os.path.dirname(spec)
specfn = os.path.basename(spec)
for patch in ec['patches']:
if isinstance(patch, (tuple, list)):
patch = patch[0]
# only check actual patch files, not other files being copied via the patch functionality
if patch.endswith('.patch'):
patch_full = os.path.join(specdir, patch)
msg = "Patch file %s is available for %s" % (patch_full, specfn)
self.assertTrue(os.path.isfile(patch_full), msg)
ext_patches = []
for ext in ec['exts_list']:
if isinstance(ext, (tuple, list)) and len(ext) == 3:
self.assertTrue(isinstance(ext[2], dict), "3rd element of extension spec is a dictionary")
for ext_patch in ext[2].get('patches', []):
if isinstance(ext_patch, (tuple, list)):
ext_patch = ext_patch[0]
# only check actual patch files, not other files being copied via the patch functionality
if ext_patch.endswith('.patch'):
ext_patch_full = os.path.join(specdir, ext_patch)
msg = "Patch file %s is available for %s" % (ext_patch_full, specfn)
self.assertTrue(os.path.isfile(ext_patch_full), msg)
# check whether all extra_options defined for used easyblock are defined
extra_opts = app.extra_options()
for key in extra_opts:
self.assertTrue(key in app.cfg)
app.close_log()
os.remove(app.logfile)
# dump the easyconfig file
handle, test_ecfile = tempfile.mkstemp()
os.close(handle)
ec.dump(test_ecfile)
dumped_ec = EasyConfigParser(test_ecfile).get_config_dict()
os.remove(test_ecfile)
# inject dummy values for templates that are only known at a later stage
dummy_template_values = {
'builddir': '/dummy/builddir',
'installdir': '/dummy/installdir',
}
ec.template_values.update(dummy_template_values)
ec_dict = ec.parser.get_config_dict()
orig_toolchain = ec_dict['toolchain']
for key in ec_dict:
# skip parameters for which value is equal to default value
orig_val = ec_dict[key]
if key in DEFAULT_CONFIG and orig_val == DEFAULT_CONFIG[key][0]:
continue
if key in extra_opts and orig_val == extra_opts[key][0]:
continue
if key not in DEFAULT_CONFIG and key not in extra_opts:
continue
orig_val = resolve_template(ec_dict[key], ec.template_values)
dumped_val = resolve_template(dumped_ec[key], ec.template_values)
# take into account that dumped value for *dependencies may include hard-coded subtoolchains
# if no easyconfig was found for the dependency with the 'parent' toolchain,
# if may get resolved using a subtoolchain, which is then hardcoded in the dumped easyconfig
if key in DEPENDENCY_PARAMETERS:
# number of dependencies should remain the same
self.assertEqual(len(orig_val), len(dumped_val))
for orig_dep, dumped_dep in zip(orig_val, dumped_val):
# name/version should always match
self.assertEqual(orig_dep[:2], dumped_dep[:2])
# 3rd value is versionsuffix;
if len(dumped_dep) >= 3:
# if no versionsuffix was specified in original dep spec, then dumped value should be empty string
if len(orig_dep) >= 3:
self.assertEqual(dumped_dep[2], orig_dep[2])
else:
self.assertEqual(dumped_dep[2], '')
# 4th value is toolchain spec
if len(dumped_dep) >= 4:
if len(orig_dep) >= 4:
self.assertEqual(dumped_dep[3], orig_dep[3])
else:
# if a subtoolchain is specifed (only) in the dumped easyconfig,
# it should *not* be the same as the parent toolchain
self.assertNotEqual(dumped_dep[3], (orig_toolchain['name'], orig_toolchain['version']))
else:
self.assertEqual(orig_val, dumped_val)
# cache the parsed easyconfig, to avoid that it is parsed again
self.parsed_easyconfigs.append(ecs[0])
# test passed, so set back to True
single_tests_ok = True and prev_single_tests_ok
def find_minimally_resolved_modules(easyconfigs, avail_modules, existing_modules,
retain_all_deps=False, use_existing_modules=True):
"""
Figure out which modules are resolved already, using minimal subtoolchains for dependencies.
@param_easyconfigs: list of parsed easyconfigs
@param avail_modules: list of available modules (used to check for resolved modules)
@param existing_modules: list of existing modules (including non-available ones); used to determine
minimal toolchain to use, only if use_existing_modules is True
@param retain_all_deps: retain all dependencies, regardless of whether modules are available for them or not
@param use_existing_modules: if a module is available with a particular (sub)toolchain, use it & stop searching
"""
_log.experimental("Using minimal toolchains when resolving dependencies")
ordered_ecs = []
new_easyconfigs = []
modtool = modules_tool()
# copy, we don't want to modify the origin list of available modules
avail_modules = avail_modules[:]
# Create a (temporary sub-)directory to store minimal easyconfigs
minimal_ecs_dir = os.path.join(tempfile.gettempdir(), 'minimal-easyconfigs')
mkdir(minimal_ecs_dir, parents=True)
for easyconfig in easyconfigs:
toolchain_hierarchy = get_toolchain_hierarchy(easyconfig['ec']['toolchain'])
new_ec = easyconfig.copy()
deps = []
for dep in easyconfig['dependencies']:
dep_resolved = False
orig_dep = dep
new_dep = None
full_mod_name = dep.get('full_mod_name', ActiveMNS().det_full_module_name(dep))
# treat external modules as resolved when retain_all_deps is enabled (e.g., under --dry-run),
# since no corresponding easyconfig can be found for them
if retain_all_deps and dep.get('external_module', False):
_log.debug("Treating dependency marked as external dependency as resolved: %s", dep)
dep_resolved = True
elif dep['toolchain'] != easyconfig['ec']['toolchain']:
# in the case where the toolchain of a dependency is different to the parent toolchain we do nothing
# we only find minimal dependencies if the dependency uses the same toolchain as the parent
# that is, we respect explicitly defined toolchains for dependencies.
dep_resolved = module_is_available(full_mod_name, modtool, avail_modules, dep['hidden'])
else: # parent and dependency use same toolchain
if use_existing_modules:
# check whether a module using one of the (sub)toolchains is available for this dependency
# if so, pick the minimal subtoolchain for which a module is available
for toolchain in toolchain_hierarchy:
cand_dep = copy.deepcopy(dep)
cand_dep['toolchain'] = toolchain
full_mod_name = ActiveMNS().det_full_module_name(cand_dep)
cand_dep['full_mod_name'] = full_mod_name
dep_resolved = module_is_available(full_mod_name, modtool, existing_modules, cand_dep['hidden'])
if dep_resolved:
new_dep = cand_dep
_log.debug("Module found for dep %s using toolchain %s: %s", dep, toolchain, full_mod_name)
break
if not dep_resolved:
# if no module was found for this dependency with any of the (sub)modules,
# or if EasyBuild was configured not to take existing modules into account first,
# we find the minimal easyconfig and update the dependency
res = robot_find_minimal_easyconfig_for_dependency(dep)
if res is not None:
new_dep, _ = res
# now check for the existence of the module of the dep
full_mod_name = ActiveMNS().det_full_module_name(new_dep)
dep_resolved = module_is_available(full_mod_name, modtool, avail_modules, new_dep['hidden'])
_log.debug("Is replaced dep %s (module %s) resolved?: %s", new_dep, full_mod_name, dep_resolved)
else:
_log.debug("Irresolvable minimal dependency found in robot search: %s" % orig_dep)
# update the dependency in the parsed easyconfig if it was replaced
if new_dep is not None:
new_ec = deep_refresh_dependencies(new_ec, new_dep)
_log.debug("Updated easyconfig after replacing dep %s with %s: %s", orig_dep, new_dep, new_ec)
dep = new_dep
if not dep_resolved or (retain_all_deps and dep['full_mod_name'] not in avail_modules):
# no module available (yet) => retain dependency as one to be resolved
deps.append(dep)
# update list of dependencies with only those unresolved
new_ec['dependencies'] = deps
if new_ec['dependencies']:
# not all dependencies are resolved yet, so retain the easyconfig
new_easyconfigs.append(new_ec)
else:
# if all dependencies have been resolved, add module for this easyconfig in the list of available modules
avail_modules.append(new_ec['full_mod_name'])
# dump easyconfig using minimal toolchain for dependencies
# FIXME: only dump when something actually changed?
newspec = '%s-%s.eb' % (new_ec['ec']['name'], det_full_ec_version(new_ec['ec']))
newspec = os.path.join(minimal_ecs_dir, newspec)
_log.debug("Attempting dumping minimal easyconfig to %s and adding it to final list", newspec)
try:
# only copy if the files are not the same file already (yes, it happens)
if os.path.exists(newspec):
_log.debug("Not creating minimal easyconfig file %s since it already exists", newspec)
else:
_log.info("Updating %s: %s is new minimal toolchain version", new_ec['spec'], newspec)
new_ec['spec'] = newspec
new_ec['ec'].dump(new_ec['spec'])
ordered_ecs.append(new_ec)
_log.debug("Adding easyconfig %s to final list" % new_ec['spec'])
except (IOError, OSError) as err:
raise EasyBuildError("Failed to create easyconfig %s: %s", newspec, err)
return ordered_ecs, new_easyconfigs, avail_modules
def mk_key(spec):
"""Create key for dictionary with all dependencies."""
if 'ec' in spec:
spec = spec['ec']
return (spec['name'], det_full_ec_version(spec))
def template_easyconfig_test(self, spec):
"""Tests for an individual easyconfig: parsing, instantiating easyblock, check patches, ..."""
# set to False, so it's False in case of this test failing
global single_tests_ok
prev_single_tests_ok = single_tests_ok
single_tests_ok = False
# parse easyconfig
ecs = process_easyconfig(spec)
if len(ecs) == 1:
ec = ecs[0]['ec']
else:
self.assertTrue(
False,
"easyconfig %s does not contain blocks, yields only one parsed easyconfig"
% spec)
# check easyconfig file name
expected_fn = '%s-%s.eb' % (ec['name'], det_full_ec_version(ec))
msg = "Filename '%s' of parsed easyconfig matches expected filename '%s'" % (
spec, expected_fn)
self.assertEqual(os.path.basename(spec), expected_fn, msg)
name, easyblock = fetch_parameters_from_easyconfig(ec.rawtxt,
['name', 'easyblock'])
# make sure easyconfig file is in expected location
expected_subdir = os.path.join('easybuild', 'easyconfigs',
name.lower()[0], name)
subdir = os.path.join(*spec.split(os.path.sep)[-5:-1])
fail_msg = "Easyconfig file %s not in expected subdirectory %s" % (
spec, expected_subdir)
self.assertEqual(expected_subdir, subdir, fail_msg)
# sanity check for software name
self.assertTrue(ec['name'], name)
# instantiate easyblock with easyconfig file
app_class = get_easyblock_class(easyblock, name=name)
# check that automagic fallback to ConfigureMake isn't done (deprecated behaviour)
fn = os.path.basename(spec)
error_msg = "%s relies on automagic fallback to ConfigureMake, should use easyblock = 'ConfigureMake' instead" % fn
self.assertTrue(easyblock or not app_class is ConfigureMake, error_msg)
app = app_class(ec)
# more sanity checks
self.assertTrue(name, app.name)
self.assertTrue(ec['version'], app.version)
# make sure all patch files are available
specdir = os.path.dirname(spec)
specfn = os.path.basename(spec)
for patch in ec['patches']:
if isinstance(patch, (tuple, list)):
patch = patch[0]
# only check actual patch files, not other files being copied via the patch functionality
if patch.endswith('.patch'):
patch_full = os.path.join(specdir, patch)
msg = "Patch file %s is available for %s" % (patch_full, specfn)
self.assertTrue(os.path.isfile(patch_full), msg)
ext_patches = []
for ext in ec['exts_list']:
if isinstance(ext, (tuple, list)) and len(ext) == 3:
self.assertTrue(isinstance(ext[2], dict),
"3rd element of extension spec is a dictionary")
for ext_patch in ext[2].get('patches', []):
if isinstance(ext_patch, (tuple, list)):
ext_patch = ext_patch[0]
# only check actual patch files, not other files being copied via the patch functionality
if ext_patch.endswith('.patch'):
ext_patch_full = os.path.join(specdir, ext_patch)
msg = "Patch file %s is available for %s" % (
ext_patch_full, specfn)
self.assertTrue(os.path.isfile(ext_patch_full), msg)
# check whether all extra_options defined for used easyblock are defined
for key in app.extra_options():
self.assertTrue(key in app.cfg)
app.close_log()
os.remove(app.logfile)
# dump the easyconfig file
handle, test_ecfile = tempfile.mkstemp()
os.close(handle)
ec.dump(test_ecfile)
dumped_ec = EasyConfig(test_ecfile)
os.remove(test_ecfile)
# inject dummy values for templates that are only known at a later stage
dummy_template_values = {
'builddir': '/dummy/builddir',
'installdir': '/dummy/installdir',
}
ec.template_values.update(dummy_template_values)
dumped_ec.template_values.update(dummy_template_values)
for key in sorted(ec._config):
self.assertEqual(ec[key], dumped_ec[key])
# cache the parsed easyconfig, to avoid that it is parsed again
self.parsed_easyconfigs.append(ecs[0])
# test passed, so set back to True
single_tests_ok = True and prev_single_tests_ok
def name(self, ec):
"""Determine package name"""
self.log.debug("Easyconfig dict passed to name() looks like: %s ", ec)
return '%s-%s' % (ec['name'], det_full_ec_version(ec))
def resolve_dependencies(unprocessed, build_specs=None, retain_all_deps=False):
"""
Work through the list of easyconfigs to determine an optimal order
@param unprocessed: list of easyconfigs
@param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...)
@param retain_all_deps: boolean indicating whether all dependencies must be retained, regardless of availability;
retain all deps when True, check matching build option when False
"""
robot = build_option("robot_path")
# retain all dependencies if specified by either the resp. build option or the dedicated named argument
retain_all_deps = build_option("retain_all_deps") or retain_all_deps
if retain_all_deps:
# assume that no modules are available when forced, to retain all dependencies
avail_modules = []
_log.info("Forcing all dependencies to be retained.")
else:
# Get a list of all available modules (format: [(name, installversion), ...])
avail_modules = modules_tool().available()
if len(avail_modules) == 0:
_log.warning("No installed modules. Your MODULEPATH is probably incomplete: %s" % os.getenv("MODULEPATH"))
ordered_ecs = []
# all available modules can be used for resolving dependencies except those that will be installed
being_installed = [p["full_mod_name"] for p in unprocessed]
avail_modules = [m for m in avail_modules if not m in being_installed]
_log.debug("unprocessed before resolving deps: %s" % unprocessed)
# resolve all dependencies, put a safeguard in place to avoid an infinite loop (shouldn't occur though)
irresolvable = []
loopcnt = 0
maxloopcnt = 10000
while unprocessed:
# make sure this stops, we really don't want to get stuck in an infinite loop
loopcnt += 1
if loopcnt > maxloopcnt:
raise EasyBuildError(
"Maximum loop cnt %s reached, so quitting (unprocessed: %s, irresolvable: %s)",
maxloopcnt,
unprocessed,
irresolvable,
)
# first try resolving dependencies without using external dependencies
last_processed_count = -1
while len(avail_modules) > last_processed_count:
last_processed_count = len(avail_modules)
res = find_resolved_modules(unprocessed, avail_modules, retain_all_deps=retain_all_deps)
more_ecs, unprocessed, avail_modules = res
for ec in more_ecs:
if not ec["full_mod_name"] in [x["full_mod_name"] for x in ordered_ecs]:
ordered_ecs.append(ec)
# dependencies marked as external modules should be resolved via available modules at this point
missing_external_modules = [
d["full_mod_name"] for ec in unprocessed for d in ec["dependencies"] if d.get("external_module", False)
]
if missing_external_modules:
raise EasyBuildError(
"Missing modules for one or more dependencies marked as external modules: %s", missing_external_modules
)
# robot: look for existing dependencies, add them
if robot and unprocessed:
# rely on EasyBuild module naming scheme when resolving dependencies, since we know that will
# generate sensible module names that include the necessary information for the resolution to work
# (name, version, toolchain, versionsuffix)
being_installed = [EasyBuildMNS().det_full_module_name(p["ec"]) for p in unprocessed]
additional = []
for entry in unprocessed:
# do not choose an entry that is being installed in the current run
# if they depend, you probably want to rebuild them using the new dependency
deps = entry["dependencies"]
candidates = [d for d in deps if not EasyBuildMNS().det_full_module_name(d) in being_installed]
if candidates:
cand_dep = candidates[0]
# find easyconfig, might not find any
_log.debug("Looking for easyconfig for %s" % str(cand_dep))
# note: robot_find_easyconfig may return None
path = robot_find_easyconfig(cand_dep["name"], det_full_ec_version(cand_dep))
if path is None:
# no easyconfig found for dependency, add to list of irresolvable dependencies
if cand_dep not in irresolvable:
_log.debug("Irresolvable dependency found: %s" % cand_dep)
irresolvable.append(cand_dep)
# remove irresolvable dependency from list of dependencies so we can continue
entry["dependencies"].remove(cand_dep)
else:
_log.info("Robot: resolving dependency %s with %s" % (cand_dep, path))
# build specs should not be passed down to resolved dependencies,
# to avoid that e.g. --try-toolchain trickles down into the used toolchain itself
hidden = cand_dep.get("hidden", False)
processed_ecs = process_easyconfig(path, validate=not retain_all_deps, hidden=hidden)
# ensure that selected easyconfig provides required dependency
mods = [spec["ec"].full_mod_name for spec in processed_ecs]
dep_mod_name = ActiveMNS().det_full_module_name(cand_dep)
if not dep_mod_name in mods:
raise EasyBuildError(
"easyconfig file %s does not contain module %s (mods: %s)", path, dep_mod_name, mods
)
for ec in processed_ecs:
if not ec in unprocessed + additional:
additional.append(ec)
_log.debug("Added %s as dependency of %s" % (ec, entry))
else:
mod_name = EasyBuildMNS().det_full_module_name(entry["ec"])
_log.debug("No more candidate dependencies to resolve for %s" % mod_name)
# add additional (new) easyconfigs to list of stuff to process
unprocessed.extend(additional)
_log.debug("Unprocessed dependencies: %s", unprocessed)
elif not robot:
# no use in continuing if robot is not enabled, dependencies won't be resolved anyway
irresolvable = [dep for x in unprocessed for dep in x["dependencies"]]
break
if irresolvable:
_log.warning("Irresolvable dependencies (details): %s" % irresolvable)
irresolvable_mods_eb = [EasyBuildMNS().det_full_module_name(dep) for dep in irresolvable]
_log.warning("Irresolvable dependencies (EasyBuild module names): %s" % ", ".join(irresolvable_mods_eb))
irresolvable_mods = [ActiveMNS().det_full_module_name(dep) for dep in irresolvable]
raise EasyBuildError("Irresolvable dependencies encountered: %s", ", ".join(irresolvable_mods))
_log.info("Dependency resolution complete, building as follows: %s" % ordered_ecs)
return ordered_ecs
def resolve_dependencies(unprocessed, build_specs=None, retain_all_deps=False):
"""
Work through the list of easyconfigs to determine an optimal order
@param unprocessed: list of easyconfigs
@param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...)
@param retain_all_deps: boolean indicating whether all dependencies must be retained, regardless of availability;
retain all deps when True, check matching build option when False
"""
robot = build_option('robot_path')
# retain all dependencies if specified by either the resp. build option or the dedicated named argument
retain_all_deps = build_option('retain_all_deps') or retain_all_deps
if retain_all_deps:
# assume that no modules are available when forced, to retain all dependencies
avail_modules = []
_log.info("Forcing all dependencies to be retained.")
else:
# Get a list of all available modules (format: [(name, installversion), ...])
avail_modules = modules_tool().available()
if len(avail_modules) == 0:
_log.warning(
"No installed modules. Your MODULEPATH is probably incomplete: %s"
% os.getenv('MODULEPATH'))
ordered_ecs = []
# all available modules can be used for resolving dependencies except those that will be installed
being_installed = [p['full_mod_name'] for p in unprocessed]
avail_modules = [m for m in avail_modules if not m in being_installed]
_log.debug('unprocessed before resolving deps: %s' % unprocessed)
# resolve all dependencies, put a safeguard in place to avoid an infinite loop (shouldn't occur though)
irresolvable = []
loopcnt = 0
maxloopcnt = 10000
while unprocessed:
# make sure this stops, we really don't want to get stuck in an infinite loop
loopcnt += 1
if loopcnt > maxloopcnt:
tup = (maxloopcnt, unprocessed, irresolvable)
msg = "Maximum loop cnt %s reached, so quitting (unprocessed: %s, irresolvable: %s)" % tup
_log.error(msg)
# first try resolving dependencies without using external dependencies
last_processed_count = -1
while len(avail_modules) > last_processed_count:
last_processed_count = len(avail_modules)
res = find_resolved_modules(unprocessed,
avail_modules,
retain_all_deps=retain_all_deps)
more_ecs, unprocessed, avail_modules = res
for ec in more_ecs:
if not ec['full_mod_name'] in [
x['full_mod_name'] for x in ordered_ecs
]:
ordered_ecs.append(ec)
# robot: look for existing dependencies, add them
if robot and unprocessed:
# rely on EasyBuild module naming scheme when resolving dependencies, since we know that will
# generate sensible module names that include the necessary information for the resolution to work
# (name, version, toolchain, versionsuffix)
being_installed = [
EasyBuildMNS().det_full_module_name(p['ec'])
for p in unprocessed
]
additional = []
for entry in unprocessed:
# do not choose an entry that is being installed in the current run
# if they depend, you probably want to rebuild them using the new dependency
deps = entry['dependencies']
candidates = [
d for d in deps if not EasyBuildMNS().det_full_module_name(
d) in being_installed
]
if candidates:
cand_dep = candidates[0]
# find easyconfig, might not find any
_log.debug("Looking for easyconfig for %s" % str(cand_dep))
# note: robot_find_easyconfig may return None
path = robot_find_easyconfig(cand_dep['name'],
det_full_ec_version(cand_dep))
if path is None:
# no easyconfig found for dependency, add to list of irresolvable dependencies
if cand_dep not in irresolvable:
_log.debug("Irresolvable dependency found: %s" %
cand_dep)
irresolvable.append(cand_dep)
# remove irresolvable dependency from list of dependencies so we can continue
entry['dependencies'].remove(cand_dep)
else:
_log.info("Robot: resolving dependency %s with %s" %
(cand_dep, path))
# build specs should not be passed down to resolved dependencies,
# to avoid that e.g. --try-toolchain trickles down into the used toolchain itself
hidden = cand_dep.get('hidden', False)
processed_ecs = process_easyconfig(
path, validate=not retain_all_deps, hidden=hidden)
# ensure that selected easyconfig provides required dependency
mods = [
spec['ec'].full_mod_name for spec in processed_ecs
]
dep_mod_name = ActiveMNS().det_full_module_name(
cand_dep)
if not dep_mod_name in mods:
tup = (path, dep_mod_name, mods)
_log.error(
"easyconfig file %s does not contain module %s (mods: %s)"
% tup)
for ec in processed_ecs:
if not ec in unprocessed + additional:
additional.append(ec)
_log.debug("Added %s as dependency of %s" %
(ec, entry))
else:
mod_name = EasyBuildMNS().det_full_module_name(entry['ec'])
_log.debug(
"No more candidate dependencies to resolve for %s" %
mod_name)
# add additional (new) easyconfigs to list of stuff to process
unprocessed.extend(additional)
elif not robot:
# no use in continuing if robot is not enabled, dependencies won't be resolved anyway
irresolvable = [
dep for x in unprocessed for dep in x['dependencies']
]
break
if irresolvable:
_log.warning("Irresolvable dependencies (details): %s" % irresolvable)
irresolvable_mods_eb = [
EasyBuildMNS().det_full_module_name(dep) for dep in irresolvable
]
_log.warning("Irresolvable dependencies (EasyBuild module names): %s" %
', '.join(irresolvable_mods_eb))
irresolvable_mods = [
ActiveMNS().det_full_module_name(dep) for dep in irresolvable
]
_log.error('Irresolvable dependencies encountered: %s' %
', '.join(irresolvable_mods))
_log.info("Dependency resolution complete, building as follows:\n%s" %
ordered_ecs)
return ordered_ecs
def main():
"""Main function."""
if len(sys.argv) == 3:
ec = sys.argv[1]
tc = sys.argv[2]
else:
error("Usage %s <easyconfig> <toolchain> [<name=version>]" %
sys.argv[0])
tc_name, tc_ver = tc.split('/')
print("Updating %s for %s toolchain version %s..." % (ec, tc_name, tc_ver))
set_up_configuration(silent=True)
modtool = modules_tool()
robot_path = build_option('robot_path')
ec_path = det_easyconfig_paths([ec])[0]
print("Found %s easyconfig file at %s" % (ec, ec_path))
parsed_ecs, _ = parse_easyconfigs([(ec_path, False)], validate=False)
print("Resolving dependencies... ", end='')
ecs = resolve_dependencies(parsed_ecs, modtool, retain_all_deps=True)
print("found stack of %d easyconfigs" % len(ecs))
print("Filtering toolchain and its dependencies...")
ec_tc = parsed_ecs[0]['ec']['toolchain']
ecs_to_remove = [{
'name': ec_tc['name'],
'version': ec_tc['version'],
'toolchain': {
'name': SYSTEM_TOOLCHAIN_NAME
}
}]
updated_ecs = {}
# if GCCcore is used as toolchain, determine binutils version to use
if tc_name == 'GCCcore':
binutils_pattern = '^binutils.*-%s-%s.*.eb$' % (tc_name, tc_ver)
_, res = search_file(robot_path, binutils_pattern)
if res:
if len(res) == 1:
parsed_ecs, _ = parse_easyconfigs([(res[0], False)])
binutils_ec = parsed_ecs[0]
tc = copy.copy(binutils_ec['ec']['toolchain'])
ecs_to_remove.append({
'name': 'binutils',
'version': binutils_ec['ec'].version,
'toolchain': tc
})
else:
error("Found more than one easyconfig matching '%s': %s" %
(binutils_pattern, res))
else:
error("No easyconfig file found for binutils using pattern '%s'" %
binutils_pattern)
while (ecs_to_remove):
to_remove = ecs_to_remove.pop(0)
print("Removing %(name)s/%(version)s (toolchain: %(toolchain)s)" %
to_remove)
for ec in ecs:
if ec['ec'].name == to_remove['name'] and ec['ec'].version == to_remove['version'] and \
ec['ec']['toolchain']['name'] == to_remove['toolchain']['name']:
ecs.remove(ec)
ecs_to_remove.extend(dep for dep in ec['ec']['dependencies'] +
ec['ec']['builddependencies'])
updated_ecs[ec['full_mod_name']] = {
'builddependencies': [],
'dependencies': [],
'toolchain': copy.copy(ec['ec']['toolchain']),
'version': ec['ec'].version,
}
break
ecs_to_write = []
for ec in ecs:
ec_fn = os.path.basename(ec['spec'])
print(term.bold("Determining version for %s..." % ec_fn))
full_mod_name = ec['full_mod_name']
ec_tc = copy.copy(ec['ec']['toolchain'])
# update toolchain (unless it's SYSTEM)
if ec_tc['name'] != SYSTEM_TOOLCHAIN_NAME:
if ec_tc['name'] == tc_name:
ec_tc['version'] = tc_ver
else:
error("Don't know how to update toolchain %s" % ec_tc['name'])
# update (build) dependencies
build_deps = []
for dep in ec['ec']['builddependencies']:
new_dep_ver = updated_ecs[dep['full_mod_name']]['version']
build_deps.append((dep['name'], new_dep_ver))
deps = []
for dep in ec['ec']['dependencies']:
new_dep_ver = updated_ecs[dep['full_mod_name']]['version']
deps.append((dep['name'], new_dep_ver))
# determine software version to use;
# first, try searching for an existing easyconfig with specified toolchain;
# if that fails, try to determine latest upstream version
ec_pattern = '^%s.*-%s-%s.*.eb$' % (ec['ec'].name, tc_name, tc_ver)
_, res = search_file(robot_path, ec_pattern)
if res:
if len(res) == 1:
parsed_ecs, _ = parse_easyconfigs([(res[0], False)])
ec = parsed_ecs[0]
new_version = ec['ec'].version
print(
term.green(
"Found existing easyconfig, sticking to version %s" %
new_version))
else:
error("Multiple hits found using '%s': %s" % (res, ec_pattern))
else:
new_version = update_version(ec['ec'])
ecs_to_write.append(ec)
if new_version is None:
print(
term.yellow(
"No new version found for %s, using existing version" %
full_mod_name))
new_version = ec['ec'].version
updated_ecs[full_mod_name] = {
'builddependencies': build_deps,
'dependencies': deps,
'toolchain': ec_tc,
'version': new_version,
}
for ec in ecs_to_write:
full_mod_name = ec['full_mod_name']
pprint.pprint(full_mod_name)
ec = ec['ec']
ectxt = ec.rawtxt
key_pattern = r'^%s\s*=.*'
list_key_pattern = r'^%s\s*=\s*\[([^\]]|\n)*\s*\]'
new_version = updated_ecs[full_mod_name]['version']
if ec.version != new_version:
regex = re.compile(key_pattern % 'version', re.M)
ectxt = regex.sub("version = '%s'" % new_version, ectxt)
# if version got updated, also wipe the checksums
regex = re.compile(list_key_pattern % 'checksums', re.M)
ectxt = regex.sub("checksums = []", ectxt)
# toolchain
tc_str = "toolchain = {'name': '%(name)s', 'version': '%(version)s'}" % updated_ecs[
full_mod_name]['toolchain']
regex = re.compile(key_pattern % 'toolchain', re.M)
ectxt = regex.sub(tc_str, ectxt)
# dependencies
for key in ('builddependencies', 'dependencies'):
deps_str = '%s = [\n' % key
for dep in updated_ecs[full_mod_name][key]:
deps_str += ' ' + str(dep) + ',\n'
deps_str += ']'
regex = re.compile(list_key_pattern % key, re.M)
ectxt = regex.sub(deps_str, ectxt)
specs = {
'name': ec.name,
'toolchain': updated_ecs[full_mod_name]['toolchain'],
'version': new_version,
'versionsuffix': ec['versionsuffix'],
}
ec_fn = '%s-%s.eb' % (ec.name, det_full_ec_version(specs))
write_file(ec_fn, ectxt)
print(term.green("%s written" % ec_fn))
