def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir=None):
"""
Take an easyconfig spec, parse it, map it to a target toolchain and dump it out
:param ec_spec: Location of original easyconfig file
:param toolchain_mapping: Mapping between source toolchain and target toolchain
:param targetdir: Directory to dump the modified easyconfig file in
:return: Location of the modified easyconfig file
"""
# Fully parse the original easyconfig
parsed_ec = process_easyconfig(ec_spec, validate=False)[0]['ec']
# Replace the toolchain if the mapping exists
tc_name = parsed_ec['toolchain']['name']
if tc_name in toolchain_mapping:
new_toolchain = toolchain_mapping[tc_name]
_log.debug("Replacing parent toolchain %s with %s", parsed_ec['toolchain'], new_toolchain)
parsed_ec['toolchain'] = new_toolchain
# Replace the toolchains of all the dependencies
for key in DEPENDENCY_PARAMETERS:
# loop over a *copy* of dependency dicts (with resolved templates);
# to update the original dep dict, we need to get a reference with templating disabled...
val = parsed_ec[key]
orig_val = parsed_ec.get_ref(key)
if key in parsed_ec.iterate_options:
val = flatten(val)
orig_val = flatten(orig_val)
for idx, dep in enumerate(val):
# reference to original dep dict, this is the one we should be updating
orig_dep = orig_val[idx]
# skip dependencies that are marked as external modules
if dep['external_module']:
continue
dep_tc_name = dep['toolchain']['name']
if dep_tc_name in toolchain_mapping:
orig_dep['toolchain'] = toolchain_mapping[dep_tc_name]
# Replace the binutils version (if necessary)
if 'binutils' in toolchain_mapping and (dep['name'] == 'binutils' and dep_tc_name == GCCcore.NAME):
orig_dep.update(toolchain_mapping['binutils'])
# set module names
orig_dep['short_mod_name'] = ActiveMNS().det_short_module_name(dep)
orig_dep['full_mod_name'] = ActiveMNS().det_full_module_name(dep)
# Determine the name of the modified easyconfig and dump it to target_dir
ec_filename = '%s-%s.eb' % (parsed_ec['name'], det_full_ec_version(parsed_ec))
tweaked_spec = os.path.join(targetdir or tempfile.gettempdir(), ec_filename)
parsed_ec.dump(tweaked_spec, always_overwrite=False, backup=True)
_log.debug("Dumped easyconfig tweaked via --try-toolchain* to %s", tweaked_spec)
return tweaked_spec
def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir=None):
"""
Take an easyconfig spec, parse it, map it to a target toolchain and dump it out
:param ec_spec: Location of original easyconfig file
:param toolchain_mapping: Mapping between source toolchain and target toolchain
:param targetdir: Directory to dump the modified easyconfig file in
:return: Location of the modified easyconfig file
"""
# Fully parse the original easyconfig
parsed_ec = process_easyconfig(ec_spec, validate=False)[0]['ec']
# Replace the toolchain if the mapping exists
tc_name = parsed_ec['toolchain']['name']
if tc_name in toolchain_mapping:
new_toolchain = toolchain_mapping[tc_name]
_log.debug("Replacing parent toolchain %s with %s", parsed_ec['toolchain'], new_toolchain)
parsed_ec['toolchain'] = new_toolchain
# Replace the toolchains of all the dependencies
for key in DEPENDENCY_PARAMETERS:
# loop over a *copy* of dependency dicts (with resolved templates);
# to update the original dep dict, we need to get a reference with templating disabled...
val = parsed_ec[key]
orig_val = parsed_ec.get_ref(key)
if key in parsed_ec.iterate_options:
val = flatten(val)
orig_val = flatten(orig_val)
for idx, dep in enumerate(val):
# reference to original dep dict, this is the one we should be updating
orig_dep = orig_val[idx]
# skip dependencies that are marked as external modules
if dep['external_module']:
continue
dep_tc_name = dep['toolchain']['name']
if dep_tc_name in toolchain_mapping:
orig_dep['toolchain'] = toolchain_mapping[dep_tc_name]
# Replace the binutils version (if necessary)
if 'binutils' in toolchain_mapping and (dep['name'] == 'binutils' and dep_tc_name == GCCcore.NAME):
orig_dep.update(toolchain_mapping['binutils'])
# set module names
orig_dep['short_mod_name'] = ActiveMNS().det_short_module_name(dep)
orig_dep['full_mod_name'] = ActiveMNS().det_full_module_name(dep)
# Determine the name of the modified easyconfig and dump it to target_dir
ec_filename = '%s-%s.eb' % (parsed_ec['name'], det_full_ec_version(parsed_ec))
tweaked_spec = os.path.join(targetdir or tempfile.gettempdir(), ec_filename)
parsed_ec.dump(tweaked_spec, always_overwrite=False, backup=True)
_log.debug("Dumped easyconfig tweaked via --try-toolchain* to %s", tweaked_spec)
return tweaked_spec
def sanity_check_step(self):
"""Custom sanity check for EasyBuild."""
# list of dirs to check, by package
# boolean indicates whether dir is expected to reside in Python lib/pythonX/site-packages dir
subdirs_by_pkg = [
('framework', [('easybuild/framework', True),
('easybuild/tools', True)]),
('easyblocks', [('easybuild/easyblocks', True)]),
('easyconfigs', [('easybuild/easyconfigs', False)]),
]
# final list of directories to check, by setup tool
# order matters, e.g. setuptools before distutils
eb_dirs = OrderedDict()
eb_dirs['setuptools'] = []
eb_dirs['distutils.core'] = flatten([x[1] for x in subdirs_by_pkg])
# determine setup tool (setuptools or distutils)
setup_tool = None
for tool in eb_dirs.keys():
self.log.debug("Trying %s.." % tool)
try:
exec "from %s import setup" % tool
del setup
setup_tool = tool
break
except ImportError:
pass
self.log.debug('setup_tool: %s' % setup_tool)
# for a setuptools installation, we need to figure out the egg dirs since we don't know the individual package versions
if setup_tool == 'setuptools':
try:
installed_dirs = os.listdir(
os.path.join(self.installdir, self.pylibdir))
for (pkg, subdirs) in subdirs_by_pkg:
sel_dirs = [
x for x in installed_dirs
if x.startswith('easybuild_%s' % pkg)
]
if not len(sel_dirs) == 1:
self.log.error(
"Failed to isolate installed egg dir for easybuild-%s"
% pkg)
for (subdir, _) in subdirs:
# eggs always go in Python lib/pythonX/site-packages dir with setuptools
eb_dirs['setuptools'].append(
(os.path.join(sel_dirs[0], subdir), True))
except OSError, err:
self.log.error(
"Failed to determine sanity check dir paths: %s" % err)
def sanity_check_step(self):
"""Custom sanity check for EasyBuild."""
# list of dirs to check, by package
# boolean indicates whether dir is expected to reside in Python lib/pythonX/site-packages dir
subdirs_by_pkg = [
('framework', [('easybuild/framework', True), ('easybuild/tools', True)]),
('easyblocks', [('easybuild/easyblocks', True)]),
('easyconfigs', [('easybuild/easyconfigs', False)]),
]
# final list of directories to check, by setup tool
# order matters, e.g. setuptools before distutils
eb_dirs = OrderedDict()
eb_dirs['setuptools'] = []
eb_dirs['distutils.core'] = flatten([x[1] for x in subdirs_by_pkg])
# determine setup tool (setuptools or distutils)
setup_tool = None
for tool in eb_dirs.keys():
self.log.debug("Trying %s.." % tool)
try:
exec "from %s import setup" % tool
del setup
setup_tool = tool
break
except ImportError:
pass
self.log.debug('setup_tool: %s' % setup_tool)
# for a setuptools installation, we need to figure out the egg dirs since we don't know the individual package versions
if setup_tool == 'setuptools':
try:
installed_dirs = os.listdir(os.path.join(self.installdir, self.pylibdir))
for (pkg, subdirs) in subdirs_by_pkg:
sel_dirs = [x for x in installed_dirs if x.startswith('easybuild_%s' % pkg)]
if not len(sel_dirs) == 1:
self.log.error("Failed to isolate installed egg dir for easybuild-%s" % pkg)
for (subdir, _) in subdirs:
# eggs always go in Python lib/pythonX/site-packages dir with setuptools
eb_dirs['setuptools'].append((os.path.join(sel_dirs[0], subdir), True))
except OSError, err:
self.log.error("Failed to determine sanity check dir paths: %s" % err)
def sanity_check_step(self):
"""Custom sanity check for EasyBuild."""
# check whether easy-install.pth contains correct entries
easy_install_pth = os.path.join(self.installdir, self.pylibdir,
'easy-install.pth')
if os.path.exists(easy_install_pth):
easy_install_pth_txt = read_file(easy_install_pth)
ignore_pkgs = ['setuptools', 'vsc-install']
if LooseVersion(self.version) > LooseVersion('3.999'):
ignore_pkgs.append('vsc-base')
for pkg in [
p for p in self.easybuild_pkgs if p not in ignore_pkgs
]:
if pkg == 'vsc-base':
# don't include strict version check for vsc-base
pkg_regex = re.compile(r"^\./%s" % pkg.replace('-', '_'),
re.M)
else:
major_minor_version = '.'.join(self.version.split('.')[:2])
pkg_regex = re.compile(
r"^\./%s-%s" %
(pkg.replace('-', '_'), major_minor_version), re.M)
if not pkg_regex.search(easy_install_pth_txt):
raise EasyBuildError(
"Failed to find pattern '%s' in %s: %s",
pkg_regex.pattern, easy_install_pth,
easy_install_pth_txt)
# list of dirs to check, by package
# boolean indicates whether dir is expected to reside in Python lib/pythonX/site-packages dir
subdirs_by_pkg = {
'easybuild-framework': [('easybuild/framework', True),
('easybuild/tools', True)],
'easybuild-easyblocks': [('easybuild/easyblocks', True)],
'easybuild-easyconfigs': [('easybuild/easyconfigs', False)],
}
if LooseVersion(self.version) >= LooseVersion('2.0') and LooseVersion(
self.version) < LooseVersion('3.999'):
subdirs_by_pkg.update({
'vsc-base': [('vsc/utils', True)],
})
# final list of directories to check, by setup tool
# order matters, e.g. setuptools before distutils
eb_dirs = OrderedDict()
eb_dirs['setuptools'] = []
eb_dirs['distutils.core'] = flatten(
[x for x in subdirs_by_pkg.values()])
# determine setup tool (setuptools or distutils)
setup_tool = None
for tool in eb_dirs.keys():
self.log.debug("Trying %s.." % tool)
try:
exec("from %s import setup" % tool)
setup_tool = tool
break
except ImportError:
pass
self.log.debug('setup_tool: %s' % setup_tool)
# for a setuptools installation, we need to figure out the egg dirs,
# since we don't know the individual package versions
if setup_tool == 'setuptools':
try:
installed_dirs = os.listdir(
os.path.join(self.installdir, self.pylibdir))
for (pkg, subdirs) in subdirs_by_pkg.items():
sel_dirs = [
x for x in installed_dirs
if x.startswith(pkg.replace('-', '_'))
]
if not len(sel_dirs) == 1:
raise EasyBuildError(
"Failed to isolate installed egg dir for %s", pkg)
for (subdir, _) in subdirs:
# eggs always go in Python lib/pythonX/site-packages dir with setuptools
eb_dirs['setuptools'].append(
(os.path.join(sel_dirs[0], subdir), True))
except OSError as err:
raise EasyBuildError(
"Failed to determine sanity check dir paths: %s", err)
# set of sanity check paths to check for EasyBuild
custom_paths = {
'files': ['bin/eb'],
'dirs': [self.pylibdir] + [[x, os.path.join(self.pylibdir, x)][y]
for (x, y) in eb_dirs[setup_tool]],
}
# make sure we don't trip over deprecated behavior in old EasyBuild versions
eb_cmd = 'eb'
if LooseVersion(self.version) <= LooseVersion('1.16.0'):
eb_cmd = 'EASYBUILD_DEPRECATED=1.0 eb'
# set of sanity check commands to run for EasyBuild
custom_commands = [
# this may spit out a wrong version, but that should be safe to ignore
# occurs when the EasyBuild being used is newer than the EasyBuild being installed
(eb_cmd, '--version'),
(eb_cmd, '-a'),
(eb_cmd, '-e ConfigureMake -a'),
]
# (temporary) cleanse copy of initial environment to avoid conflict with (potentially) loaded EasyBuild module
self.real_initial_environ = copy.deepcopy(self.initial_environ)
for env_var in ['_LMFILES_', 'LOADEDMODULES']:
if env_var in self.initial_environ:
self.initial_environ.pop(env_var)
os.environ.pop(env_var)
self.log.debug(
"Unset $%s in current env and copy of original env to make sanity check work"
% env_var)
super(EB_EasyBuildMeta,
self).sanity_check_step(custom_paths=custom_paths,
custom_commands=custom_commands)
def sanity_check_step(self):
"""Custom sanity check for EasyBuild."""
# check whether easy-install.pth contains correct entries
easy_install_pth = os.path.join(self.installdir, self.pylibdir, 'easy-install.pth')
if os.path.exists(easy_install_pth):
easy_install_pth_txt = read_file(easy_install_pth)
for pkg in [p for p in self.easybuild_pkgs if p not in ['setuptools', 'vsc-install']]:
if pkg == 'vsc-base':
# don't include strict version check for vsc-base
pkg_regex = re.compile(r"^\./%s" % pkg.replace('-', '_'), re.M)
else:
major_minor_version = '.'.join(self.version.split('.')[:2])
pkg_regex = re.compile(r"^\./%s-%s" % (pkg.replace('-', '_'), major_minor_version), re.M)
if not pkg_regex.search(easy_install_pth_txt):
raise EasyBuildError("Failed to find pattern '%s' in %s: %s",
pkg_regex.pattern, easy_install_pth, easy_install_pth_txt)
# list of dirs to check, by package
# boolean indicates whether dir is expected to reside in Python lib/pythonX/site-packages dir
subdirs_by_pkg = {
'easybuild-framework': [('easybuild/framework', True), ('easybuild/tools', True)],
'easybuild-easyblocks': [('easybuild/easyblocks', True)],
'easybuild-easyconfigs': [('easybuild/easyconfigs', False)],
}
if LooseVersion(self.version) >= LooseVersion('2.0'):
subdirs_by_pkg.update({
'vsc-base': [('vsc/utils', True)],
})
# final list of directories to check, by setup tool
# order matters, e.g. setuptools before distutils
eb_dirs = OrderedDict()
eb_dirs['setuptools'] = []
eb_dirs['distutils.core'] = flatten([x for x in subdirs_by_pkg.values()])
# determine setup tool (setuptools or distutils)
setup_tool = None
for tool in eb_dirs.keys():
self.log.debug("Trying %s.." % tool)
try:
exec "from %s import setup" % tool
del setup
setup_tool = tool
break
except ImportError:
pass
self.log.debug('setup_tool: %s' % setup_tool)
# for a setuptools installation, we need to figure out the egg dirs,
# since we don't know the individual package versions
if setup_tool == 'setuptools':
try:
installed_dirs = os.listdir(os.path.join(self.installdir, self.pylibdir))
for (pkg, subdirs) in subdirs_by_pkg.items():
sel_dirs = [x for x in installed_dirs if x.startswith(pkg.replace('-', '_'))]
if not len(sel_dirs) == 1:
raise EasyBuildError("Failed to isolate installed egg dir for %s", pkg)
for (subdir, _) in subdirs:
# eggs always go in Python lib/pythonX/site-packages dir with setuptools
eb_dirs['setuptools'].append((os.path.join(sel_dirs[0], subdir), True))
except OSError as err:
raise EasyBuildError("Failed to determine sanity check dir paths: %s", err)
# set of sanity check paths to check for EasyBuild
custom_paths = {
'files': ['bin/eb'],
'dirs': [self.pylibdir] + [[x, os.path.join(self.pylibdir, x)][y] for (x, y) in eb_dirs[setup_tool]],
}
# make sure we don't trip over deprecated behavior in old EasyBuild versions
eb_cmd = 'eb'
if LooseVersion(self.version) <= LooseVersion('1.16.0'):
eb_cmd = 'EASYBUILD_DEPRECATED=1.0 eb'
# set of sanity check commands to run for EasyBuild
custom_commands = [
# this may spit out a wrong version, but that should be safe to ignore
# occurs when the EasyBuild being used is newer than the EasyBuild being installed
(eb_cmd, '--version'),
(eb_cmd, '-a'),
(eb_cmd, '-e ConfigureMake -a'),
]
# (temporary) cleanse copy of initial environment to avoid conflict with (potentially) loaded EasyBuild module
self.real_initial_environ = copy.deepcopy(self.initial_environ)
for env_var in ['_LMFILES_', 'LOADEDMODULES']:
if env_var in self.initial_environ:
self.initial_environ.pop(env_var)
os.environ.pop(env_var)
self.log.debug("Unset $%s in current env and copy of original env to make sanity check work" % env_var)
super(EB_EasyBuildMeta, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands)
def check_conflicts(easyconfigs, modtool, check_inter_ec_conflicts=True):
"""
Check for conflicts in dependency graphs for specified easyconfigs.
:param easyconfigs: list of easyconfig files (EasyConfig instances) to check for conflicts
:param modtool: ModulesTool instance to use
:param check_inter_ec_conflicts: also check for conflicts between (dependencies of) listed easyconfigs
:return: True if one or more conflicts were found, False otherwise
"""
ordered_ecs = resolve_dependencies(easyconfigs,
modtool,
retain_all_deps=True)
def mk_key(spec):
"""Create key for dictionary with all dependencies."""
if 'ec' in spec:
spec = spec['ec']
return (spec['name'], det_full_ec_version(spec))
# determine whether any 'wrappers' are involved
wrapper_deps = {}
for ec in ordered_ecs:
# easyconfigs using ModuleRC install a 'wrapper' for their dependency
# these need to be filtered out to avoid reporting false conflicts...
if ec['ec']['easyblock'] == 'ModuleRC':
wrapper_deps[mk_key(ec)] = mk_key(ec['ec']['dependencies'][0])
def mk_dep_keys(deps):
"""Create keys for given list of dependencies."""
res = []
for dep in deps:
# filter out dependencies marked as external module
if not dep.get('external_module', False):
key = mk_key(dep)
# replace 'wrapper' dependencies with the dependency they're wrapping
if key in wrapper_deps:
key = wrapper_deps[key]
res.append(key)
return res
# construct a dictionary: (name, installver) tuple to (build) dependencies
deps_for, dep_of = {}, {}
for node in ordered_ecs:
node_key = mk_key(node)
parsed_build_deps = node['ec'].builddependencies()
# take into account listed multi-deps;
# these will be included in the list of build dependencies (see EasyConfig.handle_multi_deps),
# but should be filtered out since they're not real build dependencies
# we need to iterate over them when checking for conflicts...
if node['ec']['multi_deps']:
parsed_multi_deps = node['ec'].get_parsed_multi_deps()
parsed_build_deps = [
d for d in parsed_build_deps
if d not in flatten(parsed_multi_deps)
]
else:
parsed_multi_deps = []
# exclude external modules, since we can't check conflicts on them (we don't even know the software name)
multi_deps = [mk_dep_keys(x) for x in parsed_multi_deps]
build_deps = mk_dep_keys(parsed_build_deps)
deps = mk_dep_keys(node['ec'].all_dependencies)
# separate runtime deps from build deps & multi deps
runtime_deps = [
d for d in deps
if d not in build_deps and d not in flatten(multi_deps)
]
deps_for[node_key] = (build_deps, runtime_deps, multi_deps)
# keep track of reverse deps too
for dep in deps + flatten(multi_deps):
dep_of.setdefault(dep, set()).add(node_key)
if check_inter_ec_conflicts:
# add ghost entry that depends on each of the specified easyconfigs,
# since we want to check for conflicts between specified easyconfigs too;
# 'wrapper' easyconfigs are not included to avoid false conflicts being reported
ec_keys = [
k for k in [mk_key(e) for e in easyconfigs]
if k not in wrapper_deps
]
deps_for[(None, None)] = ([], ec_keys, [])
# iteratively expand list of dependencies
last_deps_for = None
while deps_for != last_deps_for:
last_deps_for = copy.deepcopy(deps_for)
# (Automake, _), [], [(Autoconf, _), (GCC, _)]
for (key, (build_deps, runtime_deps,
multi_deps)) in last_deps_for.items():
# extend runtime dependencies with non-build dependencies of own runtime dependencies
# Autoconf
for dep in runtime_deps:
# [], [M4, GCC]
deps_for[key][1].extend(deps_for[dep][1])
# extend multi deps with non-build dependencies of own runtime dependencies
for deplist in multi_deps:
for dep in deplist:
deps_for[key][2].extend(deps_for[dep][1])
# extend build dependencies with non-build dependencies of own build dependencies
for dep in build_deps:
deps_for[key][0].extend(deps_for[dep][1])
deps_for[key] = (sorted(nub(deps_for[key][0])),
sorted(nub(deps_for[key][1])), multi_deps)
# also track reverse deps (except for ghost entry)
if key != (None, None):
for dep in build_deps + runtime_deps:
dep_of.setdefault(dep, set()).add(key)
def check_conflict(parent, dep1, dep2):
"""
Check whether dependencies with given name/(install) version conflict with each other.
:param parent: name & install version of 'parent' software
:param dep1: name & install version of 1st dependency
:param dep2: name & install version of 2nd dependency
"""
# dependencies with the same name should have the exact same install version
# if not => CONFLICT!
conflict = dep1[0] == dep2[0] and dep1[1] != dep2[1]
if conflict:
vs_msg = "%s-%s vs %s-%s " % (dep1 + dep2)
for dep in [dep1, dep2]:
if dep in dep_of:
vs_msg += "\n\t%s-%s as dep of: " % dep + ', '.join(
'%s-%s' % d for d in sorted(dep_of[dep]))
if parent[0] is None:
sys.stderr.write(
"Conflict between (dependencies of) easyconfigs: %s\n" %
vs_msg)
else:
specname = '%s-%s' % parent
sys.stderr.write(
"Conflict found for dependencies of %s: %s\n" %
(specname, vs_msg))
return conflict
# for each of the easyconfigs, check whether the dependencies (incl. build deps) contain any conflicts
res = False
for (key, (build_deps, runtime_deps, multi_deps)) in deps_for.items():
# determine lists of runtime deps to iterate over
# only if multi_deps is used will we actually have more than one list of runtime deps...
if multi_deps:
lists_of_runtime_deps = [runtime_deps + x for x in multi_deps]
else:
lists_of_runtime_deps = [runtime_deps]
for runtime_deps in lists_of_runtime_deps:
# also check whether module itself clashes with any of its dependencies
for i, dep1 in enumerate(build_deps + runtime_deps + [key]):
for dep2 in (build_deps + runtime_deps)[i + 1:]:
# don't worry about conflicts between module itself and any of its build deps
if dep1 != key or dep2 not in build_deps:
res |= check_conflict(key, dep1, dep2)
return res
def sanity_check_step(self):
"""Custom sanity check for EasyBuild."""
# check whether easy-install.pth contains correct entries
easy_install_pth = os.path.join(self.installdir, self.pylibdir, 'easy-install.pth')
if os.path.exists(easy_install_pth):
easy_install_pth_txt = read_file(easy_install_pth)
for pkg in self.easybuild_pkgs:
if pkg == 'vsc-base':
# don't include strict version check for vsc-base
pkg_regex = re.compile(r"^\./%s" % pkg.replace('-', '_'), re.M)
else:
major_minor_version = '.'.join(self.version.split('.')[:2])
pkg_regex = re.compile(r"^\./%s-%s" % (pkg.replace('-', '_'), major_minor_version), re.M)
if not pkg_regex.search(easy_install_pth_txt):
raise EasyBuildError("Failed to find pattern '%s' in %s: %s",
pkg_regex.pattern, easy_install_pth, easy_install_pth_txt)
# list of dirs to check, by package
# boolean indicates whether dir is expected to reside in Python lib/pythonX/site-packages dir
subdirs_by_pkg = {
'easybuild-framework': [('easybuild/framework', True), ('easybuild/tools', True)],
'easybuild-easyblocks': [('easybuild/easyblocks', True)],
'easybuild-easyconfigs': [('easybuild/easyconfigs', False)],
}
if LooseVersion(self.version) >= LooseVersion('2.0'):
subdirs_by_pkg.update({
'vsc-base': [('vsc/utils', True)],
})
# final list of directories to check, by setup tool
# order matters, e.g. setuptools before distutils
eb_dirs = OrderedDict()
eb_dirs['setuptools'] = []
eb_dirs['distutils.core'] = flatten([x for x in subdirs_by_pkg.values()])
# determine setup tool (setuptools or distutils)
setup_tool = None
for tool in eb_dirs.keys():
self.log.debug("Trying %s.." % tool)
try:
exec "from %s import setup" % tool
del setup
setup_tool = tool
break
except ImportError:
pass
self.log.debug('setup_tool: %s' % setup_tool)
# for a setuptools installation, we need to figure out the egg dirs,
# since we don't know the individual package versions
if setup_tool == 'setuptools':
try:
installed_dirs = os.listdir(os.path.join(self.installdir, self.pylibdir))
for (pkg, subdirs) in subdirs_by_pkg.items():
sel_dirs = [x for x in installed_dirs if x.startswith(pkg.replace('-', '_'))]
if not len(sel_dirs) == 1:
raise EasyBuildError("Failed to isolate installed egg dir for %s", pkg)
for (subdir, _) in subdirs:
# eggs always go in Python lib/pythonX/site-packages dir with setuptools
eb_dirs['setuptools'].append((os.path.join(sel_dirs[0], subdir), True))
except OSError, err:
raise EasyBuildError("Failed to determine sanity check dir paths: %s", err)
def check_conflicts(easyconfigs, modtool, check_inter_ec_conflicts=True):
"""
Check for conflicts in dependency graphs for specified easyconfigs.
:param easyconfigs: list of easyconfig files (EasyConfig instances) to check for conflicts
:param modtool: ModulesTool instance to use
:param check_inter_ec_conflicts: also check for conflicts between (dependencies of) listed easyconfigs
:return: True if one or more conflicts were found, False otherwise
"""
ordered_ecs = resolve_dependencies(easyconfigs, modtool, retain_all_deps=True)
def mk_key(spec):
"""Create key for dictionary with all dependencies."""
if 'ec' in spec:
spec = spec['ec']
return (spec['name'], det_full_ec_version(spec))
# determine whether any 'wrappers' are involved
wrapper_deps = {}
for ec in ordered_ecs:
# easyconfigs using ModuleRC install a 'wrapper' for their dependency
# these need to be filtered out to avoid reporting false conflicts...
if ec['ec']['easyblock'] == 'ModuleRC':
wrapper_deps[mk_key(ec)] = mk_key(ec['ec']['dependencies'][0])
def mk_dep_keys(deps):
"""Create keys for given list of dependencies."""
res = []
for dep in deps:
# filter out dependencies marked as external module
if not dep.get('external_module', False):
key = mk_key(dep)
# replace 'wrapper' dependencies with the dependency they're wrapping
if key in wrapper_deps:
key = wrapper_deps[key]
res.append(key)
return res
# construct a dictionary: (name, installver) tuple to (build) dependencies
deps_for, dep_of = {}, {}
for node in ordered_ecs:
node_key = mk_key(node)
parsed_build_deps = node['ec'].builddependencies()
# take into account listed multi-deps;
# these will be included in the list of build dependencies (see EasyConfig.handle_multi_deps),
# but should be filtered out since they're not real build dependencies
# we need to iterate over them when checking for conflicts...
if node['ec']['multi_deps']:
parsed_multi_deps = node['ec'].get_parsed_multi_deps()
parsed_build_deps = [d for d in parsed_build_deps if d not in flatten(parsed_multi_deps)]
else:
parsed_multi_deps = []
# exclude external modules, since we can't check conflicts on them (we don't even know the software name)
multi_deps = [mk_dep_keys(x) for x in parsed_multi_deps]
build_deps = mk_dep_keys(parsed_build_deps)
deps = mk_dep_keys(node['ec'].all_dependencies)
# separate runtime deps from build deps & multi deps
runtime_deps = [d for d in deps if d not in build_deps and d not in flatten(multi_deps)]
deps_for[node_key] = (build_deps, runtime_deps, multi_deps)
# keep track of reverse deps too
for dep in deps + flatten(multi_deps):
dep_of.setdefault(dep, set()).add(node_key)
if check_inter_ec_conflicts:
# add ghost entry that depends on each of the specified easyconfigs,
# since we want to check for conflicts between specified easyconfigs too;
# 'wrapper' easyconfigs are not included to avoid false conflicts being reported
ec_keys = [k for k in [mk_key(e) for e in easyconfigs] if k not in wrapper_deps]
deps_for[(None, None)] = ([], ec_keys, [])
# iteratively expand list of dependencies
last_deps_for = None
while deps_for != last_deps_for:
last_deps_for = copy.deepcopy(deps_for)
# (Automake, _), [], [(Autoconf, _), (GCC, _)]
for (key, (build_deps, runtime_deps, multi_deps)) in last_deps_for.items():
# extend runtime dependencies with non-build dependencies of own runtime dependencies
# Autoconf
for dep in runtime_deps:
# [], [M4, GCC]
deps_for[key][1].extend(deps_for[dep][1])
# extend multi deps with non-build dependencies of own runtime dependencies
for deplist in multi_deps:
for dep in deplist:
deps_for[key][2].extend(deps_for[dep][1])
# extend build dependencies with non-build dependencies of own build dependencies
for dep in build_deps:
deps_for[key][0].extend(deps_for[dep][1])
deps_for[key] = (sorted(nub(deps_for[key][0])), sorted(nub(deps_for[key][1])), multi_deps)
# also track reverse deps (except for ghost entry)
if key != (None, None):
for dep in build_deps + runtime_deps:
dep_of.setdefault(dep, set()).add(key)
def check_conflict(parent, dep1, dep2):
"""
Check whether dependencies with given name/(install) version conflict with each other.
:param parent: name & install version of 'parent' software
:param dep1: name & install version of 1st dependency
:param dep2: name & install version of 2nd dependency
"""
# dependencies with the same name should have the exact same install version
# if not => CONFLICT!
conflict = dep1[0] == dep2[0] and dep1[1] != dep2[1]
if conflict:
vs_msg = "%s-%s vs %s-%s " % (dep1 + dep2)
for dep in [dep1, dep2]:
if dep in dep_of:
vs_msg += "\n\t%s-%s as dep of: " % dep + ', '.join('%s-%s' % d for d in sorted(dep_of[dep]))
if parent[0] is None:
sys.stderr.write("Conflict between (dependencies of) easyconfigs: %s\n" % vs_msg)
else:
specname = '%s-%s' % parent
sys.stderr.write("Conflict found for dependencies of %s: %s\n" % (specname, vs_msg))
return conflict
# for each of the easyconfigs, check whether the dependencies (incl. build deps) contain any conflicts
res = False
for (key, (build_deps, runtime_deps, multi_deps)) in deps_for.items():
# determine lists of runtime deps to iterate over
# only if multi_deps is used will we actually have more than one list of runtime deps...
if multi_deps:
lists_of_runtime_deps = [runtime_deps + x for x in multi_deps]
else:
lists_of_runtime_deps = [runtime_deps]
for runtime_deps in lists_of_runtime_deps:
# also check whether module itself clashes with any of its dependencies
for i, dep1 in enumerate(build_deps + runtime_deps + [key]):
for dep2 in (build_deps + runtime_deps)[i+1:]:
# don't worry about conflicts between module itself and any of its build deps
if dep1 != key or dep2 not in build_deps:
res |= check_conflict(key, dep1, dep2)
return res