def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir=None): """ Take an easyconfig spec, parse it, map it to a target toolchain and dump it out :param ec_spec: Location of original easyconfig file :param toolchain_mapping: Mapping between source toolchain and target toolchain :param targetdir: Directory to dump the modified easyconfig file in :return: Location of the modified easyconfig file """ # Fully parse the original easyconfig parsed_ec = process_easyconfig(ec_spec, validate=False)[0]['ec'] # Replace the toolchain if the mapping exists tc_name = parsed_ec['toolchain']['name'] if tc_name in toolchain_mapping: new_toolchain = toolchain_mapping[tc_name] _log.debug("Replacing parent toolchain %s with %s", parsed_ec['toolchain'], new_toolchain) parsed_ec['toolchain'] = new_toolchain # Replace the toolchains of all the dependencies for key in DEPENDENCY_PARAMETERS: # loop over a *copy* of dependency dicts (with resolved templates); # to update the original dep dict, we need to get a reference with templating disabled... val = parsed_ec[key] orig_val = parsed_ec.get_ref(key) if key in parsed_ec.iterate_options: val = flatten(val) orig_val = flatten(orig_val) for idx, dep in enumerate(val): # reference to original dep dict, this is the one we should be updating orig_dep = orig_val[idx] # skip dependencies that are marked as external modules if dep['external_module']: continue dep_tc_name = dep['toolchain']['name'] if dep_tc_name in toolchain_mapping: orig_dep['toolchain'] = toolchain_mapping[dep_tc_name] # Replace the binutils version (if necessary) if 'binutils' in toolchain_mapping and (dep['name'] == 'binutils' and dep_tc_name == GCCcore.NAME): orig_dep.update(toolchain_mapping['binutils']) # set module names orig_dep['short_mod_name'] = ActiveMNS().det_short_module_name(dep) orig_dep['full_mod_name'] = ActiveMNS().det_full_module_name(dep) # Determine the name of the modified easyconfig and dump it to target_dir ec_filename = '%s-%s.eb' % (parsed_ec['name'], det_full_ec_version(parsed_ec)) tweaked_spec = os.path.join(targetdir or tempfile.gettempdir(), ec_filename) parsed_ec.dump(tweaked_spec, always_overwrite=False, backup=True) _log.debug("Dumped easyconfig tweaked via --try-toolchain* to %s", tweaked_spec) return tweaked_spec
def sanity_check_step(self): """Custom sanity check for EasyBuild.""" # list of dirs to check, by package # boolean indicates whether dir is expected to reside in Python lib/pythonX/site-packages dir subdirs_by_pkg = [ ('framework', [('easybuild/framework', True), ('easybuild/tools', True)]), ('easyblocks', [('easybuild/easyblocks', True)]), ('easyconfigs', [('easybuild/easyconfigs', False)]), ] # final list of directories to check, by setup tool # order matters, e.g. setuptools before distutils eb_dirs = OrderedDict() eb_dirs['setuptools'] = [] eb_dirs['distutils.core'] = flatten([x[1] for x in subdirs_by_pkg]) # determine setup tool (setuptools or distutils) setup_tool = None for tool in eb_dirs.keys(): self.log.debug("Trying %s.." % tool) try: exec "from %s import setup" % tool del setup setup_tool = tool break except ImportError: pass self.log.debug('setup_tool: %s' % setup_tool) # for a setuptools installation, we need to figure out the egg dirs since we don't know the individual package versions if setup_tool == 'setuptools': try: installed_dirs = os.listdir( os.path.join(self.installdir, self.pylibdir)) for (pkg, subdirs) in subdirs_by_pkg: sel_dirs = [ x for x in installed_dirs if x.startswith('easybuild_%s' % pkg) ] if not len(sel_dirs) == 1: self.log.error( "Failed to isolate installed egg dir for easybuild-%s" % pkg) for (subdir, _) in subdirs: # eggs always go in Python lib/pythonX/site-packages dir with setuptools eb_dirs['setuptools'].append( (os.path.join(sel_dirs[0], subdir), True)) except OSError, err: self.log.error( "Failed to determine sanity check dir paths: %s" % err)
def sanity_check_step(self): """Custom sanity check for EasyBuild.""" # list of dirs to check, by package # boolean indicates whether dir is expected to reside in Python lib/pythonX/site-packages dir subdirs_by_pkg = [ ('framework', [('easybuild/framework', True), ('easybuild/tools', True)]), ('easyblocks', [('easybuild/easyblocks', True)]), ('easyconfigs', [('easybuild/easyconfigs', False)]), ] # final list of directories to check, by setup tool # order matters, e.g. setuptools before distutils eb_dirs = OrderedDict() eb_dirs['setuptools'] = [] eb_dirs['distutils.core'] = flatten([x[1] for x in subdirs_by_pkg]) # determine setup tool (setuptools or distutils) setup_tool = None for tool in eb_dirs.keys(): self.log.debug("Trying %s.." % tool) try: exec "from %s import setup" % tool del setup setup_tool = tool break except ImportError: pass self.log.debug('setup_tool: %s' % setup_tool) # for a setuptools installation, we need to figure out the egg dirs since we don't know the individual package versions if setup_tool == 'setuptools': try: installed_dirs = os.listdir(os.path.join(self.installdir, self.pylibdir)) for (pkg, subdirs) in subdirs_by_pkg: sel_dirs = [x for x in installed_dirs if x.startswith('easybuild_%s' % pkg)] if not len(sel_dirs) == 1: self.log.error("Failed to isolate installed egg dir for easybuild-%s" % pkg) for (subdir, _) in subdirs: # eggs always go in Python lib/pythonX/site-packages dir with setuptools eb_dirs['setuptools'].append((os.path.join(sel_dirs[0], subdir), True)) except OSError, err: self.log.error("Failed to determine sanity check dir paths: %s" % err)
def sanity_check_step(self): """Custom sanity check for EasyBuild.""" # check whether easy-install.pth contains correct entries easy_install_pth = os.path.join(self.installdir, self.pylibdir, 'easy-install.pth') if os.path.exists(easy_install_pth): easy_install_pth_txt = read_file(easy_install_pth) ignore_pkgs = ['setuptools', 'vsc-install'] if LooseVersion(self.version) > LooseVersion('3.999'): ignore_pkgs.append('vsc-base') for pkg in [ p for p in self.easybuild_pkgs if p not in ignore_pkgs ]: if pkg == 'vsc-base': # don't include strict version check for vsc-base pkg_regex = re.compile(r"^\./%s" % pkg.replace('-', '_'), re.M) else: major_minor_version = '.'.join(self.version.split('.')[:2]) pkg_regex = re.compile( r"^\./%s-%s" % (pkg.replace('-', '_'), major_minor_version), re.M) if not pkg_regex.search(easy_install_pth_txt): raise EasyBuildError( "Failed to find pattern '%s' in %s: %s", pkg_regex.pattern, easy_install_pth, easy_install_pth_txt) # list of dirs to check, by package # boolean indicates whether dir is expected to reside in Python lib/pythonX/site-packages dir subdirs_by_pkg = { 'easybuild-framework': [('easybuild/framework', True), ('easybuild/tools', True)], 'easybuild-easyblocks': [('easybuild/easyblocks', True)], 'easybuild-easyconfigs': [('easybuild/easyconfigs', False)], } if LooseVersion(self.version) >= LooseVersion('2.0') and LooseVersion( self.version) < LooseVersion('3.999'): subdirs_by_pkg.update({ 'vsc-base': [('vsc/utils', True)], }) # final list of directories to check, by setup tool # order matters, e.g. setuptools before distutils eb_dirs = OrderedDict() eb_dirs['setuptools'] = [] eb_dirs['distutils.core'] = flatten( [x for x in subdirs_by_pkg.values()]) # determine setup tool (setuptools or distutils) setup_tool = None for tool in eb_dirs.keys(): self.log.debug("Trying %s.." % tool) try: exec("from %s import setup" % tool) setup_tool = tool break except ImportError: pass self.log.debug('setup_tool: %s' % setup_tool) # for a setuptools installation, we need to figure out the egg dirs, # since we don't know the individual package versions if setup_tool == 'setuptools': try: installed_dirs = os.listdir( os.path.join(self.installdir, self.pylibdir)) for (pkg, subdirs) in subdirs_by_pkg.items(): sel_dirs = [ x for x in installed_dirs if x.startswith(pkg.replace('-', '_')) ] if not len(sel_dirs) == 1: raise EasyBuildError( "Failed to isolate installed egg dir for %s", pkg) for (subdir, _) in subdirs: # eggs always go in Python lib/pythonX/site-packages dir with setuptools eb_dirs['setuptools'].append( (os.path.join(sel_dirs[0], subdir), True)) except OSError as err: raise EasyBuildError( "Failed to determine sanity check dir paths: %s", err) # set of sanity check paths to check for EasyBuild custom_paths = { 'files': ['bin/eb'], 'dirs': [self.pylibdir] + [[x, os.path.join(self.pylibdir, x)][y] for (x, y) in eb_dirs[setup_tool]], } # make sure we don't trip over deprecated behavior in old EasyBuild versions eb_cmd = 'eb' if LooseVersion(self.version) <= LooseVersion('1.16.0'): eb_cmd = 'EASYBUILD_DEPRECATED=1.0 eb' # set of sanity check commands to run for EasyBuild custom_commands = [ # this may spit out a wrong version, but that should be safe to ignore # occurs when the EasyBuild being used is newer than the EasyBuild being installed (eb_cmd, '--version'), (eb_cmd, '-a'), (eb_cmd, '-e ConfigureMake -a'), ] # (temporary) cleanse copy of initial environment to avoid conflict with (potentially) loaded EasyBuild module self.real_initial_environ = copy.deepcopy(self.initial_environ) for env_var in ['_LMFILES_', 'LOADEDMODULES']: if env_var in self.initial_environ: self.initial_environ.pop(env_var) os.environ.pop(env_var) self.log.debug( "Unset $%s in current env and copy of original env to make sanity check work" % env_var) super(EB_EasyBuildMeta, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands)
def sanity_check_step(self): """Custom sanity check for EasyBuild.""" # check whether easy-install.pth contains correct entries easy_install_pth = os.path.join(self.installdir, self.pylibdir, 'easy-install.pth') if os.path.exists(easy_install_pth): easy_install_pth_txt = read_file(easy_install_pth) for pkg in [p for p in self.easybuild_pkgs if p not in ['setuptools', 'vsc-install']]: if pkg == 'vsc-base': # don't include strict version check for vsc-base pkg_regex = re.compile(r"^\./%s" % pkg.replace('-', '_'), re.M) else: major_minor_version = '.'.join(self.version.split('.')[:2]) pkg_regex = re.compile(r"^\./%s-%s" % (pkg.replace('-', '_'), major_minor_version), re.M) if not pkg_regex.search(easy_install_pth_txt): raise EasyBuildError("Failed to find pattern '%s' in %s: %s", pkg_regex.pattern, easy_install_pth, easy_install_pth_txt) # list of dirs to check, by package # boolean indicates whether dir is expected to reside in Python lib/pythonX/site-packages dir subdirs_by_pkg = { 'easybuild-framework': [('easybuild/framework', True), ('easybuild/tools', True)], 'easybuild-easyblocks': [('easybuild/easyblocks', True)], 'easybuild-easyconfigs': [('easybuild/easyconfigs', False)], } if LooseVersion(self.version) >= LooseVersion('2.0'): subdirs_by_pkg.update({ 'vsc-base': [('vsc/utils', True)], }) # final list of directories to check, by setup tool # order matters, e.g. setuptools before distutils eb_dirs = OrderedDict() eb_dirs['setuptools'] = [] eb_dirs['distutils.core'] = flatten([x for x in subdirs_by_pkg.values()]) # determine setup tool (setuptools or distutils) setup_tool = None for tool in eb_dirs.keys(): self.log.debug("Trying %s.." % tool) try: exec "from %s import setup" % tool del setup setup_tool = tool break except ImportError: pass self.log.debug('setup_tool: %s' % setup_tool) # for a setuptools installation, we need to figure out the egg dirs, # since we don't know the individual package versions if setup_tool == 'setuptools': try: installed_dirs = os.listdir(os.path.join(self.installdir, self.pylibdir)) for (pkg, subdirs) in subdirs_by_pkg.items(): sel_dirs = [x for x in installed_dirs if x.startswith(pkg.replace('-', '_'))] if not len(sel_dirs) == 1: raise EasyBuildError("Failed to isolate installed egg dir for %s", pkg) for (subdir, _) in subdirs: # eggs always go in Python lib/pythonX/site-packages dir with setuptools eb_dirs['setuptools'].append((os.path.join(sel_dirs[0], subdir), True)) except OSError as err: raise EasyBuildError("Failed to determine sanity check dir paths: %s", err) # set of sanity check paths to check for EasyBuild custom_paths = { 'files': ['bin/eb'], 'dirs': [self.pylibdir] + [[x, os.path.join(self.pylibdir, x)][y] for (x, y) in eb_dirs[setup_tool]], } # make sure we don't trip over deprecated behavior in old EasyBuild versions eb_cmd = 'eb' if LooseVersion(self.version) <= LooseVersion('1.16.0'): eb_cmd = 'EASYBUILD_DEPRECATED=1.0 eb' # set of sanity check commands to run for EasyBuild custom_commands = [ # this may spit out a wrong version, but that should be safe to ignore # occurs when the EasyBuild being used is newer than the EasyBuild being installed (eb_cmd, '--version'), (eb_cmd, '-a'), (eb_cmd, '-e ConfigureMake -a'), ] # (temporary) cleanse copy of initial environment to avoid conflict with (potentially) loaded EasyBuild module self.real_initial_environ = copy.deepcopy(self.initial_environ) for env_var in ['_LMFILES_', 'LOADEDMODULES']: if env_var in self.initial_environ: self.initial_environ.pop(env_var) os.environ.pop(env_var) self.log.debug("Unset $%s in current env and copy of original env to make sanity check work" % env_var) super(EB_EasyBuildMeta, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands)
def check_conflicts(easyconfigs, modtool, check_inter_ec_conflicts=True): """ Check for conflicts in dependency graphs for specified easyconfigs. :param easyconfigs: list of easyconfig files (EasyConfig instances) to check for conflicts :param modtool: ModulesTool instance to use :param check_inter_ec_conflicts: also check for conflicts between (dependencies of) listed easyconfigs :return: True if one or more conflicts were found, False otherwise """ ordered_ecs = resolve_dependencies(easyconfigs, modtool, retain_all_deps=True) def mk_key(spec): """Create key for dictionary with all dependencies.""" if 'ec' in spec: spec = spec['ec'] return (spec['name'], det_full_ec_version(spec)) # determine whether any 'wrappers' are involved wrapper_deps = {} for ec in ordered_ecs: # easyconfigs using ModuleRC install a 'wrapper' for their dependency # these need to be filtered out to avoid reporting false conflicts... if ec['ec']['easyblock'] == 'ModuleRC': wrapper_deps[mk_key(ec)] = mk_key(ec['ec']['dependencies'][0]) def mk_dep_keys(deps): """Create keys for given list of dependencies.""" res = [] for dep in deps: # filter out dependencies marked as external module if not dep.get('external_module', False): key = mk_key(dep) # replace 'wrapper' dependencies with the dependency they're wrapping if key in wrapper_deps: key = wrapper_deps[key] res.append(key) return res # construct a dictionary: (name, installver) tuple to (build) dependencies deps_for, dep_of = {}, {} for node in ordered_ecs: node_key = mk_key(node) parsed_build_deps = node['ec'].builddependencies() # take into account listed multi-deps; # these will be included in the list of build dependencies (see EasyConfig.handle_multi_deps), # but should be filtered out since they're not real build dependencies # we need to iterate over them when checking for conflicts... if node['ec']['multi_deps']: parsed_multi_deps = node['ec'].get_parsed_multi_deps() parsed_build_deps = [ d for d in parsed_build_deps if d not in flatten(parsed_multi_deps) ] else: parsed_multi_deps = [] # exclude external modules, since we can't check conflicts on them (we don't even know the software name) multi_deps = [mk_dep_keys(x) for x in parsed_multi_deps] build_deps = mk_dep_keys(parsed_build_deps) deps = mk_dep_keys(node['ec'].all_dependencies) # separate runtime deps from build deps & multi deps runtime_deps = [ d for d in deps if d not in build_deps and d not in flatten(multi_deps) ] deps_for[node_key] = (build_deps, runtime_deps, multi_deps) # keep track of reverse deps too for dep in deps + flatten(multi_deps): dep_of.setdefault(dep, set()).add(node_key) if check_inter_ec_conflicts: # add ghost entry that depends on each of the specified easyconfigs, # since we want to check for conflicts between specified easyconfigs too; # 'wrapper' easyconfigs are not included to avoid false conflicts being reported ec_keys = [ k for k in [mk_key(e) for e in easyconfigs] if k not in wrapper_deps ] deps_for[(None, None)] = ([], ec_keys, []) # iteratively expand list of dependencies last_deps_for = None while deps_for != last_deps_for: last_deps_for = copy.deepcopy(deps_for) # (Automake, _), [], [(Autoconf, _), (GCC, _)] for (key, (build_deps, runtime_deps, multi_deps)) in last_deps_for.items(): # extend runtime dependencies with non-build dependencies of own runtime dependencies # Autoconf for dep in runtime_deps: # [], [M4, GCC] deps_for[key][1].extend(deps_for[dep][1]) # extend multi deps with non-build dependencies of own runtime dependencies for deplist in multi_deps: for dep in deplist: deps_for[key][2].extend(deps_for[dep][1]) # extend build dependencies with non-build dependencies of own build dependencies for dep in build_deps: deps_for[key][0].extend(deps_for[dep][1]) deps_for[key] = (sorted(nub(deps_for[key][0])), sorted(nub(deps_for[key][1])), multi_deps) # also track reverse deps (except for ghost entry) if key != (None, None): for dep in build_deps + runtime_deps: dep_of.setdefault(dep, set()).add(key) def check_conflict(parent, dep1, dep2): """ Check whether dependencies with given name/(install) version conflict with each other. :param parent: name & install version of 'parent' software :param dep1: name & install version of 1st dependency :param dep2: name & install version of 2nd dependency """ # dependencies with the same name should have the exact same install version # if not => CONFLICT! conflict = dep1[0] == dep2[0] and dep1[1] != dep2[1] if conflict: vs_msg = "%s-%s vs %s-%s " % (dep1 + dep2) for dep in [dep1, dep2]: if dep in dep_of: vs_msg += "\n\t%s-%s as dep of: " % dep + ', '.join( '%s-%s' % d for d in sorted(dep_of[dep])) if parent[0] is None: sys.stderr.write( "Conflict between (dependencies of) easyconfigs: %s\n" % vs_msg) else: specname = '%s-%s' % parent sys.stderr.write( "Conflict found for dependencies of %s: %s\n" % (specname, vs_msg)) return conflict # for each of the easyconfigs, check whether the dependencies (incl. build deps) contain any conflicts res = False for (key, (build_deps, runtime_deps, multi_deps)) in deps_for.items(): # determine lists of runtime deps to iterate over # only if multi_deps is used will we actually have more than one list of runtime deps... if multi_deps: lists_of_runtime_deps = [runtime_deps + x for x in multi_deps] else: lists_of_runtime_deps = [runtime_deps] for runtime_deps in lists_of_runtime_deps: # also check whether module itself clashes with any of its dependencies for i, dep1 in enumerate(build_deps + runtime_deps + [key]): for dep2 in (build_deps + runtime_deps)[i + 1:]: # don't worry about conflicts between module itself and any of its build deps if dep1 != key or dep2 not in build_deps: res |= check_conflict(key, dep1, dep2) return res
def sanity_check_step(self): """Custom sanity check for EasyBuild.""" # check whether easy-install.pth contains correct entries easy_install_pth = os.path.join(self.installdir, self.pylibdir, 'easy-install.pth') if os.path.exists(easy_install_pth): easy_install_pth_txt = read_file(easy_install_pth) for pkg in self.easybuild_pkgs: if pkg == 'vsc-base': # don't include strict version check for vsc-base pkg_regex = re.compile(r"^\./%s" % pkg.replace('-', '_'), re.M) else: major_minor_version = '.'.join(self.version.split('.')[:2]) pkg_regex = re.compile(r"^\./%s-%s" % (pkg.replace('-', '_'), major_minor_version), re.M) if not pkg_regex.search(easy_install_pth_txt): raise EasyBuildError("Failed to find pattern '%s' in %s: %s", pkg_regex.pattern, easy_install_pth, easy_install_pth_txt) # list of dirs to check, by package # boolean indicates whether dir is expected to reside in Python lib/pythonX/site-packages dir subdirs_by_pkg = { 'easybuild-framework': [('easybuild/framework', True), ('easybuild/tools', True)], 'easybuild-easyblocks': [('easybuild/easyblocks', True)], 'easybuild-easyconfigs': [('easybuild/easyconfigs', False)], } if LooseVersion(self.version) >= LooseVersion('2.0'): subdirs_by_pkg.update({ 'vsc-base': [('vsc/utils', True)], }) # final list of directories to check, by setup tool # order matters, e.g. setuptools before distutils eb_dirs = OrderedDict() eb_dirs['setuptools'] = [] eb_dirs['distutils.core'] = flatten([x for x in subdirs_by_pkg.values()]) # determine setup tool (setuptools or distutils) setup_tool = None for tool in eb_dirs.keys(): self.log.debug("Trying %s.." % tool) try: exec "from %s import setup" % tool del setup setup_tool = tool break except ImportError: pass self.log.debug('setup_tool: %s' % setup_tool) # for a setuptools installation, we need to figure out the egg dirs, # since we don't know the individual package versions if setup_tool == 'setuptools': try: installed_dirs = os.listdir(os.path.join(self.installdir, self.pylibdir)) for (pkg, subdirs) in subdirs_by_pkg.items(): sel_dirs = [x for x in installed_dirs if x.startswith(pkg.replace('-', '_'))] if not len(sel_dirs) == 1: raise EasyBuildError("Failed to isolate installed egg dir for %s", pkg) for (subdir, _) in subdirs: # eggs always go in Python lib/pythonX/site-packages dir with setuptools eb_dirs['setuptools'].append((os.path.join(sel_dirs[0], subdir), True)) except OSError, err: raise EasyBuildError("Failed to determine sanity check dir paths: %s", err)
def check_conflicts(easyconfigs, modtool, check_inter_ec_conflicts=True): """ Check for conflicts in dependency graphs for specified easyconfigs. :param easyconfigs: list of easyconfig files (EasyConfig instances) to check for conflicts :param modtool: ModulesTool instance to use :param check_inter_ec_conflicts: also check for conflicts between (dependencies of) listed easyconfigs :return: True if one or more conflicts were found, False otherwise """ ordered_ecs = resolve_dependencies(easyconfigs, modtool, retain_all_deps=True) def mk_key(spec): """Create key for dictionary with all dependencies.""" if 'ec' in spec: spec = spec['ec'] return (spec['name'], det_full_ec_version(spec)) # determine whether any 'wrappers' are involved wrapper_deps = {} for ec in ordered_ecs: # easyconfigs using ModuleRC install a 'wrapper' for their dependency # these need to be filtered out to avoid reporting false conflicts... if ec['ec']['easyblock'] == 'ModuleRC': wrapper_deps[mk_key(ec)] = mk_key(ec['ec']['dependencies'][0]) def mk_dep_keys(deps): """Create keys for given list of dependencies.""" res = [] for dep in deps: # filter out dependencies marked as external module if not dep.get('external_module', False): key = mk_key(dep) # replace 'wrapper' dependencies with the dependency they're wrapping if key in wrapper_deps: key = wrapper_deps[key] res.append(key) return res # construct a dictionary: (name, installver) tuple to (build) dependencies deps_for, dep_of = {}, {} for node in ordered_ecs: node_key = mk_key(node) parsed_build_deps = node['ec'].builddependencies() # take into account listed multi-deps; # these will be included in the list of build dependencies (see EasyConfig.handle_multi_deps), # but should be filtered out since they're not real build dependencies # we need to iterate over them when checking for conflicts... if node['ec']['multi_deps']: parsed_multi_deps = node['ec'].get_parsed_multi_deps() parsed_build_deps = [d for d in parsed_build_deps if d not in flatten(parsed_multi_deps)] else: parsed_multi_deps = [] # exclude external modules, since we can't check conflicts on them (we don't even know the software name) multi_deps = [mk_dep_keys(x) for x in parsed_multi_deps] build_deps = mk_dep_keys(parsed_build_deps) deps = mk_dep_keys(node['ec'].all_dependencies) # separate runtime deps from build deps & multi deps runtime_deps = [d for d in deps if d not in build_deps and d not in flatten(multi_deps)] deps_for[node_key] = (build_deps, runtime_deps, multi_deps) # keep track of reverse deps too for dep in deps + flatten(multi_deps): dep_of.setdefault(dep, set()).add(node_key) if check_inter_ec_conflicts: # add ghost entry that depends on each of the specified easyconfigs, # since we want to check for conflicts between specified easyconfigs too; # 'wrapper' easyconfigs are not included to avoid false conflicts being reported ec_keys = [k for k in [mk_key(e) for e in easyconfigs] if k not in wrapper_deps] deps_for[(None, None)] = ([], ec_keys, []) # iteratively expand list of dependencies last_deps_for = None while deps_for != last_deps_for: last_deps_for = copy.deepcopy(deps_for) # (Automake, _), [], [(Autoconf, _), (GCC, _)] for (key, (build_deps, runtime_deps, multi_deps)) in last_deps_for.items(): # extend runtime dependencies with non-build dependencies of own runtime dependencies # Autoconf for dep in runtime_deps: # [], [M4, GCC] deps_for[key][1].extend(deps_for[dep][1]) # extend multi deps with non-build dependencies of own runtime dependencies for deplist in multi_deps: for dep in deplist: deps_for[key][2].extend(deps_for[dep][1]) # extend build dependencies with non-build dependencies of own build dependencies for dep in build_deps: deps_for[key][0].extend(deps_for[dep][1]) deps_for[key] = (sorted(nub(deps_for[key][0])), sorted(nub(deps_for[key][1])), multi_deps) # also track reverse deps (except for ghost entry) if key != (None, None): for dep in build_deps + runtime_deps: dep_of.setdefault(dep, set()).add(key) def check_conflict(parent, dep1, dep2): """ Check whether dependencies with given name/(install) version conflict with each other. :param parent: name & install version of 'parent' software :param dep1: name & install version of 1st dependency :param dep2: name & install version of 2nd dependency """ # dependencies with the same name should have the exact same install version # if not => CONFLICT! conflict = dep1[0] == dep2[0] and dep1[1] != dep2[1] if conflict: vs_msg = "%s-%s vs %s-%s " % (dep1 + dep2) for dep in [dep1, dep2]: if dep in dep_of: vs_msg += "\n\t%s-%s as dep of: " % dep + ', '.join('%s-%s' % d for d in sorted(dep_of[dep])) if parent[0] is None: sys.stderr.write("Conflict between (dependencies of) easyconfigs: %s\n" % vs_msg) else: specname = '%s-%s' % parent sys.stderr.write("Conflict found for dependencies of %s: %s\n" % (specname, vs_msg)) return conflict # for each of the easyconfigs, check whether the dependencies (incl. build deps) contain any conflicts res = False for (key, (build_deps, runtime_deps, multi_deps)) in deps_for.items(): # determine lists of runtime deps to iterate over # only if multi_deps is used will we actually have more than one list of runtime deps... if multi_deps: lists_of_runtime_deps = [runtime_deps + x for x in multi_deps] else: lists_of_runtime_deps = [runtime_deps] for runtime_deps in lists_of_runtime_deps: # also check whether module itself clashes with any of its dependencies for i, dep1 in enumerate(build_deps + runtime_deps + [key]): for dep2 in (build_deps + runtime_deps)[i+1:]: # don't worry about conflicts between module itself and any of its build deps if dep1 != key or dep2 not in build_deps: res |= check_conflict(key, dep1, dep2) return res