def tweak(easyconfigs, build_specs, targetdir=None): """Tweak list of easyconfigs according to provided build specifications.""" # make sure easyconfigs all feature the same toolchain (otherwise we *will* run into trouble) toolchains = nub( ['%(name)s/%(version)s' % ec['ec']['toolchain'] for ec in easyconfigs]) if len(toolchains) > 1: _log.error( "Multiple toolchains featured in easyconfigs, --try-X not supported in that case: %s" % toolchains) if 'name' in build_specs or 'version' in build_specs: # no recursion if software name/version build specification are included # in that case, do not construct full dependency graph orig_ecs = easyconfigs _log.debug( "Software name/version found, so not applying build specifications recursively: %s" % build_specs) else: # build specifications should be applied to the whole dependency graph # obtain full dependency graph for specified easyconfigs # easyconfigs will be ordered 'top-to-bottom': toolchain dependencies and toolchain first _log.debug( "Applying build specifications recursively (no software name/version found): %s" % build_specs) orig_ecs = resolve_dependencies(easyconfigs, retain_all_deps=True) # keep track of originally listed easyconfigs (via their path) listed_ec_paths = [ec['spec'] for ec in easyconfigs] # obtain full dependency graph for specified easyconfigs # easyconfigs will be ordered 'top-to-bottom': toolchain dependencies and toolchain first orig_ecs = resolve_dependencies(easyconfigs, retain_all_deps=True) # determine toolchain based on last easyconfigs toolchain = orig_ecs[-1]['ec']['toolchain'] _log.debug("Filtering using toolchain %s" % toolchain) # filter easyconfigs unless a dummy toolchain is used: drop toolchain and toolchain dependencies if toolchain['name'] != DUMMY_TOOLCHAIN_NAME: while orig_ecs[0]['ec']['toolchain'] != toolchain: orig_ecs = orig_ecs[1:] # generate tweaked easyconfigs, and continue with those instead tweaked_easyconfigs = [] for orig_ec in orig_ecs: new_ec_file = tweak_one(orig_ec['spec'], None, build_specs, targetdir=targetdir) # only return tweaked easyconfigs for easyconfigs which were listed originally # easyconfig files for dependencies are also generated but not included, and will be resolved via --robot if orig_ec['spec'] in listed_ec_paths: new_ecs = process_easyconfig(new_ec_file, build_specs=build_specs) tweaked_easyconfigs.extend(new_ecs) return tweaked_easyconfigs
def tweak(easyconfigs, build_specs): """Tweak list of easyconfigs according to provided build specifications.""" # make sure easyconfigs all feature the same toolchain (otherwise we *will* run into trouble) toolchains = nub( ['%(name)s/%(version)s' % ec['ec']['toolchain'] for ec in easyconfigs]) if len(toolchains) > 1: _log.error( "Multiple toolchains featured in easyconfigs, --try-X not supported in that case: %s" % toolchains) # obtain full dependency graph for specified easyconfigs # easyconfigs will be ordered 'top-to-bottom': toolchain dependencies and toolchain first orig_ecs = resolve_dependencies(easyconfigs, retain_all_deps=True) # determine toolchain based on last easyconfigs toolchain = orig_ecs[-1]['ec']['toolchain'] _log.debug("Filtering using toolchain %s" % toolchain) # filter easyconfigs unless a dummy toolchain is used: drop toolchain and toolchain dependencies if toolchain['name'] != DUMMY_TOOLCHAIN_NAME: while orig_ecs[0]['ec']['toolchain'] != toolchain: orig_ecs = orig_ecs[1:] # generate tweaked easyconfigs, and continue with those instead easyconfigs = [] for orig_ec in orig_ecs: new_ec_file = tweak_one(orig_ec['spec'], None, build_specs) new_ecs = process_easyconfig(new_ec_file, build_specs=build_specs) easyconfigs.extend(new_ecs) return easyconfigs
def tweak(easyconfigs, build_specs): """Tweak list of easyconfigs according to provided build specifications.""" # make sure easyconfigs all feature the same toolchain (otherwise we *will* run into trouble) toolchains = nub(['%(name)s/%(version)s' % ec['ec']['toolchain'] for ec in easyconfigs]) if len(toolchains) > 1: _log.error("Multiple toolchains featured in easyconfigs, --try-X not supported in that case: %s" % toolchains) # obtain full dependency graph for specified easyconfigs # easyconfigs will be ordered 'top-to-bottom': toolchain dependencies and toolchain first orig_ecs = resolve_dependencies(easyconfigs, retain_all_deps=True) # determine toolchain based on last easyconfigs toolchain = orig_ecs[-1]['ec']['toolchain'] _log.debug("Filtering using toolchain %s" % toolchain) # filter easyconfigs unless a dummy toolchain is used: drop toolchain and toolchain dependencies if toolchain['name'] != DUMMY_TOOLCHAIN_NAME: while orig_ecs[0]['ec']['toolchain'] != toolchain: orig_ecs = orig_ecs[1:] # generate tweaked easyconfigs, and continue with those instead easyconfigs = [] for orig_ec in orig_ecs: new_ec_file = tweak_one(orig_ec['spec'], None, build_specs) new_ecs = process_easyconfig(new_ec_file, build_specs=build_specs) easyconfigs.extend(new_ecs) return easyconfigs
def tweak(easyconfigs, build_specs, targetdir=None): """Tweak list of easyconfigs according to provided build specifications.""" # make sure easyconfigs all feature the same toolchain (otherwise we *will* run into trouble) toolchains = nub(['%(name)s/%(version)s' % ec['ec']['toolchain'] for ec in easyconfigs]) if len(toolchains) > 1: _log.error("Multiple toolchains featured in easyconfigs, --try-X not supported in that case: %s" % toolchains) if 'name' in build_specs or 'version' in build_specs: # no recursion if software name/version build specification are included # in that case, do not construct full dependency graph orig_ecs = easyconfigs _log.debug("Software name/version found, so not applying build specifications recursively: %s" % build_specs) else: # build specifications should be applied to the whole dependency graph # obtain full dependency graph for specified easyconfigs # easyconfigs will be ordered 'top-to-bottom': toolchain dependencies and toolchain first _log.debug("Applying build specifications recursively (no software name/version found): %s" % build_specs) orig_ecs = resolve_dependencies(easyconfigs, retain_all_deps=True) # keep track of originally listed easyconfigs (via their path) listed_ec_paths = [ec['spec'] for ec in easyconfigs] # obtain full dependency graph for specified easyconfigs # easyconfigs will be ordered 'top-to-bottom': toolchain dependencies and toolchain first orig_ecs = resolve_dependencies(easyconfigs, retain_all_deps=True) # determine toolchain based on last easyconfigs toolchain = orig_ecs[-1]['ec']['toolchain'] _log.debug("Filtering using toolchain %s" % toolchain) # filter easyconfigs unless a dummy toolchain is used: drop toolchain and toolchain dependencies if toolchain['name'] != DUMMY_TOOLCHAIN_NAME: while orig_ecs[0]['ec']['toolchain'] != toolchain: orig_ecs = orig_ecs[1:] # generate tweaked easyconfigs, and continue with those instead tweaked_easyconfigs = [] for orig_ec in orig_ecs: new_ec_file = tweak_one(orig_ec['spec'], None, build_specs, targetdir=targetdir) # only return tweaked easyconfigs for easyconfigs which were listed originally # easyconfig files for dependencies are also generated but not included, and will be resolved via --robot if orig_ec['spec'] in listed_ec_paths: new_ecs = process_easyconfig(new_ec_file, build_specs=build_specs) tweaked_easyconfigs.extend(new_ecs) return tweaked_easyconfigs
def test_build_easyconfigs_in_parallel(self): """Basic test for build_easyconfigs_in_parallel function.""" easyconfig_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'gzip-1.5-goolf-1.4.10.eb') easyconfigs = process_easyconfig(easyconfig_file, validate=False) ordered_ecs = resolve_dependencies(easyconfigs) build_easyconfigs_in_parallel("echo %(spec)s", ordered_ecs)
def process_all_easyconfigs(self): """Process all easyconfigs and resolve inter-easyconfig dependencies.""" # all available easyconfig files easyconfigs_path = get_paths_for("easyconfigs")[0] specs = glob.glob('%s/*/*/*.eb' % easyconfigs_path) # parse all easyconfigs if they haven't been already if not self.parsed_easyconfigs: for spec in specs: self.parsed_easyconfigs.extend(process_easyconfig(spec)) self.ordered_specs = resolve_dependencies(self.parsed_easyconfigs)
# dry_run: print all easyconfigs and dependencies, and whether they are already built if options.dry_run or options.dry_run_short: print_dry_run(easyconfigs, short=not options.dry_run, build_specs=build_specs) if any([options.dry_run, options.dry_run_short, options.regtest, options.search, options.search_short]): cleanup(logfile, eb_tmpdir, testing) sys.exit(0) # skip modules that are already installed unless forced if not options.force: easyconfigs = skip_available(easyconfigs, testing=testing) # determine an order that will allow all specs in the set to build if len(easyconfigs) > 0: print_msg("resolving dependencies ...", log=_log, silent=testing) ordered_ecs = resolve_dependencies(easyconfigs, build_specs=build_specs) else: print_msg("No easyconfigs left to be built.", log=_log, silent=testing) ordered_ecs = [] # create dependency graph and exit if options.dep_graph: _log.info("Creating dependency graph %s" % options.dep_graph) dep_graph(options.dep_graph, ordered_ecs) sys.exit(0) # submit build as job(s) and exit if options.job: curdir = os.getcwd() # the options to ignore (help options can't reach here)
for ecfile in ecfiles: try: easyconfigs.extend(process_easyconfig(ecfile, build_specs=build_specs)) except EasyBuildError, err: test_results.append((ecfile, 'parsing_easyconfigs', 'easyconfig file error: %s' % err, _log)) # skip easyconfigs for which a module is already available, unless forced if not build_option('force'): _log.debug("Skipping easyconfigs from %s that already have a module available..." % easyconfigs) easyconfigs = skip_available(easyconfigs) _log.debug("Retained easyconfigs after skipping: %s" % easyconfigs) if build_option('sequential'): return build_easyconfigs(easyconfigs, output_dir, test_results) else: resolved = resolve_dependencies(easyconfigs, build_specs=build_specs) cmd = "eb %(spec)s --regtest --sequential -ld" command = "unset TMPDIR && cd %s && %s; " % (cur_dir, cmd) # retry twice in case of failure, to avoid fluke errors command += "if [ $? -ne 0 ]; then %(cmd)s --force && %(cmd)s --force; fi" % {'cmd': cmd} jobs = build_easyconfigs_in_parallel(command, resolved, output_dir=output_dir) print "List of submitted jobs:" for job in jobs: print "%s: %s" % (job.name, job.jobid) print "(%d jobs submitted)" % len(jobs) # determine leaf nodes in dependency graph, and report them all_deps = set()
except EasyBuildError, err: test_results.append((ecfile, 'parsing_easyconfigs', 'easyconfig file error: %s' % err, _log)) # skip easyconfigs for which a module is already available, unless forced if not build_option('force'): _log.debug( "Skipping easyconfigs from %s that already have a module available..." % easyconfigs) easyconfigs = skip_available(easyconfigs) _log.debug("Retained easyconfigs after skipping: %s" % easyconfigs) if build_option('sequential'): return build_easyconfigs(easyconfigs, output_dir, test_results) else: resolved = resolve_dependencies(easyconfigs, build_specs=build_specs) cmd = "eb %(spec)s --regtest --sequential -ld" command = "unset TMPDIR && cd %s && %s; " % (cur_dir, cmd) # retry twice in case of failure, to avoid fluke errors command += "if [ $? -ne 0 ]; then %(cmd)s --force && %(cmd)s --force; fi" % { 'cmd': cmd } jobs = build_easyconfigs_in_parallel(command, resolved, output_dir=output_dir) print "List of submitted jobs:" for job in jobs: print "%s: %s" % (job.name, job.jobid)
if any([ options.dry_run, options.dry_run_short, options.regtest, options.search, options.search_short ]): cleanup(logfile, eb_tmpdir, testing) sys.exit(0) # skip modules that are already installed unless forced if not options.force: easyconfigs = skip_available(easyconfigs, testing=testing) # determine an order that will allow all specs in the set to build if len(easyconfigs) > 0: print_msg("resolving dependencies ...", log=_log, silent=testing) ordered_ecs = resolve_dependencies(easyconfigs, build_specs=build_specs) else: print_msg("No easyconfigs left to be built.", log=_log, silent=testing) ordered_ecs = [] # create dependency graph and exit if options.dep_graph: _log.info("Creating dependency graph %s" % options.dep_graph) dep_graph(options.dep_graph, ordered_ecs) sys.exit(0) # submit build as job(s) and exit if options.job: curdir = os.getcwd() # the options to ignore (help options can't reach here)
def test_resolve_dependencies(self): """ Test with some basic testcases (also check if he can find dependencies inside the given directory """ easyconfig = { 'spec': '_', 'module': 'name/version', 'dependencies': [] } build_options = { 'ignore_osdeps': True, 'robot_path': None, 'validate': False, } res = resolve_dependencies([deepcopy(easyconfig)], build_options=build_options) self.assertEqual([easyconfig], res) easyconfig_dep = { 'ec': { 'name': 'foo', 'version': '1.2.3', 'versionsuffix': '', 'toolchain': {'name': 'dummy', 'version': 'dummy'}, }, 'spec': '_', 'module': 'foo/1.2.3', 'dependencies': [{ 'name': 'gzip', 'version': '1.4', 'versionsuffix': '', 'toolchain': {'name': 'dummy', 'version': 'dummy'}, 'dummy': True, }], 'parsed': True, } build_options.update({'robot_path': self.base_easyconfig_dir}) res = resolve_dependencies([deepcopy(easyconfig_dep)], build_options=build_options) # dependency should be found, order should be correct self.assertEqual(len(res), 2) self.assertEqual('gzip/1.4', res[0]['module']) self.assertEqual('foo/1.2.3', res[-1]['module']) # here we have include a Dependency in the easyconfig list easyconfig['module'] = 'gzip/1.4' ecs = [deepcopy(easyconfig_dep), deepcopy(easyconfig)] build_options.update({'robot_path': None}) res = resolve_dependencies(ecs, build_options=build_options) # all dependencies should be resolved self.assertEqual(0, sum(len(ec['dependencies']) for ec in res)) # this should not resolve (cannot find gzip-1.4.eb), both with and without robot enabled ecs = [deepcopy(easyconfig_dep)] msg = "Irresolvable dependencies encountered" self.assertErrorRegex(EasyBuildError, msg, resolve_dependencies, ecs, build_options=build_options) # test if dependencies of an automatically found file are also loaded easyconfig_dep['dependencies'] = [{ 'name': 'gzip', 'version': '1.4', 'versionsuffix': '', 'toolchain': {'name': 'GCC', 'version': '4.6.3'}, 'dummy': True, }] ecs = [deepcopy(easyconfig_dep)] build_options.update({'robot_path': self.base_easyconfig_dir}) res = resolve_dependencies([deepcopy(easyconfig_dep)], build_options=build_options) # GCC should be first (required by gzip dependency) self.assertEqual('GCC/4.6.3', res[0]['module']) self.assertEqual('foo/1.2.3', res[-1]['module']) # make sure that only missing stuff is built, and that available modules are not rebuilt # monkey patch MockModule to pretend that all ingredients required for goolf-1.4.10 toolchain are present MockModule.avail_modules = [ 'GCC/4.7.2', 'OpenMPI/1.6.4-GCC-4.7.2', 'OpenBLAS/0.2.6-gompi-1.4.10-LAPACK-3.4.2', 'FFTW/3.3.3-gompi-1.4.10', 'ScaLAPACK/2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2', ] easyconfig_dep['dependencies'] = [{ 'name': 'goolf', 'version': '1.4.10', 'versionsuffix': '', 'toolchain': {'name': 'dummy', 'version': 'dummy'}, 'dummy': True, }] ecs = [deepcopy(easyconfig_dep)] res = resolve_dependencies(ecs, build_options=build_options) # there should only be two retained builds, i.e. the software itself and the goolf toolchain as dep self.assertEqual(len(res), 2) # goolf should be first, the software itself second self.assertEqual('goolf/1.4.10', res[0]['module']) self.assertEqual('foo/1.2.3', res[1]['module']) # force doesn't trigger rebuild of all deps, but listed easyconfigs for which a module is available are rebuilt build_options.update({'force': True}) easyconfig['module'] = 'this/is/already/there' MockModule.avail_modules.append('this/is/already/there') ecs = [deepcopy(easyconfig_dep), deepcopy(easyconfig)] res = resolve_dependencies(ecs, build_options=build_options) # there should only be three retained builds, foo + goolf dep and the additional build (even though a module is available) self.assertEqual(len(res), 3) # goolf should be first, the software itself second self.assertEqual('this/is/already/there', res[0]['module']) self.assertEqual('goolf/1.4.10', res[1]['module']) self.assertEqual('foo/1.2.3', res[2]['module']) # build that are listed but already have a module available are not retained without force build_options.update({'force': False}) newecs = skip_available(ecs, testing=True) # skip available builds since force is not enabled res = resolve_dependencies(newecs, build_options=build_options) self.assertEqual(len(res), 2) self.assertEqual('goolf/1.4.10', res[0]['module']) self.assertEqual('foo/1.2.3', res[1]['module']) # with retain_all_deps enabled, all dependencies ae retained build_options.update({'retain_all_deps': True}) ecs = [deepcopy(easyconfig_dep)] newecs = skip_available(ecs, testing=True) # skip available builds since force is not enabled res = resolve_dependencies(newecs, build_options=build_options) self.assertEqual(len(res), 9) self.assertEqual('GCC/4.7.2', res[0]['module']) self.assertEqual('goolf/1.4.10', res[-2]['module']) self.assertEqual('foo/1.2.3', res[-1]['module']) build_options.update({'retain_all_deps': False}) # provide even less goolf ingredients (no OpenBLAS/ScaLAPACK), make sure the numbers add up MockModule.avail_modules = [ 'GCC/4.7.2', 'OpenMPI/1.6.4-GCC-4.7.2', 'gompi/1.4.10', 'FFTW/3.3.3-gompi-1.4.10', ] easyconfig_dep['dependencies'] = [{ 'name': 'goolf', 'version': '1.4.10', 'versionsuffix': '', 'toolchain': {'name': 'dummy', 'version': 'dummy'}, 'dummy': True, }] ecs = [deepcopy(easyconfig_dep)] res = resolve_dependencies([deepcopy(easyconfig_dep)], build_options=build_options) # there should only be two retained builds, i.e. the software itself and the goolf toolchain as dep self.assertEqual(len(res), 4) # goolf should be first, the software itself second self.assertEqual('OpenBLAS/0.2.6-gompi-1.4.10-LAPACK-3.4.2', res[0]['module']) self.assertEqual('ScaLAPACK/2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2', res[1]['module']) self.assertEqual('goolf/1.4.10', res[2]['module']) self.assertEqual('foo/1.2.3', res[3]['module'])
def test_resolve_dependencies(self): """ Test with some basic testcases (also check if he can find dependencies inside the given directory """ easyconfig = { 'spec': '_', 'module': 'name/version', 'dependencies': [] } build_options = { 'allow_modules_tool_mismatch': True, 'robot_path': None, 'validate': False, } init_config(build_options=build_options) res = resolve_dependencies([deepcopy(easyconfig)]) self.assertEqual([easyconfig], res) easyconfig_dep = { 'ec': { 'name': 'foo', 'version': '1.2.3', 'versionsuffix': '', 'toolchain': { 'name': 'dummy', 'version': 'dummy' }, }, 'spec': '_', 'module': 'foo/1.2.3', 'dependencies': [{ 'name': 'gzip', 'version': '1.4', 'versionsuffix': '', 'toolchain': { 'name': 'dummy', 'version': 'dummy' }, 'dummy': True, }], 'parsed': True, } build_options.update({'robot_path': self.base_easyconfig_dir}) init_config(build_options=build_options) res = resolve_dependencies([deepcopy(easyconfig_dep)]) # dependency should be found, order should be correct self.assertEqual(len(res), 2) self.assertEqual('gzip/1.4', res[0]['module']) self.assertEqual('foo/1.2.3', res[-1]['module']) # here we have include a Dependency in the easyconfig list easyconfig['module'] = 'gzip/1.4' ecs = [deepcopy(easyconfig_dep), deepcopy(easyconfig)] build_options.update({'robot_path': None}) init_config(build_options=build_options) res = resolve_dependencies(ecs) # all dependencies should be resolved self.assertEqual(0, sum(len(ec['dependencies']) for ec in res)) # this should not resolve (cannot find gzip-1.4.eb), both with and without robot enabled ecs = [deepcopy(easyconfig_dep)] msg = "Irresolvable dependencies encountered" self.assertErrorRegex(EasyBuildError, msg, resolve_dependencies, ecs) # test if dependencies of an automatically found file are also loaded easyconfig_dep['dependencies'] = [{ 'name': 'gzip', 'version': '1.4', 'versionsuffix': '', 'toolchain': { 'name': 'GCC', 'version': '4.6.3' }, 'dummy': True, }] ecs = [deepcopy(easyconfig_dep)] build_options.update({'robot_path': self.base_easyconfig_dir}) init_config(build_options=build_options) res = resolve_dependencies([deepcopy(easyconfig_dep)]) # GCC should be first (required by gzip dependency) self.assertEqual('GCC/4.6.3', res[0]['module']) self.assertEqual('foo/1.2.3', res[-1]['module']) # make sure that only missing stuff is built, and that available modules are not rebuilt # monkey patch MockModule to pretend that all ingredients required for goolf-1.4.10 toolchain are present MockModule.avail_modules = [ 'GCC/4.7.2', 'OpenMPI/1.6.4-GCC-4.7.2', 'OpenBLAS/0.2.6-gompi-1.4.10-LAPACK-3.4.2', 'FFTW/3.3.3-gompi-1.4.10', 'ScaLAPACK/2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2', ] easyconfig_dep['dependencies'] = [{ 'name': 'goolf', 'version': '1.4.10', 'versionsuffix': '', 'toolchain': { 'name': 'dummy', 'version': 'dummy' }, 'dummy': True, }] ecs = [deepcopy(easyconfig_dep)] res = resolve_dependencies(ecs) # there should only be two retained builds, i.e. the software itself and the goolf toolchain as dep self.assertEqual(len(res), 2) # goolf should be first, the software itself second self.assertEqual('goolf/1.4.10', res[0]['module']) self.assertEqual('foo/1.2.3', res[1]['module']) # force doesn't trigger rebuild of all deps, but listed easyconfigs for which a module is available are rebuilt build_options.update({'force': True}) init_config(build_options=build_options) easyconfig['module'] = 'this/is/already/there' MockModule.avail_modules.append('this/is/already/there') ecs = [deepcopy(easyconfig_dep), deepcopy(easyconfig)] res = resolve_dependencies(ecs) # there should only be three retained builds, foo + goolf dep and the additional build (even though a module is available) self.assertEqual(len(res), 3) # goolf should be first, the software itself second self.assertEqual('this/is/already/there', res[0]['module']) self.assertEqual('goolf/1.4.10', res[1]['module']) self.assertEqual('foo/1.2.3', res[2]['module']) # build that are listed but already have a module available are not retained without force build_options.update({'force': False}) init_config(build_options=build_options) newecs = skip_available( ecs, testing=True) # skip available builds since force is not enabled res = resolve_dependencies(newecs) self.assertEqual(len(res), 2) self.assertEqual('goolf/1.4.10', res[0]['module']) self.assertEqual('foo/1.2.3', res[1]['module']) # with retain_all_deps enabled, all dependencies ae retained build_options.update({'retain_all_deps': True}) init_config(build_options=build_options) ecs = [deepcopy(easyconfig_dep)] newecs = skip_available( ecs, testing=True) # skip available builds since force is not enabled res = resolve_dependencies(newecs) self.assertEqual(len(res), 9) self.assertEqual('GCC/4.7.2', res[0]['module']) self.assertEqual('goolf/1.4.10', res[-2]['module']) self.assertEqual('foo/1.2.3', res[-1]['module']) build_options.update({'retain_all_deps': False}) init_config(build_options=build_options) # provide even less goolf ingredients (no OpenBLAS/ScaLAPACK), make sure the numbers add up MockModule.avail_modules = [ 'GCC/4.7.2', 'OpenMPI/1.6.4-GCC-4.7.2', 'gompi/1.4.10', 'FFTW/3.3.3-gompi-1.4.10', ] easyconfig_dep['dependencies'] = [{ 'name': 'goolf', 'version': '1.4.10', 'versionsuffix': '', 'toolchain': { 'name': 'dummy', 'version': 'dummy' }, 'dummy': True, }] ecs = [deepcopy(easyconfig_dep)] res = resolve_dependencies([deepcopy(easyconfig_dep)]) # there should only be two retained builds, i.e. the software itself and the goolf toolchain as dep self.assertEqual(len(res), 4) # goolf should be first, the software itself second self.assertEqual('OpenBLAS/0.2.6-gompi-1.4.10-LAPACK-3.4.2', res[0]['module']) self.assertEqual( 'ScaLAPACK/2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2', res[1]['module']) self.assertEqual('goolf/1.4.10', res[2]['module']) self.assertEqual('foo/1.2.3', res[3]['module'])
def test_resolve_dependencies(self): """ Test with some basic testcases (also check if he can find dependencies inside the given directory """ easyconfig = {"spec": "_", "module": "name/version", "dependencies": []} build_options = {"ignore_osdeps": True, "robot_path": None, "validate": False} res = resolve_dependencies([deepcopy(easyconfig)], build_options=build_options) self.assertEqual([easyconfig], res) easyconfig_dep = { "ec": { "name": "foo", "version": "1.2.3", "versionsuffix": "", "toolchain": {"name": "dummy", "version": "dummy"}, }, "spec": "_", "module": "foo/1.2.3", "dependencies": [ { "name": "gzip", "version": "1.4", "versionsuffix": "", "toolchain": {"name": "dummy", "version": "dummy"}, "dummy": True, } ], "parsed": True, } build_options.update({"robot_path": self.base_easyconfig_dir}) res = resolve_dependencies([deepcopy(easyconfig_dep)], build_options=build_options) # dependency should be found, order should be correct self.assertEqual(len(res), 2) self.assertEqual("gzip/1.4", res[0]["module"]) self.assertEqual("foo/1.2.3", res[-1]["module"]) # here we have include a Dependency in the easyconfig list easyconfig["module"] = "gzip/1.4" ecs = [deepcopy(easyconfig_dep), deepcopy(easyconfig)] build_options.update({"robot_path": None}) res = resolve_dependencies(ecs, build_options=build_options) # all dependencies should be resolved self.assertEqual(0, sum(len(ec["dependencies"]) for ec in res)) # this should not resolve (cannot find gzip-1.4.eb) ecs = [deepcopy(easyconfig_dep)] self.assertRaises(EasyBuildError, resolve_dependencies, ecs, build_options=build_options) # test if dependencies of an automatically found file are also loaded easyconfig_dep["dependencies"] = [ { "name": "gzip", "version": "1.4", "versionsuffix": "", "toolchain": {"name": "GCC", "version": "4.6.3"}, "dummy": True, } ] ecs = [deepcopy(easyconfig_dep)] build_options.update({"robot_path": self.base_easyconfig_dir}) res = resolve_dependencies([deepcopy(easyconfig_dep)], build_options=build_options) # GCC should be first (required by gzip dependency) self.assertEqual("GCC/4.6.3", res[0]["module"]) self.assertEqual("foo/1.2.3", res[-1]["module"]) # make sure that only missing stuff is built, and that available modules are not rebuilt # monkey patch MockModule to pretend that all ingredients required for goolf-1.4.10 toolchain are present MockModule.avail_modules = [ "GCC/4.7.2", "OpenMPI/1.6.4-GCC-4.7.2", "OpenBLAS/0.2.6-gompi-1.4.10-LAPACK-3.4.2", "FFTW/3.3.3-gompi-1.4.10", "ScaLAPACK/2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2", ] easyconfig_dep["dependencies"] = [ { "name": "goolf", "version": "1.4.10", "versionsuffix": "", "toolchain": {"name": "dummy", "version": "dummy"}, "dummy": True, } ] ecs = [deepcopy(easyconfig_dep)] res = resolve_dependencies(ecs, build_options=build_options) # there should only be two retained builds, i.e. the software itself and the goolf toolchain as dep self.assertEqual(len(res), 2) # goolf should be first, the software itself second self.assertEqual("goolf/1.4.10", res[0]["module"]) self.assertEqual("foo/1.2.3", res[1]["module"]) # force doesn't trigger rebuild of all deps, but listed easyconfigs for which a module is available are rebuilt build_options.update({"force": True}) easyconfig["module"] = "this/is/already/there" MockModule.avail_modules.append("this/is/already/there") ecs = [deepcopy(easyconfig_dep), deepcopy(easyconfig)] res = resolve_dependencies(ecs, build_options=build_options) # there should only be three retained builds, foo + goolf dep and the additional build (even though a module is available) self.assertEqual(len(res), 3) # goolf should be first, the software itself second self.assertEqual("this/is/already/there", res[0]["module"]) self.assertEqual("goolf/1.4.10", res[1]["module"]) self.assertEqual("foo/1.2.3", res[2]["module"]) # build that are listed but already have a module available are not retained without force build_options.update({"force": False}) newecs = skip_available(ecs, testing=True) # skip available builds since force is not enabled res = resolve_dependencies(newecs, build_options=build_options) self.assertEqual(len(res), 2) self.assertEqual("goolf/1.4.10", res[0]["module"]) self.assertEqual("foo/1.2.3", res[1]["module"]) # with retain_all_deps enabled, all dependencies ae retained build_options.update({"retain_all_deps": True}) ecs = [deepcopy(easyconfig_dep)] newecs = skip_available(ecs, testing=True) # skip available builds since force is not enabled res = resolve_dependencies(newecs, build_options=build_options) self.assertEqual(len(res), 9) self.assertEqual("GCC/4.7.2", res[0]["module"]) self.assertEqual("goolf/1.4.10", res[-2]["module"]) self.assertEqual("foo/1.2.3", res[-1]["module"]) build_options.update({"retain_all_deps": False}) # provide even less goolf ingredients (no OpenBLAS/ScaLAPACK), make sure the numbers add up MockModule.avail_modules = ["GCC/4.7.2", "OpenMPI/1.6.4-GCC-4.7.2", "gompi/1.4.10", "FFTW/3.3.3-gompi-1.4.10"] easyconfig_dep["dependencies"] = [ { "name": "goolf", "version": "1.4.10", "versionsuffix": "", "toolchain": {"name": "dummy", "version": "dummy"}, "dummy": True, } ] ecs = [deepcopy(easyconfig_dep)] res = resolve_dependencies([deepcopy(easyconfig_dep)], build_options=build_options) # there should only be two retained builds, i.e. the software itself and the goolf toolchain as dep self.assertEqual(len(res), 4) # goolf should be first, the software itself second self.assertEqual("OpenBLAS/0.2.6-gompi-1.4.10-LAPACK-3.4.2", res[0]["module"]) self.assertEqual("ScaLAPACK/2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2", res[1]["module"]) self.assertEqual("goolf/1.4.10", res[2]["module"]) self.assertEqual("foo/1.2.3", res[3]["module"])