def test_build_easyconfigs_in_parallel_slurm(self): """Test build_easyconfigs_in_parallel(), using (mocked) Slurm as backend for --job.""" # install mocked versions of 'sbatch' and 'scontrol' commands sbatch = os.path.join(self.test_prefix, 'bin', 'sbatch') write_file(sbatch, MOCKED_SBATCH) adjust_permissions(sbatch, stat.S_IXUSR, add=True) scontrol = os.path.join(self.test_prefix, 'bin', 'scontrol') write_file(scontrol, MOCKED_SCONTROL) adjust_permissions(scontrol, stat.S_IXUSR, add=True) os.environ['PATH'] = os.path.pathsep.join( [os.path.join(self.test_prefix, 'bin'), os.getenv('PATH')]) topdir = os.path.dirname(os.path.abspath(__file__)) test_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'g', 'gzip', 'gzip-1.5-foss-2018a.eb') foss_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'f', 'foss', 'foss-2018a.eb') build_options = { 'external_modules_metadata': {}, 'robot_path': os.path.join(topdir, 'easyconfigs', 'test_ecs'), 'valid_module_classes': config.module_classes(), 'validate': False, 'job_cores': 3, 'job_max_walltime': 5, 'force': True, } init_config(args=['--job-backend=Slurm'], build_options=build_options) easyconfigs = process_easyconfig(test_ec) + process_easyconfig(foss_ec) ordered_ecs = resolve_dependencies(easyconfigs, self.modtool) self.mock_stdout(True) jobs = build_easyconfigs_in_parallel("echo '%(spec)s'", ordered_ecs, prepare_first=False) self.mock_stdout(False) # jobs are submitted for foss & gzip (listed easyconfigs) self.assertEqual(len(jobs), 2) # last job (gzip) has a dependency on second-to-last job (foss) self.assertEqual(jobs[0].job_specs['job-name'], 'foss-2018a') expected = { 'dependency': 'afterok:%s' % jobs[0].jobid, 'hold': True, 'job-name': 'gzip-1.5-foss-2018a', 'nodes': 1, 'ntasks': 3, 'ntasks-per-node': 3, 'output': '%x-%j.out', 'time': 300, # 60*5 (unit is minutes) 'wrap': "echo '%s'" % test_ec, } self.assertEqual(jobs[1].job_specs, expected)
def test_build_easyconfigs_in_parallel_slurm(self): """Test build_easyconfigs_in_parallel(), using (mocked) Slurm as backend for --job.""" # install mocked versions of 'sbatch' and 'scontrol' commands sbatch = os.path.join(self.test_prefix, 'bin', 'sbatch') write_file(sbatch, MOCKED_SBATCH) adjust_permissions(sbatch, stat.S_IXUSR, add=True) scontrol = os.path.join(self.test_prefix, 'bin', 'scontrol') write_file(scontrol, MOCKED_SCONTROL) adjust_permissions(scontrol, stat.S_IXUSR, add=True) os.environ['PATH'] = os.path.pathsep.join([os.path.join(self.test_prefix, 'bin'), os.getenv('PATH')]) topdir = os.path.dirname(os.path.abspath(__file__)) test_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'g', 'gzip', 'gzip-1.5-foss-2018a.eb') foss_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'f', 'foss', 'foss-2018a.eb') build_options = { 'external_modules_metadata': {}, 'robot_path': os.path.join(topdir, 'easyconfigs', 'test_ecs'), 'valid_module_classes': config.module_classes(), 'validate': False, 'job_cores': 3, 'job_max_walltime': 5, 'force': True, } init_config(args=['--job-backend=Slurm'], build_options=build_options) easyconfigs = process_easyconfig(test_ec) + process_easyconfig(foss_ec) ordered_ecs = resolve_dependencies(easyconfigs, self.modtool) self.mock_stdout(True) jobs = build_easyconfigs_in_parallel("echo '%(spec)s'", ordered_ecs, prepare_first=False) self.mock_stdout(False) # jobs are submitted for foss & gzip (listed easyconfigs) self.assertEqual(len(jobs), 2) # last job (gzip) has a dependency on second-to-last job (foss) self.assertEqual(jobs[0].job_specs['job-name'], 'foss-2018a') expected = { 'dependency': 'afterok:%s' % jobs[0].jobid, 'hold': True, 'job-name': 'gzip-1.5-foss-2018a', 'nodes': 1, 'ntasks': 3, 'ntasks-per-node': 3, 'output': '%x-%j.out', 'time': 300, # 60*5 (unit is minutes) 'wrap': "echo '%s'" % test_ec, } self.assertEqual(jobs[1].job_specs, expected)
def test_build_easyconfigs_in_parallel(self): """Basic test for build_easyconfigs_in_parallel function.""" easyconfig_file = os.path.join(os.path.dirname(__file__), "easyconfigs", "gzip-1.5-goolf-1.4.10.eb") easyconfigs = process_easyconfig(easyconfig_file) ordered_ecs = resolve_dependencies(easyconfigs) jobs = build_easyconfigs_in_parallel("echo %(spec)s", ordered_ecs, prepare_first=False) self.assertEqual(len(jobs), 8)
def test_get_easyblock_instance(self): """Test get_easyblock_instance function.""" # adjust PYTHONPATH such that test easyblocks are found testdir = os.path.abspath(os.path.dirname(__file__)) import easybuild eb_blocks_path = os.path.join(testdir, 'sandbox') if eb_blocks_path not in sys.path: sys.path.append(eb_blocks_path) easybuild = reload(easybuild) import easybuild.easyblocks reload(easybuild.easyblocks) from easybuild.easyblocks.toy import EB_toy ec = process_easyconfig( os.path.join(testdir, 'easyconfigs', 'toy-0.0.eb'))[0] eb = get_easyblock_instance(ec) self.assertTrue(isinstance(eb, EB_toy)) # check whether 'This is easyblock' log message is there tup = ('EB_toy', 'easybuild.easyblocks.toy', '.*test/framework/sandbox/easybuild/easyblocks/t/toy.pyc*') eb_log_msg_re = re.compile( r"INFO This is easyblock %s from module %s (%s)" % tup, re.M) logtxt = read_file(eb.logfile) self.assertTrue( eb_log_msg_re.search(logtxt), "Pattern '%s' found in: %s" % (eb_log_msg_re.pattern, logtxt))
def test_patch_step(self): """Test patch step.""" test_easyconfigs = os.path.join( os.path.abspath(os.path.dirname(__file__)), 'easyconfigs') ec = process_easyconfig(os.path.join(test_easyconfigs, 'toy-0.0.eb'))[0] orig_sources = ec['ec']['sources'][:] toy_patches = [ 'toy-0.0_typo.patch', # test for applying patch ('toy-extra.txt', 'toy-0.0'), # test for patch-by-copy ] self.assertEqual(ec['ec']['patches'], toy_patches) # test applying patches without sources ec['ec']['sources'] = [] eb = EasyBlock(ec['ec']) eb.fetch_step() eb.extract_step() self.assertErrorRegex(EasyBuildError, '.*', eb.patch_step) # test actual patching of unpacked sources ec['ec']['sources'] = orig_sources eb = EasyBlock(ec['ec']) eb.fetch_step() eb.extract_step() eb.patch_step() # verify that patches were applied toydir = os.path.join(eb.builddir, 'toy-0.0') self.assertEqual(sorted(os.listdir(toydir)), ['toy-extra.txt', 'toy.source', 'toy.source.orig']) self.assertTrue("and very proud of it" in read_file( os.path.join(toydir, 'toy.source'))) self.assertEqual(read_file(os.path.join(toydir, 'toy-extra.txt')), 'moar!\n')
def test_guess_start_dir(self): """Test guessing the start dir.""" test_easyconfigs = os.path.join( os.path.abspath(os.path.dirname(__file__)), 'easyconfigs') ec = process_easyconfig(os.path.join(test_easyconfigs, 'toy-0.0.eb'))[0] def check_start_dir(expected_start_dir): """Check start dir.""" eb = EasyBlock(ec['ec']) eb.silent = True eb.cfg['stop'] = 'patch' eb.run_all_steps(False) eb.guess_start_dir() abs_expected_start_dir = os.path.join(eb.builddir, expected_start_dir) self.assertTrue( os.path.samefile(eb.cfg['start_dir'], abs_expected_start_dir)) self.assertTrue( os.path.samefile(os.getcwd(), abs_expected_start_dir)) # default (no start_dir specified): use unpacked dir as start dir self.assertEqual(ec['ec']['start_dir'], None) check_start_dir('toy-0.0') # using start_dir equal to the one we're in is OK ec['ec']['start_dir'] = '%(name)s-%(version)s' self.assertEqual(ec['ec']['start_dir'], 'toy-0.0') check_start_dir('toy-0.0') # clean error when specified start dir does not exist ec['ec']['start_dir'] = 'thisstartdirisnotthere' err_pattern = "Specified start dir .*/toy-0.0/thisstartdirisnotthere does not exist" self.assertErrorRegex(EasyBuildError, err_pattern, check_start_dir, 'whatever')
def test_prepare_step(self): """Test prepare step (setting up build environment).""" test_easyconfigs = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') ec = process_easyconfig(os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb'))[0] mkdir(os.path.join(self.test_buildpath, 'toy', '0.0', 'dummy-dummy'), parents=True) eb = EasyBlock(ec['ec']) eb.silent = True eb.prepare_step() self.assertEqual(self.modtool.list(), []) os.environ['THIS_IS_AN_UNWANTED_ENV_VAR'] = 'foo' eb.cfg['unwanted_env_vars'] = ['THIS_IS_AN_UNWANTED_ENV_VAR'] eb.cfg['allow_system_deps'] = [('Python', '1.2.3')] init_config(build_options={'extra_modules': ['GCC/4.7.2']}) eb.prepare_step() self.assertEqual(os.environ.get('THIS_IS_AN_UNWANTED_ENV_VAR'), None) self.assertEqual(os.environ.get('EBROOTPYTHON'), 'Python') self.assertEqual(os.environ.get('EBVERSIONPYTHON'), '1.2.3') self.assertEqual(len(self.modtool.list()), 1) self.assertEqual(self.modtool.list()[0]['mod_name'], 'GCC/4.7.2')
def test_get_easyblock_instance(self): """Test get_easyblock_instance function.""" # adjust PYTHONPATH such that test easyblocks are found testdir = os.path.abspath(os.path.dirname(__file__)) import easybuild eb_blocks_path = os.path.join(testdir, "sandbox") if not eb_blocks_path in sys.path: sys.path.append(eb_blocks_path) easybuild = reload(easybuild) import easybuild.easyblocks reload(easybuild.easyblocks) from easybuild.easyblocks.toy import EB_toy ec = process_easyconfig(os.path.join(testdir, "easyconfigs", "toy-0.0.eb"))[0] eb = get_easyblock_instance(ec) self.assertTrue(isinstance(eb, EB_toy)) # check whether 'This is easyblock' log message is there tup = ("EB_toy", "easybuild.easyblocks.toy", ".*test/framework/sandbox/easybuild/easyblocks/toy.pyc*") eb_log_msg_re = re.compile(r"INFO This is easyblock %s from module %s (%s)" % tup, re.M) logtxt = read_file(eb.logfile) self.assertTrue(eb_log_msg_re.search(logtxt), "Pattern '%s' found in: %s" % (eb_log_msg_re.pattern, logtxt))
def test_patch_step(self): """Test patch step.""" test_easyconfigs = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') ec = process_easyconfig(os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb'))[0] orig_sources = ec['ec']['sources'][:] toy_patches = [ 'toy-0.0_typo.patch', # test for applying patch ('toy-extra.txt', 'toy-0.0'), # test for patch-by-copy ] self.assertEqual(ec['ec']['patches'], toy_patches) # test applying patches without sources ec['ec']['sources'] = [] eb = EasyBlock(ec['ec']) eb.fetch_step() eb.extract_step() self.assertErrorRegex(EasyBuildError, '.*', eb.patch_step) # test actual patching of unpacked sources ec['ec']['sources'] = orig_sources eb = EasyBlock(ec['ec']) eb.fetch_step() eb.extract_step() eb.patch_step() # verify that patches were applied toydir = os.path.join(eb.builddir, 'toy-0.0') self.assertEqual(sorted(os.listdir(toydir)), ['toy-extra.txt', 'toy.source', 'toy.source.orig']) self.assertTrue("and very proud of it" in read_file(os.path.join(toydir, 'toy.source'))) self.assertEqual(read_file(os.path.join(toydir, 'toy-extra.txt')), 'moar!\n')
def test_guess_start_dir(self): """Test guessing the start dir.""" test_easyconfigs = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') ec = process_easyconfig(os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb'))[0] def check_start_dir(expected_start_dir): """Check start dir.""" eb = EasyBlock(ec['ec']) eb.silent = True eb.cfg['stop'] = 'patch' eb.run_all_steps(False) eb.guess_start_dir() abs_expected_start_dir = os.path.join(eb.builddir, expected_start_dir) self.assertTrue(os.path.samefile(eb.cfg['start_dir'], abs_expected_start_dir)) self.assertTrue(os.path.samefile(os.getcwd(), abs_expected_start_dir)) # default (no start_dir specified): use unpacked dir as start dir self.assertEqual(ec['ec']['start_dir'], None) check_start_dir('toy-0.0') # using start_dir equal to the one we're in is OK ec['ec']['start_dir'] = '%(name)s-%(version)s' self.assertEqual(ec['ec']['start_dir'], 'toy-0.0') check_start_dir('toy-0.0') # clean error when specified start dir does not exist ec['ec']['start_dir'] = 'thisstartdirisnotthere' err_pattern = "Specified start dir .*/toy-0.0/thisstartdirisnotthere does not exist" self.assertErrorRegex(EasyBuildError, err_pattern, check_start_dir, 'whatever')
def test_changed_files_pull_request(self): """Specific checks only done for the (easyconfig) files that were changed in a pull request.""" # $TRAVIS_PULL_REQUEST should be a PR number, otherwise we're not running tests for a PR if re.match('^[0-9]+$', os.environ.get('TRAVIS_PULL_REQUEST', '(none)')): # target branch should be anything other than 'master'; # usually is 'develop', but could also be a release branch like '3.7.x' travis_branch = os.environ.get('TRAVIS_BRANCH', None) if travis_branch and travis_branch != 'master': if not self.parsed_easyconfigs: self.process_all_easyconfigs() # relocate to top-level directory of repository to run 'git diff' command top_dir = os.path.dirname( os.path.dirname(get_paths_for('easyconfigs')[0])) cwd = change_dir(top_dir) # get list of changed easyconfigs cmd = "git diff --name-only --diff-filter=AM %s...HEAD" % travis_branch out, ec = run_cmd(cmd, simple=False) changed_ecs_filenames = [ os.path.basename(f) for f in out.strip().split('\n') if f.endswith('.eb') ] print("\nList of changed easyconfig files in this PR: %s" % '\n'.join(changed_ecs_filenames)) change_dir(cwd) # grab parsed easyconfigs for changed easyconfig files changed_ecs = [] for ec_fn in changed_ecs_filenames: match = None for ec in self.parsed_easyconfigs: if os.path.basename(ec['spec']) == ec_fn: match = ec['ec'] break if match: changed_ecs.append(match) else: # if no easyconfig is found, it's possible some archived easyconfigs were touched in the PR... # so as a last resort, try to find the easyconfig file in __archive__ easyconfigs_path = get_paths_for("easyconfigs")[0] specs = glob.glob('%s/__archive__/*/*/%s' % (easyconfigs_path, ec_fn)) if len(specs) == 1: ec = process_easyconfig(specs[0])[0] changed_ecs.append(ec['ec']) else: error_msg = "Failed to find parsed easyconfig for %s" % ec_fn error_msg += " (and could not isolate it in easyconfigs archive either)" self.assertTrue(False, error_msg) # run checks on changed easyconfigs self.check_sha256_checksums(changed_ecs) self.check_python_packages(changed_ecs)
def test_prepare_step(self): """Test prepare step (setting up build environment).""" test_easyconfigs = os.path.join( os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') ec = process_easyconfig( os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb'))[0] mkdir(os.path.join(self.test_buildpath, 'toy', '0.0', 'dummy-dummy'), parents=True) eb = EasyBlock(ec['ec']) eb.silent = True eb.prepare_step() self.assertEqual(self.modtool.list(), []) os.environ['THIS_IS_AN_UNWANTED_ENV_VAR'] = 'foo' eb.cfg['unwanted_env_vars'] = ['THIS_IS_AN_UNWANTED_ENV_VAR'] eb.cfg['allow_system_deps'] = [('Python', '1.2.3')] init_config(build_options={'extra_modules': ['GCC/4.7.2']}) eb.prepare_step() self.assertEqual(os.environ.get('THIS_IS_AN_UNWANTED_ENV_VAR'), None) self.assertEqual(os.environ.get('EBROOTPYTHON'), 'Python') self.assertEqual(os.environ.get('EBVERSIONPYTHON'), '1.2.3') self.assertEqual(len(self.modtool.list()), 1) self.assertEqual(self.modtool.list()[0]['mod_name'], 'GCC/4.7.2')
def test_build_easyconfigs_in_parallel(self): """Basic test for build_easyconfigs_in_parallel function.""" easyconfig_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'gzip-1.5-goolf-1.4.10.eb') easyconfigs = process_easyconfig(easyconfig_file, validate=False) ordered_ecs = resolve_dependencies(easyconfigs) build_easyconfigs_in_parallel("echo %(spec)s", ordered_ecs)
def test_obtain_file(self): """Test obtain_file method.""" toy_tarball = 'toy-0.0.tar.gz' testdir = os.path.abspath(os.path.dirname(__file__)) sandbox_sources = os.path.join(testdir, 'sandbox', 'sources') toy_tarball_path = os.path.join(sandbox_sources, 'toy', toy_tarball) tmpdir = tempfile.mkdtemp() tmpdir_subdir = os.path.join(tmpdir, 'testing') mkdir(tmpdir_subdir, parents=True) del os.environ['EASYBUILD_SOURCEPATH'] # defined by setUp ec = process_easyconfig(os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb'))[0] eb = EasyBlock(ec['ec']) # 'downloading' a file to (first) sourcepath works init_config(args=["--sourcepath=%s:/no/such/dir:%s" % (tmpdir, testdir)]) shutil.copy2(toy_tarball_path, tmpdir_subdir) res = eb.obtain_file(toy_tarball, urls=['file://%s' % tmpdir_subdir]) self.assertEqual(res, os.path.join(tmpdir, 't', 'toy', toy_tarball)) # finding a file in sourcepath works init_config(args=["--sourcepath=%s:/no/such/dir:%s" % (sandbox_sources, tmpdir)]) res = eb.obtain_file(toy_tarball) self.assertEqual(res, toy_tarball_path) # sourcepath has preference over downloading res = eb.obtain_file(toy_tarball, urls=['file://%s' % tmpdir_subdir]) self.assertEqual(res, toy_tarball_path) # obtain_file yields error for non-existing files fn = 'thisisclearlyanonexistingfile' error_regex = "Couldn't find file %s anywhere, and downloading it didn't work either" % fn self.assertErrorRegex(EasyBuildError, error_regex, eb.obtain_file, fn, urls=['file://%s' % tmpdir_subdir]) # file specifications via URL also work, are downloaded to (first) sourcepath init_config(args=["--sourcepath=%s:/no/such/dir:%s" % (tmpdir, sandbox_sources)]) urls = ["http://hpcugent.github.io/easybuild/index.html", "https://hpcugent.github.io/easybuild/index.html"] for file_url in urls: fn = os.path.basename(file_url) res = None try: res = eb.obtain_file(file_url) except EasyBuildError, err: # if this fails, it should be because there's no online access download_fail_regex = re.compile('socket error') self.assertTrue(download_fail_regex.search(str(err))) # result may be None during offline testing if res is not None: loc = os.path.join(tmpdir, 't', 'toy', fn) self.assertEqual(res, loc) self.assertTrue(os.path.exists(loc), "%s file is found at %s" % (fn, loc)) txt = open(loc, 'r').read() eb_regex = re.compile("EasyBuild: building software with ease") self.assertTrue(eb_regex.search(txt)) else: print "ignoring failure to download %s in test_obtain_file, testing offline?" % file_url
def regtest(easyconfig_paths, build_specs=None): """ Run regression test, using easyconfigs available in given path @param easyconfig_paths: path of easyconfigs to run regtest on @param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...) """ cur_dir = os.getcwd() aggregate_regtest = build_option('aggregate_regtest') if aggregate_regtest is not None: output_file = os.path.join( aggregate_regtest, "%s-aggregate.xml" % os.path.basename(aggregate_regtest)) aggregate_xml_in_dirs(aggregate_regtest, output_file) _log.info("aggregated xml files inside %s, output written to: %s" % (aggregate_regtest, output_file)) sys.exit(0) # create base directory, which is used to place # all log files and the test output as xml basename = "easybuild-test-%s" % datetime.now().strftime("%Y%m%d%H%M%S") var = config.OLDSTYLE_ENVIRONMENT_VARIABLES['test_output_path'] regtest_output_dir = build_option('regtest_output_dir') if regtest_output_dir is not None: output_dir = regtest_output_dir elif var in os.environ: output_dir = os.path.abspath(os.environ[var]) else: # default: current dir + easybuild-test-[timestamp] output_dir = os.path.join(cur_dir, basename) mkdir(output_dir, parents=True) # find all easyconfigs ecfiles = [] if easyconfig_paths: for path in easyconfig_paths: ecfiles += find_easyconfigs( path, ignore_dirs=build_option('ignore_dirs')) else: _log.error("No easyconfig paths specified.") test_results = [] # process all the found easyconfig files easyconfigs = [] for ecfile in ecfiles: try: easyconfigs.extend( process_easyconfig(ecfile, build_specs=build_specs)) except EasyBuildError, err: test_results.append((ecfile, 'parsing_easyconfigs', 'easyconfig file error: %s' % err, _log))
def test_build_easyconfigs_in_parallel(self): """Basic test for build_easyconfigs_in_parallel function.""" easyconfig_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'gzip-1.5-goolf-1.4.10.eb') easyconfigs = process_easyconfig(easyconfig_file) ordered_ecs = resolve_dependencies(easyconfigs) jobs = build_easyconfigs_in_parallel("echo %(spec)s", ordered_ecs, prepare_first=False) self.assertEqual(len(jobs), 8)
def test_mns(): """Test default module naming scheme.""" # test default naming scheme for ec_file in [f for f in ec_files if not 'broken' in os.path.basename(f)]: ec_path = os.path.abspath(ec_file) ecs = process_easyconfig(ec_path, validate=False) # derive module name directly from easyconfig file name ec_fn = os.path.basename(ec_file) if ec_fn in ec2mod_map: # only check first, ignore any others (occurs when blocks are used (format v1.0 only)) self.assertEqual(ec2mod_map[ec_fn], ActiveMNS().det_full_module_name(ecs[0]['ec']))
def test_changed_files_pull_request(self): """Specific checks only done for the (easyconfig) files that were changed in a pull request.""" # $TRAVIS_PULL_REQUEST should be a PR number, otherwise we're not running tests for a PR if re.match('^[0-9]+$', os.environ.get('TRAVIS_PULL_REQUEST', '(none)')): # target branch should be anything other than 'master'; # usually is 'develop', but could also be a release branch like '3.7.x' travis_branch = os.environ.get('TRAVIS_BRANCH', None) if travis_branch and travis_branch != 'master': if not self.parsed_easyconfigs: self.process_all_easyconfigs() # relocate to top-level directory of repository to run 'git diff' command top_dir = os.path.dirname(os.path.dirname(get_paths_for('easyconfigs')[0])) cwd = change_dir(top_dir) # get list of changed easyconfigs cmd = "git diff --name-only --diff-filter=AM %s...HEAD" % travis_branch out, ec = run_cmd(cmd, simple=False) changed_ecs_filenames = [os.path.basename(f) for f in out.strip().split('\n') if f.endswith('.eb')] print("\nList of changed easyconfig files in this PR: %s" % '\n'.join(changed_ecs_filenames)) change_dir(cwd) # grab parsed easyconfigs for changed easyconfig files changed_ecs = [] for ec_fn in changed_ecs_filenames: match = None for ec in self.parsed_easyconfigs: if os.path.basename(ec['spec']) == ec_fn: match = ec['ec'] break if match: changed_ecs.append(match) else: # if no easyconfig is found, it's possible some archived easyconfigs were touched in the PR... # so as a last resort, try to find the easyconfig file in __archive__ easyconfigs_path = get_paths_for("easyconfigs")[0] specs = glob.glob('%s/__archive__/*/*/%s' % (easyconfigs_path, ec_fn)) if len(specs) == 1: ec = process_easyconfig(specs[0])[0] changed_ecs.append(ec['ec']) else: error_msg = "Failed to find parsed easyconfig for %s" % ec_fn error_msg += " (and could not isolate it in easyconfigs archive either)" self.assertTrue(False, error_msg) # run checks on changed easyconfigs self.check_sha256_checksums(changed_ecs) self.check_python_packages(changed_ecs)
def process_all_easyconfigs(self): """Process all easyconfigs and resolve inter-easyconfig dependencies.""" # all available easyconfig files easyconfigs_path = get_paths_for("easyconfigs")[0] specs = glob.glob('%s/*/*/*.eb' % easyconfigs_path) # parse all easyconfigs if they haven't been already if not self.parsed_easyconfigs: for spec in specs: self.parsed_easyconfigs.extend(process_easyconfig(spec)) self.ordered_specs = resolve_dependencies(self.parsed_easyconfigs)
def test_build_easyconfigs_in_parallel_gc3pie(self): """Test build_easyconfigs_in_parallel(), using GC3Pie with local config as backend for --job.""" try: import gc3libs # noqa (ignore unused import) except ImportError: print "GC3Pie not available, skipping test" return # put GC3Pie config in place to use local host and fork/exec resourcedir = os.path.join(self.test_prefix, 'gc3pie') gc3pie_cfgfile = os.path.join(self.test_prefix, 'gc3pie_local.ini') gc3pie_cfgtxt = GC3PIE_LOCAL_CONFIGURATION % { 'resourcedir': resourcedir, 'time': which('time'), } write_file(gc3pie_cfgfile, gc3pie_cfgtxt) output_dir = os.path.join(self.test_prefix, 'subdir', 'gc3pie_output_dir') # purposely pre-create output dir, and put a file in it (to check whether GC3Pie tries to rename the output dir) mkdir(output_dir, parents=True) write_file(os.path.join(output_dir, 'foo'), 'bar') # remove write permissions on parent dir of specified output dir, # to check that GC3Pie does not try to rename the (already existing) output directory... adjust_permissions(os.path.dirname(output_dir), stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH, add=False, recursive=False) topdir = os.path.dirname(os.path.abspath(__file__)) build_options = { 'job_backend_config': gc3pie_cfgfile, 'job_max_walltime': 24, 'job_output_dir': output_dir, 'job_polling_interval': 0.2, # quick polling 'job_target_resource': 'ebtestlocalhost', 'robot_path': os.path.join(topdir, 'easyconfigs', 'test_ecs'), 'silent': True, 'valid_module_classes': config.module_classes(), 'validate': False, } init_config(args=['--job-backend=GC3Pie'], build_options=build_options) ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') easyconfigs = process_easyconfig(ec_file) ordered_ecs = resolve_dependencies(easyconfigs, self.modtool) topdir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) test_easyblocks_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sandbox') cmd = "PYTHONPATH=%s:%s:$PYTHONPATH eb %%(spec)s -df" % (topdir, test_easyblocks_path) build_easyconfigs_in_parallel(cmd, ordered_ecs, prepare_first=False) self.assertTrue(os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0')) self.assertTrue(os.path.join(self.test_installpath, 'software', 'toy', '0.0', 'bin', 'toy'))
def regtest(easyconfig_paths, build_options=None, build_specs=None): """ Run regression test, using easyconfigs available in given path @param easyconfig_paths: path of easyconfigs to run regtest on @param build_options: dictionary specifying build options (e.g. robot_path, check_osdeps, ...) @param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...) """ cur_dir = os.getcwd() aggregate_regtest = build_options.get('aggregate_regtest', None) if aggregate_regtest is not None: output_file = os.path.join(aggregate_regtest, "%s-aggregate.xml" % os.path.basename(aggregate_regtest)) aggregate_xml_in_dirs(aggregate_regtest, output_file) _log.info("aggregated xml files inside %s, output written to: %s" % (aggregate_regtest, output_file)) sys.exit(0) # create base directory, which is used to place # all log files and the test output as xml basename = "easybuild-test-%s" % datetime.now().strftime("%Y%m%d%H%M%S") var = config.oldstyle_environment_variables['test_output_path'] regtest_output_dir = build_options.get('regtest_output_dir', None) if regtest_output_dir is not None: output_dir = regtest_output_dir elif var in os.environ: output_dir = os.path.abspath(os.environ[var]) else: # default: current dir + easybuild-test-[timestamp] output_dir = os.path.join(cur_dir, basename) if not os.path.isdir(output_dir): os.makedirs(output_dir) # find all easyconfigs ecfiles = [] if easyconfig_paths: for path in easyconfig_paths: ecfiles += find_easyconfigs(path, ignore_dirs=build_options.get('ignore_dirs', [])) else: _log.error("No easyconfig paths specified.") test_results = [] # process all the found easyconfig files easyconfigs = [] for ecfile in ecfiles: try: easyconfigs.extend(process_easyconfig(ecfile, build_options=build_options, build_specs=build_specs)) except EasyBuildError, err: test_results.append((ecfile, 'parsing_easyconfigs', 'easyconfig file error: %s' % err, _log))
def test_get_easyblock_instance(self): """Test get_easyblock_instance function.""" from easybuild.easyblocks.toy import EB_toy testdir = os.path.abspath(os.path.dirname(__file__)) ec = process_easyconfig(os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb'))[0] eb = get_easyblock_instance(ec) self.assertTrue(isinstance(eb, EB_toy)) # check whether 'This is easyblock' log message is there tup = ('EB_toy', 'easybuild.easyblocks.toy', '.*test/framework/sandbox/easybuild/easyblocks/t/toy.pyc*') eb_log_msg_re = re.compile(r"INFO This is easyblock %s from module %s (%s)" % tup, re.M) logtxt = read_file(eb.logfile) self.assertTrue(eb_log_msg_re.search(logtxt), "Pattern '%s' found in: %s" % (eb_log_msg_re.pattern, logtxt))
def regtest(easyconfig_paths, modtool, build_specs=None): """ Run regression test, using easyconfigs available in given path @param easyconfig_paths: path of easyconfigs to run regtest on @param modtool: ModulesTool instance to use @param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...) """ cur_dir = os.getcwd() aggregate_regtest = build_option('aggregate_regtest') if aggregate_regtest is not None: output_file = os.path.join(aggregate_regtest, "%s-aggregate.xml" % os.path.basename(aggregate_regtest)) aggregate_xml_in_dirs(aggregate_regtest, output_file) _log.info("aggregated xml files inside %s, output written to: %s" % (aggregate_regtest, output_file)) sys.exit(0) # create base directory, which is used to place all log files and the test output as xml regtest_output_dir = build_option('regtest_output_dir') testoutput = build_option('testoutput') if regtest_output_dir is not None: output_dir = regtest_output_dir elif testoutput is not None: output_dir = os.path.abspath(testoutput) else: # default: current dir + easybuild-test-[timestamp] dirname = "easybuild-test-%s" % datetime.now().strftime("%Y%m%d%H%M%S") output_dir = os.path.join(cur_dir, dirname) mkdir(output_dir, parents=True) # find all easyconfigs ecfiles = [] if easyconfig_paths: for path in easyconfig_paths: ecfiles += find_easyconfigs(path, ignore_dirs=build_option('ignore_dirs')) else: raise EasyBuildError("No easyconfig paths specified.") test_results = [] # process all the found easyconfig files easyconfigs = [] for ecfile in ecfiles: try: easyconfigs.extend(process_easyconfig(ecfile, build_specs=build_specs)) except EasyBuildError, err: test_results.append((ecfile, 'parsing_easyconfigs', 'easyconfig file error: %s' % err, _log))
def test_get_easyblock_instance(self): """Test get_easyblock_instance function.""" # adjust PYTHONPATH such that test easyblocks are found testdir = os.path.abspath(os.path.dirname(__file__)) import easybuild eb_blocks_path = os.path.join(testdir, 'sandbox') if not eb_blocks_path in sys.path: sys.path.append(eb_blocks_path) easybuild = reload(easybuild) import easybuild.easyblocks reload(easybuild.easyblocks) from easybuild.easyblocks.toy import EB_toy ec = process_easyconfig(os.path.join(testdir, 'easyconfigs', 'toy-0.0.eb'))[0] eb = get_easyblock_instance(ec) self.assertTrue(isinstance(eb, EB_toy))
def test_obtain_file(self): """Test obtain_file method.""" toy_tarball = 'toy-0.0.tar.gz' testdir = os.path.abspath(os.path.dirname(__file__)) sandbox_sources = os.path.join(testdir, 'sandbox', 'sources') toy_tarball_path = os.path.join(sandbox_sources, 'toy', toy_tarball) tmpdir = tempfile.mkdtemp() tmpdir_subdir = os.path.join(tmpdir, 'testing') mkdir(tmpdir_subdir, parents=True) del os.environ['EASYBUILD_SOURCEPATH'] # defined by setUp ec = process_easyconfig(os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb'))[0] eb = EasyBlock(ec['ec']) # 'downloading' a file to (first) sourcepath works init_config(args=["--sourcepath=%s:/no/such/dir:%s" % (tmpdir, testdir)]) shutil.copy2(toy_tarball_path, tmpdir_subdir) res = eb.obtain_file(toy_tarball, urls=['file://%s' % tmpdir_subdir]) self.assertEqual(res, os.path.join(tmpdir, 't', 'toy', toy_tarball)) # finding a file in sourcepath works init_config(args=["--sourcepath=%s:/no/such/dir:%s" % (sandbox_sources, tmpdir)]) res = eb.obtain_file(toy_tarball) self.assertEqual(res, toy_tarball_path) # sourcepath has preference over downloading res = eb.obtain_file(toy_tarball, urls=['file://%s' % tmpdir_subdir]) self.assertEqual(res, toy_tarball_path) # obtain_file yields error for non-existing files fn = 'thisisclearlyanonexistingfile' error_regex = "Couldn't find file %s anywhere, and downloading it didn't work either" % fn self.assertErrorRegex(EasyBuildError, error_regex, eb.obtain_file, fn, urls=['file://%s' % tmpdir_subdir]) # file specifications via URL also work, are downloaded to (first) sourcepath init_config(args=["--sourcepath=%s:/no/such/dir:%s" % (tmpdir, sandbox_sources)]) file_url = "http://hpcugent.github.io/easybuild/index.html" fn = os.path.basename(file_url) res = None try: res = eb.obtain_file(file_url) except EasyBuildError, err: # if this fails, it should be because there's no online access download_fail_regex = re.compile('socket error') self.assertTrue(download_fail_regex.search(str(err)))
def test_fetch_patches(self): """Test fetch_patches method.""" # adjust PYTHONPATH such that test easyblocks are found testdir = os.path.abspath(os.path.dirname(__file__)) ec = process_easyconfig( os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb'))[0] eb = get_easyblock_instance(ec) eb.fetch_patches() self.assertEqual(len(eb.patches), 2) self.assertEqual(eb.patches[0]['name'], 'toy-0.0_typo.patch') self.assertFalse('level' in eb.patches[0]) # reset eb.patches = [] patches = [ ('toy-0.0_typo.patch', 0), # should also be level 0 (not None or something else) ('toy-0.0_typo.patch', 4), # should be level 4 ('toy-0.0_typo.patch', 'foobar'), # sourcepath should be set to 'foobar' ('toy-0.0.tar.gz', 'some/path'), # copy mode (not a .patch file) ] # check if patch levels are parsed correctly eb.fetch_patches(patch_specs=patches) self.assertEqual(len(eb.patches), 4) self.assertEqual(eb.patches[0]['name'], 'toy-0.0_typo.patch') self.assertEqual(eb.patches[0]['level'], 0) self.assertEqual(eb.patches[1]['name'], 'toy-0.0_typo.patch') self.assertEqual(eb.patches[1]['level'], 4) self.assertEqual(eb.patches[2]['name'], 'toy-0.0_typo.patch') self.assertEqual(eb.patches[2]['sourcepath'], 'foobar') self.assertEqual(eb.patches[3]['name'], 'toy-0.0.tar.gz'), self.assertEqual(eb.patches[3]['copy'], 'some/path') patches = [ ('toy-0.0_level4.patch', False ), # should throw an error, only int's an strings allowed here ] self.assertRaises(EasyBuildError, eb.fetch_patches, patch_specs=patches)
def test_patch_step(self): """Test patch step.""" ec = process_easyconfig(os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'toy-0.0.eb'))[0] orig_sources = ec['ec']['sources'][:] # test applying patches without sources ec['ec']['sources'] = [] eb = EasyBlock(ec['ec']) eb.fetch_step() eb.extract_step() self.assertErrorRegex(EasyBuildError, '.*', eb.patch_step) # test actual patching of unpacked sources ec['ec']['sources'] = orig_sources eb = EasyBlock(ec['ec']) eb.fetch_step() eb.extract_step() eb.patch_step()
def process_all_easyconfigs(self): """Process all easyconfigs and resolve inter-easyconfig dependencies.""" # all available easyconfig files easyconfigs_path = get_paths_for("easyconfigs")[0] specs = glob.glob('%s/*/*/*.eb' % easyconfigs_path) # parse all easyconfigs if they haven't been already if not self.parsed_easyconfigs: for spec in specs: self.parsed_easyconfigs.extend(process_easyconfig(spec)) # filter out external modules for ec in self.parsed_easyconfigs: for dep in ec['dependencies'][:]: if dep.get('external_module', False): ec['dependencies'].remove(dep) self.ordered_specs = resolve_dependencies(self.parsed_easyconfigs, modules_tool(), retain_all_deps=True)
def test_obtain_file(self): """Test obtain_file method.""" toy_tarball = 'toy-0.0.tar.gz' testdir = os.path.abspath(os.path.dirname(__file__)) sandbox_sources = os.path.join(testdir, 'sandbox', 'sources') toy_tarball_path = os.path.join(sandbox_sources, 'toy', toy_tarball) tmpdir = tempfile.mkdtemp() tmpdir_subdir = os.path.join(tmpdir, 'testing') mkdir(tmpdir_subdir, parents=True) del os.environ['EASYBUILD_SOURCEPATH'] # defined by setUp ec = process_easyconfig( os.path.join(testdir, 'easyconfigs', 'toy-0.0.eb'))[0] eb = EasyBlock(ec['ec']) # 'downloading' a file to (first) sourcepath works init_config( args=["--sourcepath=%s:/no/such/dir:%s" % (tmpdir, testdir)]) shutil.copy2(toy_tarball_path, tmpdir_subdir) res = eb.obtain_file(toy_tarball, urls=[os.path.join('file://', tmpdir_subdir)]) self.assertEqual(res, os.path.join(tmpdir, 't', 'toy', toy_tarball)) # finding a file in sourcepath works init_config(args=[ "--sourcepath=%s:/no/such/dir:%s" % (sandbox_sources, tmpdir) ]) res = eb.obtain_file(toy_tarball) self.assertEqual(res, toy_tarball_path) # sourcepath has preference over downloading res = eb.obtain_file(toy_tarball, urls=[os.path.join('file://', tmpdir_subdir)]) self.assertEqual(res, toy_tarball_path) # obtain_file yields error for non-existing files fn = 'thisisclearlyanonexistingfile' try: eb.obtain_file(fn, urls=[os.path.join('file://', tmpdir_subdir)]) except EasyBuildError, err: fail_regex = re.compile( "Couldn't find file %s anywhere, and downloading it didn't work either" % fn) self.assertTrue(fail_regex.search(str(err)))
def test_fetch_patches(self): """Test fetch_patches method.""" # adjust PYTHONPATH such that test easyblocks are found testdir = os.path.abspath(os.path.dirname(__file__)) ec = process_easyconfig(os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb'))[0] eb = get_easyblock_instance(ec) eb.fetch_patches() self.assertEqual(len(eb.patches), 2) self.assertEqual(eb.patches[0]['name'], 'toy-0.0_typo.patch') self.assertFalse('level' in eb.patches[0]) # reset eb.patches = [] patches = [ ('toy-0.0_typo.patch', 0), # should also be level 0 (not None or something else) ('toy-0.0_typo.patch', 4), # should be level 4 ('toy-0.0_typo.patch', 'foobar'), # sourcepath should be set to 'foobar' ('toy-0.0.tar.gz', 'some/path'), # copy mode (not a .patch file) ] # check if patch levels are parsed correctly eb.fetch_patches(patch_specs=patches) self.assertEqual(len(eb.patches), 4) self.assertEqual(eb.patches[0]['name'], 'toy-0.0_typo.patch') self.assertEqual(eb.patches[0]['level'], 0) self.assertEqual(eb.patches[1]['name'], 'toy-0.0_typo.patch') self.assertEqual(eb.patches[1]['level'], 4) self.assertEqual(eb.patches[2]['name'], 'toy-0.0_typo.patch') self.assertEqual(eb.patches[2]['sourcepath'], 'foobar') self.assertEqual(eb.patches[3]['name'], 'toy-0.0.tar.gz'), self.assertEqual(eb.patches[3]['copy'], 'some/path') patches = [ ('toy-0.0_level4.patch', False), # should throw an error, only int's an strings allowed here ] self.assertRaises(EasyBuildError, eb.fetch_patches, patch_specs=patches)
def test_build_easyconfigs_in_parallel_pbs_python(self): """Test build_easyconfigs_in_parallel(), using (mocked) pbs_python as backend for --job.""" # put mocked functions in place PbsPython__init__ = PbsPython.__init__ PbsPython_check_version = PbsPython._check_version PbsPython_complete = PbsPython.complete PbsPython_connect_to_server = PbsPython.connect_to_server PbsPython_ppn = PbsPython.ppn pbs_python_PbsJob = pbs_python.PbsJob PbsPython.__init__ = lambda self: PbsPython__init__(self, pbs_server='localhost') PbsPython._check_version = lambda _: True PbsPython.complete = mock PbsPython.connect_to_server = mock PbsPython.ppn = mock pbs_python.PbsJob = MockPbsJob build_options = { 'robot_path': os.path.join(os.path.dirname(__file__), 'easyconfigs'), 'valid_module_classes': config.module_classes(), 'validate': False, } init_config(args=['--job-backend=PbsPython'], build_options=build_options) ec_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'gzip-1.5-goolf-1.4.10.eb') easyconfigs = process_easyconfig(ec_file) ordered_ecs = resolve_dependencies(easyconfigs) jobs = build_easyconfigs_in_parallel("echo %(spec)s", ordered_ecs, prepare_first=False) self.assertEqual(len(jobs), 8) # restore mocked stuff PbsPython.__init__ = PbsPython__init__ PbsPython._check_version = PbsPython_check_version PbsPython.complete = PbsPython_complete PbsPython.connect_to_server = PbsPython_connect_to_server PbsPython.ppn = PbsPython_ppn pbs_python.PbsJob = pbs_python_PbsJob
def test_obtain_file(self): """Test obtain_file method.""" toy_tarball = 'toy-0.0.tar.gz' testdir = os.path.abspath(os.path.dirname(__file__)) sandbox_sources = os.path.join(testdir, 'sandbox', 'sources') toy_tarball_path = os.path.join(sandbox_sources, 'toy', toy_tarball) tmpdir = tempfile.mkdtemp() tmpdir_subdir = os.path.join(tmpdir, 'testing') mkdir(tmpdir_subdir, parents=True) del os.environ['EASYBUILD_SOURCEPATH'] # defined by setUp ec = process_easyconfig(os.path.join(testdir, 'easyconfigs', 'toy-0.0.eb'))[0] eb = EasyBlock(ec['ec']) # 'downloading' a file to (first) sourcepath works init_config(args=["--sourcepath=%s:/no/such/dir:%s" % (tmpdir, testdir)]) shutil.copy2(toy_tarball_path, tmpdir_subdir) res = eb.obtain_file(toy_tarball, urls=[os.path.join('file://', tmpdir_subdir)]) self.assertEqual(res, os.path.join(tmpdir, 't', 'toy', toy_tarball)) # finding a file in sourcepath works init_config(args=["--sourcepath=%s:/no/such/dir:%s" % (sandbox_sources, tmpdir)]) res = eb.obtain_file(toy_tarball) self.assertEqual(res, toy_tarball_path) # sourcepath has preference over downloading res = eb.obtain_file(toy_tarball, urls=[os.path.join('file://', tmpdir_subdir)]) self.assertEqual(res, toy_tarball_path) # obtain_file yields error for non-existing files fn = 'thisisclearlyanonexistingfile' try: eb.obtain_file(fn, urls=[os.path.join('file://', tmpdir_subdir)]) except EasyBuildError, err: fail_regex = re.compile("Couldn't find file %s anywhere, and downloading it didn't work either" % fn) self.assertTrue(fail_regex.search(str(err)))
def template_easyconfig_test(self, spec): """Tests for an individual easyconfig: parsing, instantiating easyblock, check patches, ...""" # set to False, so it's False in case of this test failing global single_tests_ok prev_single_tests_ok = single_tests_ok single_tests_ok = False # parse easyconfig ecs = process_easyconfig(spec) if len(ecs) == 1: ec = ecs[0]['ec'] else: self.assertTrue(False, "easyconfig %s does not contain blocks, yields only one parsed easyconfig" % spec) # check easyconfig file name expected_fn = '%s-%s.eb' % (ec['name'], det_full_ec_version(ec)) msg = "Filename '%s' of parsed easyconfig matches expected filename '%s'" % (spec, expected_fn) self.assertEqual(os.path.basename(spec), expected_fn, msg) name, easyblock = fetch_parameters_from_easyconfig(ec.rawtxt, ['name', 'easyblock']) # make sure easyconfig file is in expected location expected_subdir = os.path.join('easybuild', 'easyconfigs', letter_dir_for(name), name) subdir = os.path.join(*spec.split(os.path.sep)[-5:-1]) fail_msg = "Easyconfig file %s not in expected subdirectory %s" % (spec, expected_subdir) self.assertEqual(expected_subdir, subdir, fail_msg) # sanity check for software name, moduleclass self.assertEqual(ec['name'], name) self.assertTrue(ec['moduleclass'] in build_option('valid_module_classes')) # instantiate easyblock with easyconfig file app_class = get_easyblock_class(easyblock, name=name) # check that automagic fallback to ConfigureMake isn't done (deprecated behaviour) fn = os.path.basename(spec) error_msg = "%s relies on automagic fallback to ConfigureMake, should use easyblock = 'ConfigureMake' instead" % fn self.assertTrue(easyblock or not app_class is ConfigureMake, error_msg) app = app_class(ec) # more sanity checks self.assertTrue(name, app.name) self.assertTrue(ec['version'], app.version) # make sure all patch files are available specdir = os.path.dirname(spec) specfn = os.path.basename(spec) for patch in ec['patches']: if isinstance(patch, (tuple, list)): patch = patch[0] # only check actual patch files, not other files being copied via the patch functionality if patch.endswith('.patch'): patch_full = os.path.join(specdir, patch) msg = "Patch file %s is available for %s" % (patch_full, specfn) self.assertTrue(os.path.isfile(patch_full), msg) ext_patches = [] for ext in ec['exts_list']: if isinstance(ext, (tuple, list)) and len(ext) == 3: self.assertTrue(isinstance(ext[2], dict), "3rd element of extension spec is a dictionary") for ext_patch in ext[2].get('patches', []): if isinstance(ext_patch, (tuple, list)): ext_patch = ext_patch[0] # only check actual patch files, not other files being copied via the patch functionality if ext_patch.endswith('.patch'): ext_patch_full = os.path.join(specdir, ext_patch) msg = "Patch file %s is available for %s" % (ext_patch_full, specfn) self.assertTrue(os.path.isfile(ext_patch_full), msg) # check whether all extra_options defined for used easyblock are defined extra_opts = app.extra_options() for key in extra_opts: self.assertTrue(key in app.cfg) app.close_log() os.remove(app.logfile) # dump the easyconfig file handle, test_ecfile = tempfile.mkstemp() os.close(handle) ec.dump(test_ecfile) dumped_ec = EasyConfigParser(test_ecfile).get_config_dict() os.remove(test_ecfile) # inject dummy values for templates that are only known at a later stage dummy_template_values = { 'builddir': '/dummy/builddir', 'installdir': '/dummy/installdir', } ec.template_values.update(dummy_template_values) ec_dict = ec.parser.get_config_dict() orig_toolchain = ec_dict['toolchain'] for key in ec_dict: # skip parameters for which value is equal to default value orig_val = ec_dict[key] if key in DEFAULT_CONFIG and orig_val == DEFAULT_CONFIG[key][0]: continue if key in extra_opts and orig_val == extra_opts[key][0]: continue if key not in DEFAULT_CONFIG and key not in extra_opts: continue orig_val = resolve_template(ec_dict[key], ec.template_values) dumped_val = resolve_template(dumped_ec[key], ec.template_values) # take into account that dumped value for *dependencies may include hard-coded subtoolchains # if no easyconfig was found for the dependency with the 'parent' toolchain, # if may get resolved using a subtoolchain, which is then hardcoded in the dumped easyconfig if key in DEPENDENCY_PARAMETERS: # number of dependencies should remain the same self.assertEqual(len(orig_val), len(dumped_val)) for orig_dep, dumped_dep in zip(orig_val, dumped_val): # name/version should always match self.assertEqual(orig_dep[:2], dumped_dep[:2]) # 3rd value is versionsuffix; if len(dumped_dep) >= 3: # if no versionsuffix was specified in original dep spec, then dumped value should be empty string if len(orig_dep) >= 3: self.assertEqual(dumped_dep[2], orig_dep[2]) else: self.assertEqual(dumped_dep[2], '') # 4th value is toolchain spec if len(dumped_dep) >= 4: if len(orig_dep) >= 4: self.assertEqual(dumped_dep[3], orig_dep[3]) else: # if a subtoolchain is specifed (only) in the dumped easyconfig, # it should *not* be the same as the parent toolchain self.assertNotEqual(dumped_dep[3], (orig_toolchain['name'], orig_toolchain['version'])) else: self.assertEqual(orig_val, dumped_val) # cache the parsed easyconfig, to avoid that it is parsed again self.parsed_easyconfigs.append(ecs[0]) # test passed, so set back to True single_tests_ok = True and prev_single_tests_ok
def test_build_easyconfigs_in_parallel_pbs_python(self): """Test build_easyconfigs_in_parallel(), using (mocked) pbs_python as backend for --job.""" # put mocked functions in place PbsPython__init__ = PbsPython.__init__ PbsPython_check_version = PbsPython._check_version PbsPython_complete = PbsPython.complete PbsPython_connect_to_server = PbsPython.connect_to_server PbsPython_ppn = PbsPython.ppn pbs_python_PbsJob = pbs_python.PbsJob PbsPython.__init__ = lambda self: PbsPython__init__(self, pbs_server='localhost') PbsPython._check_version = lambda _: True PbsPython.complete = mock PbsPython.connect_to_server = mock PbsPython.ppn = mock pbs_python.PbsJob = MockPbsJob topdir = os.path.dirname(os.path.abspath(__file__)) build_options = { 'external_modules_metadata': {}, 'robot_path': os.path.join(topdir, 'easyconfigs', 'test_ecs'), 'valid_module_classes': config.module_classes(), 'validate': False, 'job_cores': 3, } init_config(args=['--job-backend=PbsPython'], build_options=build_options) ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'g', 'gzip', 'gzip-1.5-foss-2018a.eb') easyconfigs = process_easyconfig(ec_file) ordered_ecs = resolve_dependencies(easyconfigs, self.modtool) jobs = build_easyconfigs_in_parallel("echo '%(spec)s'", ordered_ecs, prepare_first=False) # only one job submitted since foss/2018a module is already available self.assertEqual(len(jobs), 1) regex = re.compile("echo '.*/gzip-1.5-foss-2018a.eb'") self.assertTrue(regex.search(jobs[-1].script), "Pattern '%s' found in: %s" % (regex.pattern, jobs[-1].script)) ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'g', 'gzip', 'gzip-1.4-GCC-4.6.3.eb') ordered_ecs = resolve_dependencies(process_easyconfig(ec_file), self.modtool, retain_all_deps=True) jobs = submit_jobs(ordered_ecs, '', testing=False, prepare_first=False) # make sure command is correct, and that --hidden is there when it needs to be for i, ec in enumerate(ordered_ecs): if ec['hidden']: regex = re.compile("eb %s.* --hidden" % ec['spec']) else: regex = re.compile("eb %s" % ec['spec']) self.assertTrue(regex.search(jobs[i].script), "Pattern '%s' found in: %s" % (regex.pattern, jobs[i].script)) for job in jobs: self.assertEqual(job.cores, build_options['job_cores']) # no deps for GCC/4.6.3 (toolchain) and intel/2018a (test easyconfig with 'fake' deps) self.assertEqual(len(jobs[0].deps), 0) self.assertEqual(len(jobs[1].deps), 0) # only dependency for toy/0.0-deps is intel/2018a (dep marked as external module is filtered out) self.assertTrue('toy-0.0-deps.eb' in jobs[2].script) self.assertEqual(len(jobs[2].deps), 1) self.assertTrue('intel-2018a.eb' in jobs[2].deps[0].script) # dependencies for gzip/1.4-GCC-4.6.3: GCC/4.6.3 (toolchain) + toy/.0.0-deps self.assertTrue('gzip-1.4-GCC-4.6.3.eb' in jobs[3].script) self.assertEqual(len(jobs[3].deps), 2) regex = re.compile(r'toy-0.0-deps\.eb.* --hidden') script_txt = jobs[3].deps[0].script fail_msg = "Pattern '%s' should be found in: %s" % (regex.pattern, script_txt) self.assertTrue(regex.search(script_txt), fail_msg) self.assertTrue('GCC-4.6.3.eb' in jobs[3].deps[1].script) # also test use of --pre-create-installdir ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') ordered_ecs = resolve_dependencies(process_easyconfig(ec_file), self.modtool) # installation directory doesn't exist yet before submission toy_installdir = os.path.join(self.test_installpath, 'software', 'toy', '0.0') self.assertFalse(os.path.exists(toy_installdir)) jobs = submit_jobs(ordered_ecs, '', testing=False) self.assertEqual(len(jobs), 1) # software install dir is created (by default) as part of job submission process (fetch_step is run) self.assertTrue(os.path.exists(toy_installdir)) remove_dir(toy_installdir) remove_dir(os.path.dirname(toy_installdir)) self.assertFalse(os.path.exists(toy_installdir)) # installation directory does *not* get created when --pre-create-installdir is used build_options['pre_create_installdir'] = False init_config(args=['--job-backend=PbsPython'], build_options=build_options) jobs = submit_jobs(ordered_ecs, '', testing=False) self.assertEqual(len(jobs), 1) self.assertFalse(os.path.exists(toy_installdir)) # restore mocked stuff PbsPython.__init__ = PbsPython__init__ PbsPython._check_version = PbsPython_check_version PbsPython.complete = PbsPython_complete PbsPython.connect_to_server = PbsPython_connect_to_server PbsPython.ppn = PbsPython_ppn pbs_python.PbsJob = pbs_python_PbsJob
def template_easyconfig_test(self, spec): """Tests for an individual easyconfig: parsing, instantiating easyblock, check patches, ...""" # set to False, so it's False in case of this test failing global single_tests_ok prev_single_tests_ok = single_tests_ok single_tests_ok = False # parse easyconfig ecs = process_easyconfig(spec) if len(ecs) == 1: ec = ecs[0]['ec'] else: self.assertTrue(False, "easyconfig %s does not contain blocks, yields only one parsed easyconfig" % spec) # check easyconfig file name expected_fn = '%s-%s.eb' % (ec['name'], det_full_ec_version(ec)) msg = "Filename '%s' of parsed easconfig matches expected filename '%s'" % (spec, expected_fn) self.assertEqual(os.path.basename(spec), expected_fn, msg) name, easyblock = fetch_parameters_from_easyconfig(ec.rawtxt, ['name', 'easyblock']) # sanity check for software name self.assertTrue(ec['name'], name) # instantiate easyblock with easyconfig file app_class = get_easyblock_class(easyblock, name=name) # check that automagic fallback to ConfigureMake isn't done (deprecated behaviour) fn = os.path.basename(spec) error_msg = "%s relies on automagic fallback to ConfigureMake, should use easyblock = 'ConfigureMake' instead" % fn self.assertTrue(easyblock or not app_class is ConfigureMake, error_msg) app = app_class(ec) # more sanity checks self.assertTrue(name, app.name) self.assertTrue(ec['version'], app.version) # make sure all patch files are available specdir = os.path.dirname(spec) specfn = os.path.basename(spec) for patch in ec['patches']: if isinstance(patch, (tuple, list)): patch = patch[0] # only check actual patch files, not other files being copied via the patch functionality if patch.endswith('.patch'): patch_full = os.path.join(specdir, patch) msg = "Patch file %s is available for %s" % (patch_full, specfn) self.assertTrue(os.path.isfile(patch_full), msg) ext_patches = [] for ext in ec['exts_list']: if isinstance(ext, (tuple, list)) and len(ext) == 3: self.assertTrue(isinstance(ext[2], dict), "3rd element of extension spec is a dictionary") for ext_patch in ext[2].get('patches', []): if isinstance(ext_patch, (tuple, list)): ext_patch = ext_patch[0] # only check actual patch files, not other files being copied via the patch functionality if ext_patch.endswith('.patch'): ext_patch_full = os.path.join(specdir, ext_patch) msg = "Patch file %s is available for %s" % (ext_patch_full, specfn) self.assertTrue(os.path.isfile(ext_patch_full), msg) # check whether all extra_options defined for used easyblock are defined for key in app.extra_options(): self.assertTrue(key in app.cfg) app.close_log() os.remove(app.logfile) # cache the parsed easyconfig, to avoid that it is parsed again self.parsed_easyconfigs.append(ecs[0]) # test passed, so set back to True single_tests_ok = True and prev_single_tests_ok
def template_easyconfig_test(self, spec): """Test whether all easyconfigs can be initialized.""" # set to False, so it's False in case of this test failing global single_tests_ok prev_single_tests_ok = single_tests_ok single_tests_ok = False # parse easyconfig ecs = process_easyconfig(spec) if len(ecs) == 1: ec = ecs[0]['ec'] else: self.assertTrue(False, "easyconfig %s does not contain blocks, yields only one parsed easyconfig" % spec) # sanity check for software name name = fetch_parameter_from_easyconfig_file(spec, 'name') self.assertTrue(ec['name'], name) # try and fetch easyblock spec from easyconfig easyblock = fetch_parameter_from_easyconfig_file(spec, 'easyblock') # instantiate easyblock with easyconfig file app_class = get_easyblock_class(easyblock, name=name) app = app_class(ec) # more sanity checks self.assertTrue(name, app.name) self.assertTrue(ec['version'], app.version) # make sure all patch files are available specdir = os.path.dirname(spec) specfn = os.path.basename(spec) for patch in ec['patches']: if isinstance(patch, (tuple, list)): patch = patch[0] # only check actual patch files, not other files being copied via the patch functionality if patch.endswith('.patch'): patch_full = os.path.join(specdir, patch) msg = "Patch file %s is available for %s" % (patch_full, specfn) self.assertTrue(os.path.isfile(patch_full), msg) ext_patches = [] for ext in ec['exts_list']: if isinstance(ext, (tuple, list)) and len(ext) == 3: self.assertTrue(isinstance(ext[2], dict), "3rd element of extension spec is a dictionary") for ext_patch in ext[2].get('patches', []): if isinstance(ext_patch, (tuple, list)): ext_patch = ext_patch[0] # only check actual patch files, not other files being copied via the patch functionality if ext_patch.endswith('.patch'): ext_patch_full = os.path.join(specdir, ext_patch) msg = "Patch file %s is available for %s" % (ext_patch_full, specfn) self.assertTrue(os.path.isfile(ext_patch_full), msg) app.close_log() os.remove(app.logfile) # cache the parsed easyconfig, to avoid that it is parsed again self.parsed_easyconfigs.append(ecs[0]) # test passed, so set back to True single_tests_ok = True and prev_single_tests_ok
def test_build_easyconfigs_in_parallel_pbs_python(self): """Test build_easyconfigs_in_parallel(), using (mocked) pbs_python as backend for --job.""" # put mocked functions in place PbsPython__init__ = PbsPython.__init__ PbsPython_check_version = PbsPython._check_version PbsPython_complete = PbsPython.complete PbsPython_connect_to_server = PbsPython.connect_to_server PbsPython_ppn = PbsPython.ppn pbs_python_PbsJob = pbs_python.PbsJob PbsPython.__init__ = lambda self: PbsPython__init__(self, pbs_server='localhost') PbsPython._check_version = lambda _: True PbsPython.complete = mock PbsPython.connect_to_server = mock PbsPython.ppn = mock pbs_python.PbsJob = MockPbsJob topdir = os.path.dirname(os.path.abspath(__file__)) build_options = { 'external_modules_metadata': {}, 'robot_path': os.path.join(topdir, 'easyconfigs', 'test_ecs'), 'valid_module_classes': config.module_classes(), 'validate': False, 'job_cores': 3, } init_config(args=['--job-backend=PbsPython'], build_options=build_options) ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'g', 'gzip', 'gzip-1.5-foss-2018a.eb') easyconfigs = process_easyconfig(ec_file) ordered_ecs = resolve_dependencies(easyconfigs, self.modtool) jobs = build_easyconfigs_in_parallel("echo '%(spec)s'", ordered_ecs, prepare_first=False) # only one job submitted since foss/2018a module is already available self.assertEqual(len(jobs), 1) regex = re.compile("echo '.*/gzip-1.5-foss-2018a.eb'") self.assertTrue(regex.search(jobs[-1].script), "Pattern '%s' found in: %s" % (regex.pattern, jobs[-1].script)) ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'g', 'gzip', 'gzip-1.4-GCC-4.6.3.eb') ordered_ecs = resolve_dependencies(process_easyconfig(ec_file), self.modtool, retain_all_deps=True) jobs = submit_jobs(ordered_ecs, '', testing=False, prepare_first=False) # make sure command is correct, and that --hidden is there when it needs to be for i, ec in enumerate(ordered_ecs): if ec['hidden']: regex = re.compile("eb %s.* --hidden" % ec['spec']) else: regex = re.compile("eb %s" % ec['spec']) self.assertTrue(regex.search(jobs[i].script), "Pattern '%s' found in: %s" % (regex.pattern, jobs[i].script)) for job in jobs: self.assertEqual(job.cores, build_options['job_cores']) # no deps for GCC/4.6.3 (toolchain) and intel/2018a (test easyconfig with 'fake' deps) self.assertEqual(len(jobs[0].deps), 0) self.assertEqual(len(jobs[1].deps), 0) # only dependency for toy/0.0-deps is intel/2018a (dep marked as external module is filtered out) self.assertTrue('toy-0.0-deps.eb' in jobs[2].script) self.assertEqual(len(jobs[2].deps), 1) self.assertTrue('intel-2018a.eb' in jobs[2].deps[0].script) # dependencies for gzip/1.4-GCC-4.6.3: GCC/4.6.3 (toolchain) + toy/.0.0-deps self.assertTrue('gzip-1.4-GCC-4.6.3.eb' in jobs[3].script) self.assertEqual(len(jobs[3].deps), 2) regex = re.compile('toy-0.0-deps.eb\s* --hidden') self.assertTrue(regex.search(jobs[3].deps[0].script)) self.assertTrue('GCC-4.6.3.eb' in jobs[3].deps[1].script) # also test use of --pre-create-installdir ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') ordered_ecs = resolve_dependencies(process_easyconfig(ec_file), self.modtool) # installation directory doesn't exist yet before submission toy_installdir = os.path.join(self.test_installpath, 'software', 'toy', '0.0') self.assertFalse(os.path.exists(toy_installdir)) jobs = submit_jobs(ordered_ecs, '', testing=False) self.assertEqual(len(jobs), 1) # software install dir is created (by default) as part of job submission process (fetch_step is run) self.assertTrue(os.path.exists(toy_installdir)) remove_dir(toy_installdir) remove_dir(os.path.dirname(toy_installdir)) self.assertFalse(os.path.exists(toy_installdir)) # installation directory does *not* get created when --pre-create-installdir is used build_options['pre_create_installdir'] = False init_config(args=['--job-backend=PbsPython'], build_options=build_options) jobs = submit_jobs(ordered_ecs, '', testing=False) self.assertEqual(len(jobs), 1) self.assertFalse(os.path.exists(toy_installdir)) # restore mocked stuff PbsPython.__init__ = PbsPython__init__ PbsPython._check_version = PbsPython_check_version PbsPython.complete = PbsPython_complete PbsPython.connect_to_server = PbsPython_connect_to_server PbsPython.ppn = PbsPython_ppn pbs_python.PbsJob = pbs_python_PbsJob
def regtest(easyconfig_paths, modtool, build_specs=None): """ Run regression test, using easyconfigs available in given path :param easyconfig_paths: path of easyconfigs to run regtest on :param modtool: ModulesTool instance to use :param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...) """ cur_dir = os.getcwd() aggregate_regtest = build_option('aggregate_regtest') if aggregate_regtest is not None: output_file = os.path.join( aggregate_regtest, "%s-aggregate.xml" % os.path.basename(aggregate_regtest)) aggregate_xml_in_dirs(aggregate_regtest, output_file) _log.info("aggregated xml files inside %s, output written to: %s" % (aggregate_regtest, output_file)) sys.exit(0) # create base directory, which is used to place all log files and the test output as xml regtest_output_dir = build_option('regtest_output_dir') testoutput = build_option('testoutput') if regtest_output_dir is not None: output_dir = regtest_output_dir elif testoutput is not None: output_dir = os.path.abspath(testoutput) else: # default: current dir + easybuild-test-[timestamp] dirname = "easybuild-test-%s" % datetime.now().strftime("%Y%m%d%H%M%S") output_dir = os.path.join(cur_dir, dirname) mkdir(output_dir, parents=True) # find all easyconfigs ecfiles = [] if easyconfig_paths: for path in easyconfig_paths: ecfiles += find_easyconfigs( path, ignore_dirs=build_option('ignore_dirs')) else: raise EasyBuildError("No easyconfig paths specified.") test_results = [] # process all the found easyconfig files easyconfigs = [] for ecfile in ecfiles: try: easyconfigs.extend( process_easyconfig(ecfile, build_specs=build_specs)) except EasyBuildError as err: test_results.append((ecfile, 'parsing_easyconfigs', 'easyconfig file error: %s' % err, _log)) # skip easyconfigs for which a module is already available, unless forced if not build_option('force'): _log.debug( "Skipping easyconfigs from %s that already have a module available..." % easyconfigs) easyconfigs = skip_available(easyconfigs, modtool) _log.debug("Retained easyconfigs after skipping: %s" % easyconfigs) if build_option('sequential'): return build_easyconfigs(easyconfigs, output_dir, test_results) else: resolved = resolve_dependencies(easyconfigs, modtool) cmd = "eb %(spec)s --regtest --sequential -ld --testoutput=%(output_dir)s" command = "unset TMPDIR && cd %s && %s; " % (cur_dir, cmd) # retry twice in case of failure, to avoid fluke errors command += "if [ $? -ne 0 ]; then %(cmd)s --force && %(cmd)s --force; fi" % { 'cmd': cmd } build_easyconfigs_in_parallel(command, resolved, output_dir=output_dir) _log.info("Submitted regression test as jobs, results in %s" % output_dir) return True # success
def template_easyconfig_test(self, spec): """Tests for an individual easyconfig: parsing, instantiating easyblock, check patches, ...""" # set to False, so it's False in case of this test failing global single_tests_ok prev_single_tests_ok = single_tests_ok single_tests_ok = False # parse easyconfig ecs = process_easyconfig(spec) if len(ecs) == 1: ec = ecs[0]['ec'] # cache the parsed easyconfig, to avoid that it is parsed again self.parsed_easyconfigs.append(ecs[0]) else: self.assertTrue( False, "easyconfig %s does not contain blocks, yields only one parsed easyconfig" % spec) # check easyconfig file name expected_fn = '%s-%s.eb' % (ec['name'], det_full_ec_version(ec)) msg = "Filename '%s' of parsed easyconfig matches expected filename '%s'" % ( spec, expected_fn) self.assertEqual(os.path.basename(spec), expected_fn, msg) name, easyblock = fetch_parameters_from_easyconfig(ec.rawtxt, ['name', 'easyblock']) # make sure easyconfig file is in expected location expected_subdir = os.path.join('easybuild', 'easyconfigs', letter_dir_for(name), name) subdir = os.path.join(*spec.split(os.path.sep)[-5:-1]) fail_msg = "Easyconfig file %s not in expected subdirectory %s" % ( spec, expected_subdir) self.assertEqual(expected_subdir, subdir, fail_msg) # sanity check for software name, moduleclass self.assertEqual(ec['name'], name) self.assertTrue(ec['moduleclass'] in build_option('valid_module_classes')) # instantiate easyblock with easyconfig file app_class = get_easyblock_class(easyblock, name=name) # check that automagic fallback to ConfigureMake isn't done (deprecated behaviour) fn = os.path.basename(spec) error_msg = "%s relies on automagic fallback to ConfigureMake, should use easyblock = 'ConfigureMake' instead" % fn self.assertTrue(easyblock or app_class is not ConfigureMake, error_msg) app = app_class(ec) # more sanity checks self.assertTrue(name, app.name) self.assertTrue(ec['version'], app.version) # make sure that $root is not used, since it is not compatible with module files in Lua syntax res = re.findall('.*\$root.*', ec.rawtxt, re.M) error_msg = "Found use of '$root', not compatible with modules in Lua syntax, use '%%(installdir)s' instead: %s" self.assertFalse(res, error_msg % res) # make sure old GitHub urls for EasyBuild that include 'hpcugent' are no longer used old_urls = [ 'github.com/hpcugent/easybuild', 'hpcugent.github.com/easybuild', 'hpcugent.github.io/easybuild', ] for old_url in old_urls: self.assertFalse(old_url in ec.rawtxt, "Old URL '%s' not found in %s" % (old_url, spec)) # make sure binutils is included as a build dep if toolchain is GCCcore if ec['toolchain']['name'] == 'GCCcore': # with 'Tarball' easyblock: only unpacking, no building; Eigen is also just a tarball requires_binutils = ec['easyblock'] not in [ 'Tarball' ] and ec['name'] not in ['Eigen'] # let's also exclude the very special case where the system GCC is used as GCCcore, and only apply this # exception to the dependencies of binutils (since we should eventually build a new binutils with GCCcore) if ec['toolchain']['version'] == 'system': binutils_complete_dependencies = [ 'M4', 'Bison', 'flex', 'help2man', 'zlib', 'binutils' ] requires_binutils &= bool( ec['name'] not in binutils_complete_dependencies) # if no sources/extensions/components are specified, it's just a bundle (nothing is being compiled) requires_binutils &= bool(ec['sources'] or ec['exts_list'] or ec.get('components')) if requires_binutils: dep_names = [d['name'] for d in ec.builddependencies()] self.assertTrue( 'binutils' in dep_names, "binutils is a build dep in %s: %s" % (spec, dep_names)) # make sure all patch files are available specdir = os.path.dirname(spec) specfn = os.path.basename(spec) for patch in ec['patches']: if isinstance(patch, (tuple, list)): patch = patch[0] # only check actual patch files, not other files being copied via the patch functionality if patch.endswith('.patch'): patch_full = os.path.join(specdir, patch) msg = "Patch file %s is available for %s" % (patch_full, specfn) self.assertTrue(os.path.isfile(patch_full), msg) for ext in ec['exts_list']: if isinstance(ext, (tuple, list)) and len(ext) == 3: self.assertTrue(isinstance(ext[2], dict), "3rd element of extension spec is a dictionary") for ext_patch in ext[2].get('patches', []): if isinstance(ext_patch, (tuple, list)): ext_patch = ext_patch[0] # only check actual patch files, not other files being copied via the patch functionality if ext_patch.endswith('.patch'): ext_patch_full = os.path.join(specdir, ext_patch) msg = "Patch file %s is available for %s" % ( ext_patch_full, specfn) self.assertTrue(os.path.isfile(ext_patch_full), msg) # check whether all extra_options defined for used easyblock are defined extra_opts = app.extra_options() for key in extra_opts: self.assertTrue(key in app.cfg) app.close_log() os.remove(app.logfile) # dump the easyconfig file handle, test_ecfile = tempfile.mkstemp() os.close(handle) ec.dump(test_ecfile) dumped_ec = EasyConfigParser(test_ecfile).get_config_dict() os.remove(test_ecfile) # inject dummy values for templates that are only known at a later stage dummy_template_values = { 'builddir': '/dummy/builddir', 'installdir': '/dummy/installdir', } ec.template_values.update(dummy_template_values) ec_dict = ec.parser.get_config_dict() orig_toolchain = ec_dict['toolchain'] for key in ec_dict: # skip parameters for which value is equal to default value orig_val = ec_dict[key] if key in DEFAULT_CONFIG and orig_val == DEFAULT_CONFIG[key][0]: continue if key in extra_opts and orig_val == extra_opts[key][0]: continue if key not in DEFAULT_CONFIG and key not in extra_opts: continue orig_val = resolve_template(ec_dict[key], ec.template_values) dumped_val = resolve_template(dumped_ec[key], ec.template_values) # take into account that dumped value for *dependencies may include hard-coded subtoolchains # if no easyconfig was found for the dependency with the 'parent' toolchain, # if may get resolved using a subtoolchain, which is then hardcoded in the dumped easyconfig if key in DEPENDENCY_PARAMETERS: # number of dependencies should remain the same self.assertEqual(len(orig_val), len(dumped_val)) for orig_dep, dumped_dep in zip(orig_val, dumped_val): # name/version should always match self.assertEqual(orig_dep[:2], dumped_dep[:2]) # 3rd value is versionsuffix; if len(dumped_dep) >= 3: # if no versionsuffix was specified in original dep spec, then dumped value should be empty string if len(orig_dep) >= 3: self.assertEqual(dumped_dep[2], orig_dep[2]) else: self.assertEqual(dumped_dep[2], '') # 4th value is toolchain spec if len(dumped_dep) >= 4: if len(orig_dep) >= 4: self.assertEqual(dumped_dep[3], orig_dep[3]) else: # if a subtoolchain is specifed (only) in the dumped easyconfig, # it should *not* be the same as the parent toolchain self.assertNotEqual(dumped_dep[3], (orig_toolchain['name'], orig_toolchain['version'])) else: self.assertEqual(orig_val, dumped_val) # test passed, so set back to True single_tests_ok = True and prev_single_tests_ok
def template_easyconfig_test(self, spec): """Test whether all easyconfigs can be initialized.""" # set to False, so it's False in case of this test failing global single_tests_ok prev_single_tests_ok = single_tests_ok single_tests_ok = False # parse easyconfig ecs = process_easyconfig(spec) if len(ecs) == 1: ec = ecs[0]['ec'] else: self.assertTrue( False, "easyconfig %s does not contain blocks, yields only one parsed easyconfig" % spec) # sanity check for software name name = fetch_parameter_from_easyconfig_file(spec, 'name') self.assertTrue(ec['name'], name) # try and fetch easyblock spec from easyconfig easyblock = fetch_parameter_from_easyconfig_file(spec, 'easyblock') # instantiate easyblock with easyconfig file app_class = get_easyblock_class(easyblock, name=name) app = app_class(ec) # more sanity checks self.assertTrue(name, app.name) self.assertTrue(ec['version'], app.version) # make sure all patch files are available specdir = os.path.dirname(spec) specfn = os.path.basename(spec) for patch in ec['patches']: if isinstance(patch, (tuple, list)): patch = patch[0] # only check actual patch files, not other files being copied via the patch functionality if patch.endswith('.patch'): patch_full = os.path.join(specdir, patch) msg = "Patch file %s is available for %s" % (patch_full, specfn) self.assertTrue(os.path.isfile(patch_full), msg) ext_patches = [] for ext in ec['exts_list']: if isinstance(ext, (tuple, list)) and len(ext) == 3: self.assertTrue(isinstance(ext[2], dict), "3rd element of extension spec is a dictionary") for ext_patch in ext[2].get('patches', []): if isinstance(ext_patch, (tuple, list)): ext_patch = ext_patch[0] # only check actual patch files, not other files being copied via the patch functionality if ext_patch.endswith('.patch'): ext_patch_full = os.path.join(specdir, ext_patch) msg = "Patch file %s is available for %s" % ( ext_patch_full, specfn) self.assertTrue(os.path.isfile(ext_patch_full), msg) app.close_log() os.remove(app.logfile) # cache the parsed easyconfig, to avoid that it is parsed again self.parsed_easyconfigs.append(ecs[0]) # test passed, so set back to True single_tests_ok = True and prev_single_tests_ok
def test_build_easyconfigs_in_parallel_pbs_python(self): """Test build_easyconfigs_in_parallel(), using (mocked) pbs_python as backend for --job.""" # put mocked functions in place PbsPython__init__ = PbsPython.__init__ PbsPython_check_version = PbsPython._check_version PbsPython_complete = PbsPython.complete PbsPython_connect_to_server = PbsPython.connect_to_server PbsPython_ppn = PbsPython.ppn pbs_python_PbsJob = pbs_python.PbsJob PbsPython.__init__ = lambda self: PbsPython__init__( self, pbs_server='localhost') PbsPython._check_version = lambda _: True PbsPython.complete = mock PbsPython.connect_to_server = mock PbsPython.ppn = mock pbs_python.PbsJob = MockPbsJob topdir = os.path.dirname(os.path.abspath(__file__)) build_options = { 'external_modules_metadata': {}, 'robot_path': os.path.join(topdir, 'easyconfigs', 'test_ecs'), 'valid_module_classes': config.module_classes(), 'validate': False, 'job_cores': 3, } init_config(args=['--job-backend=PbsPython'], build_options=build_options) ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'g', 'gzip', 'gzip-1.5-goolf-1.4.10.eb') easyconfigs = process_easyconfig(ec_file) ordered_ecs = resolve_dependencies(easyconfigs, self.modtool) jobs = build_easyconfigs_in_parallel("echo '%(spec)s'", ordered_ecs, prepare_first=False) self.assertEqual(len(jobs), 8) regex = re.compile("echo '.*/gzip-1.5-goolf-1.4.10.eb'") self.assertTrue( regex.search(jobs[-1].script), "Pattern '%s' found in: %s" % (regex.pattern, jobs[-1].script)) ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'g', 'gzip', 'gzip-1.4-GCC-4.6.3.eb') ordered_ecs = resolve_dependencies(process_easyconfig(ec_file), self.modtool, retain_all_deps=True) jobs = submit_jobs(ordered_ecs, '', testing=False, prepare_first=False) # make sure command is correct, and that --hidden is there when it needs to be for i, ec in enumerate(ordered_ecs): if ec['hidden']: regex = re.compile("eb %s.* --hidden" % ec['spec']) else: regex = re.compile("eb %s" % ec['spec']) self.assertTrue( regex.search(jobs[i].script), "Pattern '%s' found in: %s" % (regex.pattern, jobs[i].script)) for job in jobs: self.assertEqual(job.cores, build_options['job_cores']) # no deps for GCC/4.6.3 (toolchain) and ictce/4.1.13 (test easyconfig with 'fake' deps) self.assertEqual(len(jobs[0].deps), 0) self.assertEqual(len(jobs[1].deps), 0) # only dependency for toy/0.0-deps is ictce/4.1.13 (dep marked as external module is filtered out) self.assertTrue('toy-0.0-deps.eb' in jobs[2].script) self.assertEqual(len(jobs[2].deps), 1) self.assertTrue('ictce-4.1.13.eb' in jobs[2].deps[0].script) # dependencies for gzip/1.4-GCC-4.6.3: GCC/4.6.3 (toolchain) + toy/.0.0-deps self.assertTrue('gzip-1.4-GCC-4.6.3.eb' in jobs[3].script) self.assertEqual(len(jobs[3].deps), 2) regex = re.compile('toy-0.0-deps.eb\s* --hidden') self.assertTrue(regex.search(jobs[3].deps[0].script)) self.assertTrue('GCC-4.6.3.eb' in jobs[3].deps[1].script) # restore mocked stuff PbsPython.__init__ = PbsPython__init__ PbsPython._check_version = PbsPython_check_version PbsPython.complete = PbsPython_complete PbsPython.connect_to_server = PbsPython_connect_to_server PbsPython.ppn = PbsPython_ppn pbs_python.PbsJob = pbs_python_PbsJob
def template_easyconfig_test(self, spec): """Tests for an individual easyconfig: parsing, instantiating easyblock, check patches, ...""" # set to False, so it's False in case of this test failing global single_tests_ok prev_single_tests_ok = single_tests_ok single_tests_ok = False # parse easyconfig ecs = process_easyconfig(spec) if len(ecs) == 1: ec = ecs[0]['ec'] else: self.assertTrue(False, "easyconfig %s does not contain blocks, yields only one parsed easyconfig" % spec) # check easyconfig file name expected_fn = '%s-%s.eb' % (ec['name'], det_full_ec_version(ec)) msg = "Filename '%s' of parsed easyconfig matches expected filename '%s'" % (spec, expected_fn) self.assertEqual(os.path.basename(spec), expected_fn, msg) name, easyblock = fetch_parameters_from_easyconfig(ec.rawtxt, ['name', 'easyblock']) # make sure easyconfig file is in expected location expected_subdir = os.path.join('easybuild', 'easyconfigs', letter_dir_for(name), name) subdir = os.path.join(*spec.split(os.path.sep)[-5:-1]) fail_msg = "Easyconfig file %s not in expected subdirectory %s" % (spec, expected_subdir) self.assertEqual(expected_subdir, subdir, fail_msg) # sanity check for software name, moduleclass self.assertEqual(ec['name'], name) self.assertTrue(ec['moduleclass'] in build_option('valid_module_classes')) # instantiate easyblock with easyconfig file app_class = get_easyblock_class(easyblock, name=name) # check that automagic fallback to ConfigureMake isn't done (deprecated behaviour) fn = os.path.basename(spec) error_msg = "%s relies on automagic fallback to ConfigureMake, should use easyblock = 'ConfigureMake' instead" % fn self.assertTrue(easyblock or not app_class is ConfigureMake, error_msg) app = app_class(ec) # more sanity checks self.assertTrue(name, app.name) self.assertTrue(ec['version'], app.version) # make sure that $root is not used, since it is not compatible with module files in Lua syntax res = re.findall('.*\$root.*', ec.rawtxt, re.M) error_msg = "Found use of '$root', not compatible with modules in Lua syntax, use '%%(installdir)s' instead: %s" self.assertFalse(res, error_msg % res) # make sure old GitHub urls for EasyBuild that include 'hpcugent' are no longer used old_urls = [ 'github.com/hpcugent/easybuild', 'hpcugent.github.com/easybuild', 'hpcugent.github.io/easybuild', ] for old_url in old_urls: self.assertFalse(old_url in ec.rawtxt, "Old URL '%s' not found in %s" % (old_url, spec)) # make sure binutils is included as a build dep if toolchain is GCCcore if ec['toolchain']['name'] == 'GCCcore': # with 'Tarball' easyblock: only unpacking, no building; Eigen is also just a tarball requires_binutils = ec['easyblock'] not in ['Tarball'] and ec['name'] not in ['Eigen'] # let's also exclude the very special case where the system GCC is used as GCCcore, and only apply this # exception to the dependencies of binutils (since we should eventually build a new binutils with GCCcore) if ec['toolchain']['version'] == 'system': binutils_complete_dependencies = ['M4', 'Bison', 'flex', 'help2man', 'zlib', 'binutils'] requires_binutils &= bool(ec['name'] not in binutils_complete_dependencies) # if no sources/extensions/components are specified, it's just a bundle (nothing is being compiled) requires_binutils &= bool(ec['sources'] or ec['exts_list'] or ec.get('components')) if requires_binutils: dep_names = [d['name'] for d in ec['builddependencies']] self.assertTrue('binutils' in dep_names, "binutils is a build dep in %s: %s" % (spec, dep_names)) # make sure all patch files are available specdir = os.path.dirname(spec) specfn = os.path.basename(spec) for patch in ec['patches']: if isinstance(patch, (tuple, list)): patch = patch[0] # only check actual patch files, not other files being copied via the patch functionality if patch.endswith('.patch'): patch_full = os.path.join(specdir, patch) msg = "Patch file %s is available for %s" % (patch_full, specfn) self.assertTrue(os.path.isfile(patch_full), msg) ext_patches = [] for ext in ec['exts_list']: if isinstance(ext, (tuple, list)) and len(ext) == 3: self.assertTrue(isinstance(ext[2], dict), "3rd element of extension spec is a dictionary") for ext_patch in ext[2].get('patches', []): if isinstance(ext_patch, (tuple, list)): ext_patch = ext_patch[0] # only check actual patch files, not other files being copied via the patch functionality if ext_patch.endswith('.patch'): ext_patch_full = os.path.join(specdir, ext_patch) msg = "Patch file %s is available for %s" % (ext_patch_full, specfn) self.assertTrue(os.path.isfile(ext_patch_full), msg) # check whether all extra_options defined for used easyblock are defined extra_opts = app.extra_options() for key in extra_opts: self.assertTrue(key in app.cfg) app.close_log() os.remove(app.logfile) # dump the easyconfig file handle, test_ecfile = tempfile.mkstemp() os.close(handle) ec.dump(test_ecfile) dumped_ec = EasyConfigParser(test_ecfile).get_config_dict() os.remove(test_ecfile) # inject dummy values for templates that are only known at a later stage dummy_template_values = { 'builddir': '/dummy/builddir', 'installdir': '/dummy/installdir', } ec.template_values.update(dummy_template_values) ec_dict = ec.parser.get_config_dict() orig_toolchain = ec_dict['toolchain'] for key in ec_dict: # skip parameters for which value is equal to default value orig_val = ec_dict[key] if key in DEFAULT_CONFIG and orig_val == DEFAULT_CONFIG[key][0]: continue if key in extra_opts and orig_val == extra_opts[key][0]: continue if key not in DEFAULT_CONFIG and key not in extra_opts: continue orig_val = resolve_template(ec_dict[key], ec.template_values) dumped_val = resolve_template(dumped_ec[key], ec.template_values) # take into account that dumped value for *dependencies may include hard-coded subtoolchains # if no easyconfig was found for the dependency with the 'parent' toolchain, # if may get resolved using a subtoolchain, which is then hardcoded in the dumped easyconfig if key in DEPENDENCY_PARAMETERS: # number of dependencies should remain the same self.assertEqual(len(orig_val), len(dumped_val)) for orig_dep, dumped_dep in zip(orig_val, dumped_val): # name/version should always match self.assertEqual(orig_dep[:2], dumped_dep[:2]) # 3rd value is versionsuffix; if len(dumped_dep) >= 3: # if no versionsuffix was specified in original dep spec, then dumped value should be empty string if len(orig_dep) >= 3: self.assertEqual(dumped_dep[2], orig_dep[2]) else: self.assertEqual(dumped_dep[2], '') # 4th value is toolchain spec if len(dumped_dep) >= 4: if len(orig_dep) >= 4: self.assertEqual(dumped_dep[3], orig_dep[3]) else: # if a subtoolchain is specifed (only) in the dumped easyconfig, # it should *not* be the same as the parent toolchain self.assertNotEqual(dumped_dep[3], (orig_toolchain['name'], orig_toolchain['version'])) else: self.assertEqual(orig_val, dumped_val) # cache the parsed easyconfig, to avoid that it is parsed again self.parsed_easyconfigs.append(ecs[0]) # test passed, so set back to True single_tests_ok = True and prev_single_tests_ok
def test_changed_files_pull_request(self): """Specific checks only done for the (easyconfig) files that were changed in a pull request.""" def get_eb_files_from_diff(diff_filter): cmd = "git diff --name-only --diff-filter=%s %s...HEAD" % (diff_filter, target_branch) out, ec = run_cmd(cmd, simple=False) return [os.path.basename(f) for f in out.strip().split('\n') if f.endswith('.eb')] # $TRAVIS_PULL_REQUEST should be a PR number, otherwise we're not running tests for a PR travis_pr_test = re.match('^[0-9]+$', os.environ.get('TRAVIS_PULL_REQUEST', '(none)')) # when testing a PR in GitHub Actions, $GITHUB_EVENT_NAME will be set to 'pull_request' github_pr_test = os.environ.get('GITHUB_EVENT_NAME') == 'pull_request' if travis_pr_test or github_pr_test: # target branch should be anything other than 'master'; # usually is 'develop', but could also be a release branch like '3.7.x' if travis_pr_test: target_branch = os.environ.get('TRAVIS_BRANCH', None) else: target_branch = os.environ.get('GITHUB_BASE_REF', None) if target_branch is None: self.assertTrue(False, "Failed to determine target branch for current pull request.") if target_branch != 'master': if not EasyConfigTest.parsed_easyconfigs: self.process_all_easyconfigs() # relocate to top-level directory of repository to run 'git diff' command top_dir = os.path.dirname(os.path.dirname(get_paths_for('easyconfigs')[0])) cwd = change_dir(top_dir) # get list of changed easyconfigs changed_ecs_filenames = get_eb_files_from_diff(diff_filter='M') added_ecs_filenames = get_eb_files_from_diff(diff_filter='A') if changed_ecs_filenames: print("\nList of changed easyconfig files in this PR: %s" % '\n'.join(changed_ecs_filenames)) if added_ecs_filenames: print("\nList of added easyconfig files in this PR: %s" % '\n'.join(added_ecs_filenames)) change_dir(cwd) # grab parsed easyconfigs for changed easyconfig files changed_ecs = [] for ec_fn in changed_ecs_filenames + added_ecs_filenames: match = None for ec in EasyConfigTest.parsed_easyconfigs: if os.path.basename(ec['spec']) == ec_fn: match = ec['ec'] break if match: changed_ecs.append(match) else: # if no easyconfig is found, it's possible some archived easyconfigs were touched in the PR... # so as a last resort, try to find the easyconfig file in __archive__ easyconfigs_path = get_paths_for("easyconfigs")[0] specs = glob.glob('%s/__archive__/*/*/%s' % (easyconfigs_path, ec_fn)) if len(specs) == 1: ec = process_easyconfig(specs[0])[0] changed_ecs.append(ec['ec']) else: error_msg = "Failed to find parsed easyconfig for %s" % ec_fn error_msg += " (and could not isolate it in easyconfigs archive either)" self.assertTrue(False, error_msg) # run checks on changed easyconfigs self.check_sha256_checksums(changed_ecs) self.check_python_packages(changed_ecs, added_ecs_filenames) self.check_sanity_check_paths(changed_ecs) self.check_https(changed_ecs)
def template_easyconfig_test(self, spec): """Tests for an individual easyconfig: parsing, instantiating easyblock, check patches, ...""" # set to False, so it's False in case of this test failing global single_tests_ok prev_single_tests_ok = single_tests_ok single_tests_ok = False # parse easyconfig ecs = process_easyconfig(spec) if len(ecs) == 1: ec = ecs[0]['ec'] else: self.assertTrue( False, "easyconfig %s does not contain blocks, yields only one parsed easyconfig" % spec) # check easyconfig file name expected_fn = '%s-%s.eb' % (ec['name'], det_full_ec_version(ec)) msg = "Filename '%s' of parsed easconfig matches expected filename '%s'" % ( spec, expected_fn) self.assertEqual(os.path.basename(spec), expected_fn, msg) # sanity check for software name name = fetch_parameter_from_easyconfig_file(spec, 'name') self.assertTrue(ec['name'], name) # try and fetch easyblock spec from easyconfig easyblock = fetch_parameter_from_easyconfig_file(spec, 'easyblock') # instantiate easyblock with easyconfig file app_class = get_easyblock_class(easyblock, name=name) # check that automagic fallback to ConfigureMake isn't done (deprecated behaviour) fn = os.path.basename(spec) error_msg = "%s relies on automagic fallback to ConfigureMake, should use easyblock = 'ConfigureMake' instead" % fn self.assertTrue(easyblock or not app_class is ConfigureMake, error_msg) app = app_class(ec) # more sanity checks self.assertTrue(name, app.name) self.assertTrue(ec['version'], app.version) # make sure all patch files are available specdir = os.path.dirname(spec) specfn = os.path.basename(spec) for patch in ec['patches']: if isinstance(patch, (tuple, list)): patch = patch[0] # only check actual patch files, not other files being copied via the patch functionality if patch.endswith('.patch'): patch_full = os.path.join(specdir, patch) msg = "Patch file %s is available for %s" % (patch_full, specfn) self.assertTrue(os.path.isfile(patch_full), msg) ext_patches = [] for ext in ec['exts_list']: if isinstance(ext, (tuple, list)) and len(ext) == 3: self.assertTrue(isinstance(ext[2], dict), "3rd element of extension spec is a dictionary") for ext_patch in ext[2].get('patches', []): if isinstance(ext_patch, (tuple, list)): ext_patch = ext_patch[0] # only check actual patch files, not other files being copied via the patch functionality if ext_patch.endswith('.patch'): ext_patch_full = os.path.join(specdir, ext_patch) msg = "Patch file %s is available for %s" % ( ext_patch_full, specfn) self.assertTrue(os.path.isfile(ext_patch_full), msg) app.close_log() os.remove(app.logfile) # cache the parsed easyconfig, to avoid that it is parsed again self.parsed_easyconfigs.append(ecs[0]) # test passed, so set back to True single_tests_ok = True and prev_single_tests_ok
def regtest(easyconfig_paths, modtool, build_specs=None): """ Run regression test, using easyconfigs available in given path :param easyconfig_paths: path of easyconfigs to run regtest on :param modtool: ModulesTool instance to use :param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...) """ cur_dir = os.getcwd() aggregate_regtest = build_option('aggregate_regtest') if aggregate_regtest is not None: output_file = os.path.join(aggregate_regtest, "%s-aggregate.xml" % os.path.basename(aggregate_regtest)) aggregate_xml_in_dirs(aggregate_regtest, output_file) _log.info("aggregated xml files inside %s, output written to: %s" % (aggregate_regtest, output_file)) sys.exit(0) # create base directory, which is used to place all log files and the test output as xml regtest_output_dir = build_option('regtest_output_dir') testoutput = build_option('testoutput') if regtest_output_dir is not None: output_dir = regtest_output_dir elif testoutput is not None: output_dir = os.path.abspath(testoutput) else: # default: current dir + easybuild-test-[timestamp] dirname = "easybuild-test-%s" % datetime.now().strftime("%Y%m%d%H%M%S") output_dir = os.path.join(cur_dir, dirname) mkdir(output_dir, parents=True) # find all easyconfigs ecfiles = [] if easyconfig_paths: for path in easyconfig_paths: ecfiles += find_easyconfigs(path, ignore_dirs=build_option('ignore_dirs')) else: raise EasyBuildError("No easyconfig paths specified.") test_results = [] # process all the found easyconfig files easyconfigs = [] for ecfile in ecfiles: try: easyconfigs.extend(process_easyconfig(ecfile, build_specs=build_specs)) except EasyBuildError as err: test_results.append((ecfile, 'parsing_easyconfigs', 'easyconfig file error: %s' % err, _log)) # skip easyconfigs for which a module is already available, unless forced if not build_option('force'): _log.debug("Skipping easyconfigs from %s that already have a module available..." % easyconfigs) easyconfigs = skip_available(easyconfigs, modtool) _log.debug("Retained easyconfigs after skipping: %s" % easyconfigs) if build_option('sequential'): return build_easyconfigs(easyconfigs, output_dir, test_results) else: resolved = resolve_dependencies(easyconfigs, modtool) cmd = "eb %(spec)s --regtest --sequential -ld --testoutput=%(output_dir)s" command = "unset TMPDIR && cd %s && %s; " % (cur_dir, cmd) # retry twice in case of failure, to avoid fluke errors command += "if [ $? -ne 0 ]; then %(cmd)s --force && %(cmd)s --force; fi" % {'cmd': cmd} build_easyconfigs_in_parallel(command, resolved, output_dir=output_dir) _log.info("Submitted regression test as jobs, results in %s" % output_dir) return True # success
def main(testing_data=(None, None, None)): """ Main function: @arg options: a tuple: (options, paths, logger, logfile, hn) as defined in parse_options This function will: - read easyconfig - build software """ # disallow running EasyBuild as root if os.getuid() == 0: sys.stderr.write("ERROR: You seem to be running EasyBuild with root privileges.\n" "That's not wise, so let's end this here.\n" "Exiting.\n") sys.exit(1) # steer behavior when testing main testing = testing_data[0] is not None args, logfile, do_build = testing_data # initialise options eb_go = eboptions.parse_options(args=args) options = eb_go.options orig_paths = eb_go.args # set temporary directory to use eb_tmpdir = set_tmpdir(options.tmpdir) # initialise logging for main if options.logtostdout: fancylogger.logToScreen(enable=True, stdout=True) else: if logfile is None: # mkstemp returns (fd,filename), fd is from os.open, not regular open! fd, logfile = tempfile.mkstemp(suffix='.log', prefix='easybuild-') os.close(fd) fancylogger.logToFile(logfile) print_msg('temporary log file in case of crash %s' % (logfile), log=None, silent=testing) global _log _log = fancylogger.getLogger(fname=False) # hello world! _log.info(this_is_easybuild()) # how was EB called? eb_command_line = eb_go.generate_cmd_line() + eb_go.args _log.info("Command line: %s" % (" ".join(eb_command_line))) _log.info("Using %s as temporary directory" % eb_tmpdir) if not options.robot is None: if options.robot: _log.info("Using robot path(s): %s" % options.robot) else: _log.error("No robot paths specified, and unable to determine easybuild-easyconfigs install path.") # do not pass options.robot, it's not a list instance (and it shouldn't be modified) robot_path = None if options.robot: robot_path = list(options.robot) # determine easybuild-easyconfigs package install path easyconfigs_paths = get_paths_for("easyconfigs", robot_path=robot_path) # keep track of paths for install easyconfigs, so we can obtain find specified easyconfigs easyconfigs_pkg_full_paths = easyconfigs_paths[:] if not easyconfigs_paths: _log.warning("Failed to determine install path for easybuild-easyconfigs package.") # specified robot paths are preferred over installed easyconfig files if robot_path: robot_path.extend(easyconfigs_paths) easyconfigs_paths = robot_path[:] _log.info("Extended list of robot paths with paths for installed easyconfigs: %s" % robot_path) # initialise the easybuild configuration config.init(options, eb_go.get_options_by_section('config')) # building a dependency graph implies force, so that all dependencies are retained # and also skips validation of easyconfigs (e.g. checking os dependencies) retain_all_deps = False if options.dep_graph: _log.info("Enabling force to generate dependency graph.") options.force = True retain_all_deps = True build_options = { 'aggregate_regtest': options.aggregate_regtest, 'check_osdeps': not options.ignore_osdeps, 'command_line': eb_command_line, 'debug': options.debug, 'dry_run': options.dry_run, 'easyblock': options.easyblock, 'experimental': options.experimental, 'force': options.force, 'ignore_dirs': options.ignore_dirs, 'modules_footer': options.modules_footer, 'only_blocks': options.only_blocks, 'recursive_mod_unload': options.recursive_module_unload, 'regtest_online': options.regtest_online, 'regtest_output_dir': options.regtest_output_dir, 'retain_all_deps': retain_all_deps, 'robot_path': robot_path, 'sequential': options.sequential, 'silent': testing, 'skip': options.skip, 'skip_test_cases': options.skip_test_cases, 'stop': options.stop, 'valid_module_classes': module_classes(), 'valid_stops': [x[0] for x in EasyBlock.get_steps()], 'validate': not options.force, } # search for easyconfigs if options.search or options.search_short: search_path = [os.getcwd()] if easyconfigs_paths: search_path = easyconfigs_paths query = options.search or options.search_short search_file(search_path, query, build_options=build_options, short=not options.search) # process software build specifications (if any), i.e. # software name/version, toolchain name/version, extra patches, ... (try_to_generate, build_specs) = process_software_build_specs(options) paths = [] if len(orig_paths) == 0: if 'name' in build_specs: paths = [obtain_path(build_specs, easyconfigs_paths, try_to_generate=try_to_generate, exit_on_error=not testing)] elif not any([options.aggregate_regtest, options.search, options.search_short, options.regtest]): print_error(("Please provide one or multiple easyconfig files, or use software build " "options to make EasyBuild search for easyconfigs"), log=_log, opt_parser=eb_go.parser, exit_on_error=not testing) else: # look for easyconfigs with relative paths in easybuild-easyconfigs package, # unless they were found at the given relative paths if easyconfigs_pkg_full_paths: # determine which easyconfigs files need to be found, if any ecs_to_find = [] for idx, orig_path in enumerate(orig_paths): if orig_path == os.path.basename(orig_path) and not os.path.exists(orig_path): ecs_to_find.append((idx, orig_path)) _log.debug("List of easyconfig files to find: %s" % ecs_to_find) # find missing easyconfigs by walking paths with installed easyconfig files for path in easyconfigs_pkg_full_paths: _log.debug("Looking for missing easyconfig files (%d left) in %s..." % (len(ecs_to_find), path)) for (subpath, dirnames, filenames) in os.walk(path, topdown=True): for idx, orig_path in ecs_to_find[:]: if orig_path in filenames: full_path = os.path.join(subpath, orig_path) _log.info("Found %s in %s: %s" % (orig_path, path, full_path)) orig_paths[idx] = full_path # if file was found, stop looking for it (first hit wins) ecs_to_find.remove((idx, orig_path)) # stop os.walk insanity as soon as we have all we need (os.walk loop) if len(ecs_to_find) == 0: break # ignore subdirs specified to be ignored by replacing items in dirnames list used by os.walk dirnames[:] = [d for d in dirnames if not d in options.ignore_dirs] # stop os.walk insanity as soon as we have all we need (paths loop) if len(ecs_to_find) == 0: break # indicate that specified paths do not contain generated easyconfig files paths = [(path, False) for path in orig_paths] _log.debug("Paths: %s" % paths) # run regtest if options.regtest or options.aggregate_regtest: _log.info("Running regression test") if paths: ec_paths = [path[0] for path in paths] else: # fallback: easybuild-easyconfigs install path ec_paths = easyconfigs_pkg_full_paths regtest_ok = regtest(ec_paths, build_options=build_options, build_specs=build_specs) if not regtest_ok: _log.info("Regression test failed (partially)!") sys.exit(31) # exit -> 3x1t -> 31 # read easyconfig files easyconfigs = [] for (path, generated) in paths: path = os.path.abspath(path) if not os.path.exists(path): print_error("Can't find path %s" % path) try: files = find_easyconfigs(path, ignore_dirs=options.ignore_dirs) for f in files: if not generated and try_to_generate and build_specs: ec_file = tweak(f, None, build_specs) else: ec_file = f ecs = process_easyconfig(ec_file, build_options=build_options, build_specs=build_specs) easyconfigs.extend(ecs) except IOError, err: _log.error("Processing easyconfigs in path %s failed: %s" % (path, err))
def template_easyconfig_test(self, spec): """Tests for an individual easyconfig: parsing, instantiating easyblock, check patches, ...""" # set to False, so it's False in case of this test failing global single_tests_ok prev_single_tests_ok = single_tests_ok single_tests_ok = False # parse easyconfig ecs = process_easyconfig(spec) if len(ecs) == 1: ec = ecs[0]['ec'] else: self.assertTrue( False, "easyconfig %s does not contain blocks, yields only one parsed easyconfig" % spec) # check easyconfig file name expected_fn = '%s-%s.eb' % (ec['name'], det_full_ec_version(ec)) msg = "Filename '%s' of parsed easyconfig matches expected filename '%s'" % ( spec, expected_fn) self.assertEqual(os.path.basename(spec), expected_fn, msg) name, easyblock = fetch_parameters_from_easyconfig(ec.rawtxt, ['name', 'easyblock']) # make sure easyconfig file is in expected location expected_subdir = os.path.join('easybuild', 'easyconfigs', name.lower()[0], name) subdir = os.path.join(*spec.split(os.path.sep)[-5:-1]) fail_msg = "Easyconfig file %s not in expected subdirectory %s" % ( spec, expected_subdir) self.assertEqual(expected_subdir, subdir, fail_msg) # sanity check for software name self.assertTrue(ec['name'], name) # instantiate easyblock with easyconfig file app_class = get_easyblock_class(easyblock, name=name) # check that automagic fallback to ConfigureMake isn't done (deprecated behaviour) fn = os.path.basename(spec) error_msg = "%s relies on automagic fallback to ConfigureMake, should use easyblock = 'ConfigureMake' instead" % fn self.assertTrue(easyblock or not app_class is ConfigureMake, error_msg) app = app_class(ec) # more sanity checks self.assertTrue(name, app.name) self.assertTrue(ec['version'], app.version) # make sure all patch files are available specdir = os.path.dirname(spec) specfn = os.path.basename(spec) for patch in ec['patches']: if isinstance(patch, (tuple, list)): patch = patch[0] # only check actual patch files, not other files being copied via the patch functionality if patch.endswith('.patch'): patch_full = os.path.join(specdir, patch) msg = "Patch file %s is available for %s" % (patch_full, specfn) self.assertTrue(os.path.isfile(patch_full), msg) ext_patches = [] for ext in ec['exts_list']: if isinstance(ext, (tuple, list)) and len(ext) == 3: self.assertTrue(isinstance(ext[2], dict), "3rd element of extension spec is a dictionary") for ext_patch in ext[2].get('patches', []): if isinstance(ext_patch, (tuple, list)): ext_patch = ext_patch[0] # only check actual patch files, not other files being copied via the patch functionality if ext_patch.endswith('.patch'): ext_patch_full = os.path.join(specdir, ext_patch) msg = "Patch file %s is available for %s" % ( ext_patch_full, specfn) self.assertTrue(os.path.isfile(ext_patch_full), msg) # check whether all extra_options defined for used easyblock are defined for key in app.extra_options(): self.assertTrue(key in app.cfg) app.close_log() os.remove(app.logfile) # dump the easyconfig file handle, test_ecfile = tempfile.mkstemp() os.close(handle) ec.dump(test_ecfile) dumped_ec = EasyConfig(test_ecfile) os.remove(test_ecfile) # inject dummy values for templates that are only known at a later stage dummy_template_values = { 'builddir': '/dummy/builddir', 'installdir': '/dummy/installdir', } ec.template_values.update(dummy_template_values) dumped_ec.template_values.update(dummy_template_values) for key in sorted(ec._config): self.assertEqual(ec[key], dumped_ec[key]) # cache the parsed easyconfig, to avoid that it is parsed again self.parsed_easyconfigs.append(ecs[0]) # test passed, so set back to True single_tests_ok = True and prev_single_tests_ok
def test_build_easyconfigs_in_parallel_pbs_python(self): """Test build_easyconfigs_in_parallel(), using (mocked) pbs_python as backend for --job.""" # put mocked functions in place PbsPython__init__ = PbsPython.__init__ PbsPython_check_version = PbsPython._check_version PbsPython_complete = PbsPython.complete PbsPython_connect_to_server = PbsPython.connect_to_server PbsPython_ppn = PbsPython.ppn pbs_python_PbsJob = pbs_python.PbsJob PbsPython.__init__ = lambda self: PbsPython__init__(self, pbs_server='localhost') PbsPython._check_version = lambda _: True PbsPython.complete = mock PbsPython.connect_to_server = mock PbsPython.ppn = mock pbs_python.PbsJob = MockPbsJob build_options = { 'external_modules_metadata': {}, 'robot_path': os.path.join(os.path.dirname(__file__), 'easyconfigs'), 'valid_module_classes': config.module_classes(), 'validate': False, 'job_cores': 3, } init_config(args=['--job-backend=PbsPython'], build_options=build_options) ec_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'gzip-1.5-goolf-1.4.10.eb') easyconfigs = process_easyconfig(ec_file) ordered_ecs = resolve_dependencies(easyconfigs) jobs = build_easyconfigs_in_parallel("echo '%(spec)s'", ordered_ecs, prepare_first=False) self.assertEqual(len(jobs), 8) regex = re.compile("echo '.*/gzip-1.5-goolf-1.4.10.eb'") self.assertTrue(regex.search(jobs[-1].script), "Pattern '%s' found in: %s" % (regex.pattern, jobs[-1].script)) ec_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'gzip-1.4-GCC-4.6.3.eb') ordered_ecs = resolve_dependencies(process_easyconfig(ec_file), retain_all_deps=True) jobs = submit_jobs(ordered_ecs, '', testing=False, prepare_first=False) # make sure command is correct, and that --hidden is there when it needs to be for i, ec in enumerate(ordered_ecs): if ec['hidden']: regex = re.compile("eb %s.* --hidden" % ec['spec']) else: regex = re.compile("eb %s" % ec['spec']) self.assertTrue(regex.search(jobs[i].script), "Pattern '%s' found in: %s" % (regex.pattern, jobs[i].script)) for job in jobs: self.assertEqual(job.cores, build_options['job_cores']) # no deps for GCC/4.6.3 (toolchain) and ictce/4.1.13 (test easyconfig with 'fake' deps) self.assertEqual(len(jobs[0].deps), 0) self.assertEqual(len(jobs[1].deps), 0) # only dependency for toy/0.0-deps is ictce/4.1.13 (dep marked as external module is filtered out) self.assertTrue('toy-0.0-deps.eb' in jobs[2].script) self.assertEqual(len(jobs[2].deps), 1) self.assertTrue('ictce-4.1.13.eb' in jobs[2].deps[0].script) # dependencies for gzip/1.4-GCC-4.6.3: GCC/4.6.3 (toolchain) + toy/.0.0-deps self.assertTrue('gzip-1.4-GCC-4.6.3.eb' in jobs[3].script) self.assertEqual(len(jobs[3].deps), 2) regex = re.compile('toy-0.0-deps.eb\s* --hidden') self.assertTrue(regex.search(jobs[3].deps[0].script)) self.assertTrue('GCC-4.6.3.eb' in jobs[3].deps[1].script) # restore mocked stuff PbsPython.__init__ = PbsPython__init__ PbsPython._check_version = PbsPython_check_version PbsPython.complete = PbsPython_complete PbsPython.connect_to_server = PbsPython_connect_to_server PbsPython.ppn = PbsPython_ppn pbs_python.PbsJob = pbs_python_PbsJob