def test_fetch_files_from_pr_cache(self): """Test caching for fetch_files_from_pr.""" if self.skip_github_tests: print("Skipping test_fetch_files_from_pr_cache, no GitHub token available?") return init_config(build_options={ 'pr_target_account': gh.GITHUB_EB_MAIN, }) # clear cache first, to make sure we start with a clean slate gh.fetch_files_from_pr.clear_cache() self.assertFalse(gh.fetch_files_from_pr._cache) pr7159_filenames = [ 'DOLFIN-2018.1.0.post1-foss-2018a-Python-3.6.4.eb', 'OpenFOAM-5.0-20180108-foss-2018a.eb', 'OpenFOAM-5.0-20180108-intel-2018a.eb', 'OpenFOAM-6-foss-2018b.eb', 'OpenFOAM-6-intel-2018a.eb', 'OpenFOAM-v1806-foss-2018b.eb', 'PETSc-3.9.3-foss-2018a.eb', 'SCOTCH-6.0.6-foss-2018a.eb', 'SCOTCH-6.0.6-foss-2018b.eb', 'SCOTCH-6.0.6-intel-2018a.eb', 'Trilinos-12.12.1-foss-2018a-Python-3.6.4.eb' ] pr7159_files = gh.fetch_easyconfigs_from_pr(7159, path=self.test_prefix, github_user=GITHUB_TEST_ACCOUNT) self.assertEqual(sorted(pr7159_filenames), sorted(os.path.basename(f) for f in pr7159_files)) # check that cache has been populated for PR 7159 self.assertEqual(len(gh.fetch_files_from_pr._cache.keys()), 1) # github_account value is None (results in using default 'easybuilders') cache_key = (7159, None, 'easybuild-easyconfigs', self.test_prefix) self.assertTrue(cache_key in gh.fetch_files_from_pr._cache.keys()) cache_entry = gh.fetch_files_from_pr._cache[cache_key] self.assertEqual(sorted([os.path.basename(f) for f in cache_entry]), sorted(pr7159_filenames)) # same query should return result from cache entry res = gh.fetch_easyconfigs_from_pr(7159, path=self.test_prefix, github_user=GITHUB_TEST_ACCOUNT) self.assertEqual(res, pr7159_files) # inject entry in cache and check result of matching query pr_id = 12345 tmpdir = os.path.join(self.test_prefix, 'easyblocks-pr-12345') pr12345_files = [ os.path.join(tmpdir, 'foo.py'), os.path.join(tmpdir, 'bar.py'), ] for fp in pr12345_files: write_file(fp, '') # github_account value is None (results in using default 'easybuilders') cache_key = (pr_id, None, 'easybuild-easyblocks', tmpdir) gh.fetch_files_from_pr.update_cache({cache_key: pr12345_files}) res = gh.fetch_easyblocks_from_pr(12345, tmpdir) self.assertEqual(sorted(pr12345_files), sorted(res))
def test_fetch_easyconfigs_from_pr(self): """Test fetch_easyconfigs_from_pr function.""" if self.github_token is None: print "Skipping test_fetch_easyconfigs_from_pr, no GitHub token available?" return tmpdir = tempfile.mkdtemp() # PR for ictce/6.2.5, see https://github.com/hpcugent/easybuild-easyconfigs/pull/726/files all_ecs = [ 'gzip-1.6-ictce-6.2.5.eb', 'icc-2013_sp1.2.144.eb', 'ictce-6.2.5.eb', 'ifort-2013_sp1.2.144.eb', 'imkl-11.1.2.144.eb', 'impi-4.1.3.049.eb' ] try: ec_files = gh.fetch_easyconfigs_from_pr( 726, path=tmpdir, github_user=GITHUB_TEST_ACCOUNT) self.assertEqual(all_ecs, sorted([os.path.basename(f) for f in ec_files])) self.assertEqual(all_ecs, sorted(os.listdir(tmpdir))) # PR for EasyBuild v1.13.0 release (250+ commits, 218 files changed) err_msg = "PR #897 contains more than .* commits, can't obtain last commit" self.assertErrorRegex(EasyBuildError, err_msg, gh.fetch_easyconfigs_from_pr, 897, github_user=GITHUB_TEST_ACCOUNT) except URLError, err: print "Ignoring URLError '%s' in test_fetch_easyconfigs_from_pr" % err
def det_easyconfig_paths(orig_paths): """ Determine paths to easyconfig files. :param orig_paths: list of original easyconfig paths :return: list of paths to easyconfig files """ from_pr = build_option('from_pr') robot_path = build_option('robot_path') # list of specified easyconfig files ec_files = orig_paths[:] if from_pr is not None: pr_files = fetch_easyconfigs_from_pr(from_pr) if ec_files: # replace paths for specified easyconfigs that are touched in PR for i, ec_file in enumerate(ec_files): for pr_file in pr_files: if ec_file == os.path.basename(pr_file): ec_files[i] = pr_file else: # if no easyconfigs are specified, use all the ones touched in the PR ec_files = [path for path in pr_files if path.endswith('.eb')] if ec_files and robot_path: ignore_subdirs = build_option('ignore_dirs') if not build_option('consider_archived_easyconfigs'): ignore_subdirs.append(EASYCONFIGS_ARCHIVE_DIR) ec_files = locate_files(ec_files, robot_path, ignore_subdirs=ignore_subdirs) return ec_files
def det_easyconfig_paths(orig_paths): """ Determine paths to easyconfig files. @param orig_paths: list of original easyconfig paths @return: list of paths to easyconfig files """ from_pr = build_option('from_pr') robot_path = build_option('robot_path') # list of specified easyconfig files ec_files = orig_paths[:] if from_pr is not None: pr_files = fetch_easyconfigs_from_pr(from_pr) if ec_files: # replace paths for specified easyconfigs that are touched in PR for i, ec_file in enumerate(ec_files): for pr_file in pr_files: if ec_file == os.path.basename(pr_file): ec_files[i] = pr_file else: # if no easyconfigs are specified, use all the ones touched in the PR ec_files = [path for path in pr_files if path.endswith('.eb')] if ec_files and robot_path: # look for easyconfigs with relative paths in robot search path, # unless they were found at the given relative paths # determine which easyconfigs files need to be found, if any ecs_to_find = [] for idx, ec_file in enumerate(ec_files): if ec_file == os.path.basename(ec_file) and not os.path.exists(ec_file): ecs_to_find.append((idx, ec_file)) _log.debug("List of easyconfig files to find: %s" % ecs_to_find) # find missing easyconfigs by walking paths in robot search path for path in robot_path: _log.debug("Looking for missing easyconfig files (%d left) in %s..." % (len(ecs_to_find), path)) for (subpath, dirnames, filenames) in os.walk(path, topdown=True): for idx, orig_path in ecs_to_find[:]: if orig_path in filenames: full_path = os.path.join(subpath, orig_path) _log.info("Found %s in %s: %s" % (orig_path, path, full_path)) ec_files[idx] = full_path # if file was found, stop looking for it (first hit wins) ecs_to_find.remove((idx, orig_path)) # stop os.walk insanity as soon as we have all we need (os.walk loop) if not ecs_to_find: break # ignore subdirs specified to be ignored by replacing items in dirnames list used by os.walk dirnames[:] = [d for d in dirnames if d not in build_option('ignore_dirs')] # stop os.walk insanity as soon as we have all we need (outer loop) if not ecs_to_find: break return ec_files
def review_pr(pr, colored=True, branch='develop'): """ Print multi-diff overview between easyconfigs in specified PR and specified branch. :param pr: pull request number in easybuild-easyconfigs repo to review :param colored: boolean indicating whether a colored multi-diff should be generated :param branch: easybuild-easyconfigs branch to compare with """ tmpdir = tempfile.mkdtemp() download_repo_path = download_repo(branch=branch, path=tmpdir) repo_path = os.path.join(download_repo_path, 'easybuild', 'easyconfigs') pr_files = [ path for path in fetch_easyconfigs_from_pr(pr) if path.endswith('.eb') ] lines = [] ecs, _ = parse_easyconfigs([(fp, False) for fp in pr_files], validate=False) for ec in ecs: files = find_related_easyconfigs(repo_path, ec['ec']) _log.debug("File in PR#%s %s has these related easyconfigs: %s" % (pr, ec['spec'], files)) if files: lines.append(multidiff(ec['spec'], files, colored=colored)) else: lines.extend([ '', "(no related easyconfigs found for %s)\n" % os.path.basename(ec['spec']) ]) return '\n'.join(lines)
def test_fetch_easyconfigs_from_pr(self): """Test fetch_easyconfigs_from_pr function.""" if self.github_token is None: print "Skipping test_fetch_easyconfigs_from_pr, no GitHub token available?" return tmpdir = tempfile.mkdtemp() # PR for rename of ffmpeg to FFmpeg, see https://github.com/hpcugent/easybuild-easyconfigs/pull/2481/files all_ecs = [ 'FFmpeg-2.4-intel-2014.06.eb', 'FFmpeg-2.4-intel-2014b.eb', 'FFmpeg-2.8-intel-2015b.eb', 'OpenCV-2.4.9-intel-2014.06.eb', 'OpenCV-2.4.9-intel-2014b.eb', 'animation-2.4-intel-2015b-R-3.2.1.eb', ] try: ec_files = gh.fetch_easyconfigs_from_pr(2481, path=tmpdir, github_user=GITHUB_TEST_ACCOUNT) self.assertEqual(all_ecs, sorted([os.path.basename(f) for f in ec_files])) self.assertEqual(all_ecs, sorted([os.path.basename(f) for f in glob.glob(os.path.join(tmpdir, '*', '*'))])) # PR for EasyBuild v1.13.0 release (250+ commits, 218 files changed) err_msg = "PR #897 contains more than .* commits, can't obtain last commit" self.assertErrorRegex(EasyBuildError, err_msg, gh.fetch_easyconfigs_from_pr, 897, github_user=GITHUB_TEST_ACCOUNT) except URLError, err: print "Ignoring URLError '%s' in test_fetch_easyconfigs_from_pr" % err
def review_pr(paths=None, pr=None, colored=True, branch='develop'): """ Print multi-diff overview between specified easyconfigs or PR and specified branch. :param pr: pull request number in easybuild-easyconfigs repo to review :param paths: path tuples (path, generated) of easyconfigs to review :param colored: boolean indicating whether a colored multi-diff should be generated :param branch: easybuild-easyconfigs branch to compare with """ tmpdir = tempfile.mkdtemp() download_repo_path = download_repo(branch=branch, path=tmpdir) repo_path = os.path.join(download_repo_path, 'easybuild', 'easyconfigs') if pr: pr_files = [path for path in fetch_easyconfigs_from_pr(pr) if path.endswith('.eb')] elif paths: pr_files = paths else: raise EasyBuildError("No PR # or easyconfig path specified") lines = [] ecs, _ = parse_easyconfigs([(fp, False) for fp in pr_files], validate=False) for ec in ecs: files = find_related_easyconfigs(repo_path, ec['ec']) if pr: pr_msg = "PR#%s" % pr else: pr_msg = "new PR" _log.debug("File in %s %s has these related easyconfigs: %s" % (pr_msg, ec['spec'], files)) if files: lines.append(multidiff(ec['spec'], files, colored=colored)) else: lines.extend(['', "(no related easyconfigs found for %s)\n" % os.path.basename(ec['spec'])]) return '\n'.join(lines)
def test_fetch_easyconfigs_from_pr(self): """Test fetch_easyconfigs_from_pr function.""" if self.github_token is None: print "Skipping test_fetch_easyconfigs_from_pr, no GitHub token available?" return tmpdir = tempfile.mkdtemp() # PR for rename of ffmpeg to FFmpeg, see https://github.com/easybuilders/easybuild-easyconfigs/pull/2481/files all_ecs = [ 'FFmpeg-2.4-intel-2014.06.eb', 'FFmpeg-2.4-intel-2014b.eb', 'FFmpeg-2.8-intel-2015b.eb', 'OpenCV-2.4.9-intel-2014.06.eb', 'OpenCV-2.4.9-intel-2014b.eb', 'animation-2.4-intel-2015b-R-3.2.1.eb', ] try: ec_files = gh.fetch_easyconfigs_from_pr(2481, path=tmpdir, github_user=GITHUB_TEST_ACCOUNT) self.assertEqual(all_ecs, sorted([os.path.basename(f) for f in ec_files])) # PR for EasyBuild v1.13.0 release (250+ commits, 218 files changed) err_msg = "PR #897 contains more than .* commits, can't obtain last commit" self.assertErrorRegex(EasyBuildError, err_msg, gh.fetch_easyconfigs_from_pr, 897, github_user=GITHUB_TEST_ACCOUNT) except URLError, err: print "Ignoring URLError '%s' in test_fetch_easyconfigs_from_pr" % err
def test_fetch_easyconfigs_from_pr(self): """Test fetch_easyconfigs_from_pr function.""" if self.skip_github_tests: print( "Skipping test_fetch_easyconfigs_from_pr, no GitHub token available?" ) return init_config(build_options={ 'pr_target_account': gh.GITHUB_EB_MAIN, }) # PR for rename of arrow to Arrow, # see https://github.com/easybuilders/easybuild-easyconfigs/pull/8007/files all_ecs_pr8007 = [ 'Arrow-0.7.1-intel-2017b-Python-3.6.3.eb', 'bat-0.3.3-fix-pyspark.patch', 'bat-0.3.3-intel-2017b-Python-3.6.3.eb', ] # PR where also files are patched in test/ # see https://github.com/easybuilders/easybuild-easyconfigs/pull/6587/files all_ecs_pr6587 = [ 'WIEN2k-18.1-foss-2018a.eb', 'WIEN2k-18.1-gimkl-2017a.eb', 'WIEN2k-18.1-intel-2018a.eb', 'libxc-4.2.3-foss-2018a.eb', 'libxc-4.2.3-gimkl-2017a.eb', 'libxc-4.2.3-intel-2018a.eb', ] # PR where files are renamed # see https://github.com/easybuilders/easybuild-easyconfigs/pull/7159/files all_ecs_pr7159 = [ 'DOLFIN-2018.1.0.post1-foss-2018a-Python-3.6.4.eb', 'OpenFOAM-5.0-20180108-foss-2018a.eb', 'OpenFOAM-5.0-20180108-intel-2018a.eb', 'OpenFOAM-6-foss-2018b.eb', 'OpenFOAM-6-intel-2018a.eb', 'OpenFOAM-v1806-foss-2018b.eb', 'PETSc-3.9.3-foss-2018a.eb', 'SCOTCH-6.0.6-foss-2018a.eb', 'SCOTCH-6.0.6-foss-2018b.eb', 'SCOTCH-6.0.6-intel-2018a.eb', 'Trilinos-12.12.1-foss-2018a-Python-3.6.4.eb' ] for pr, all_ecs in [(8007, all_ecs_pr8007), (6587, all_ecs_pr6587), (7159, all_ecs_pr7159)]: try: tmpdir = os.path.join(self.test_prefix, 'pr%s' % pr) ec_files = gh.fetch_easyconfigs_from_pr( pr, path=tmpdir, github_user=GITHUB_TEST_ACCOUNT) self.assertEqual( sorted(all_ecs), sorted([os.path.basename(f) for f in ec_files])) except URLError as err: print( "Ignoring URLError '%s' in test_fetch_easyconfigs_from_pr" % err)
def test_fetch_easyconfigs_from_pr(self): """Test fetch_easyconfigs_from_pr function.""" if self.skip_github_tests: print "Skipping test_fetch_easyconfigs_from_pr, no GitHub token available?" return init_config(build_options={ 'pr_target_account': gh.GITHUB_EB_MAIN, }) # PR for rename of ffmpeg to FFmpeg, # see https://github.com/easybuilders/easybuild-easyconfigs/pull/2481/files all_ecs_pr2481 = [ 'FFmpeg-2.4-intel-2014.06.eb', 'FFmpeg-2.4-intel-2014b.eb', 'FFmpeg-2.8-intel-2015b.eb', 'OpenCV-2.4.9-intel-2014.06.eb', 'OpenCV-2.4.9-intel-2014b.eb', 'animation-2.4-intel-2015b-R-3.2.1.eb', ] # PR where also files are patched in test/ # see https://github.com/easybuilders/easybuild-easyconfigs/pull/6587/files all_ecs_pr6587 = [ 'WIEN2k-18.1-foss-2018a.eb', 'WIEN2k-18.1-gimkl-2017a.eb', 'WIEN2k-18.1-intel-2018a.eb', 'libxc-4.2.3-foss-2018a.eb', 'libxc-4.2.3-gimkl-2017a.eb', 'libxc-4.2.3-intel-2018a.eb', ] # PR where files are renamed # see https://github.com/easybuilders/easybuild-easyconfigs/pull/7159/files all_ecs_pr7159 = [ 'DOLFIN-2018.1.0.post1-foss-2018a-Python-3.6.4.eb', 'OpenFOAM-5.0-20180108-foss-2018a.eb', 'OpenFOAM-5.0-20180108-intel-2018a.eb', 'OpenFOAM-6-foss-2018b.eb', 'OpenFOAM-6-intel-2018a.eb', 'OpenFOAM-v1806-foss-2018b.eb', 'PETSc-3.9.3-foss-2018a.eb', 'SCOTCH-6.0.6-foss-2018a.eb', 'SCOTCH-6.0.6-foss-2018b.eb', 'SCOTCH-6.0.6-intel-2018a.eb', 'Trilinos-12.12.1-foss-2018a-Python-3.6.4.eb' ] for pr, all_ecs in [(2481, all_ecs_pr2481), (6587, all_ecs_pr6587), (7159, all_ecs_pr7159)]: try: tmpdir = os.path.join(self.test_prefix, 'pr%s' % pr) ec_files = gh.fetch_easyconfigs_from_pr(pr, path=tmpdir, github_user=GITHUB_TEST_ACCOUNT) self.assertEqual(sorted(all_ecs), sorted([os.path.basename(f) for f in ec_files])) except URLError, err: print "Ignoring URLError '%s' in test_fetch_easyconfigs_from_pr" % err
def det_easyconfig_paths(orig_paths, from_pr=None, easyconfigs_pkg_paths=None): """ Determine paths to easyconfig files. @param orig_paths: list of original easyconfig paths @param from_pr: pull request number to fetch easyconfigs from @param easyconfigs_pkg_paths: paths to installed easyconfigs package """ if easyconfigs_pkg_paths is None: easyconfigs_pkg_paths = [] ec_files = orig_paths[:] if not ec_files and from_pr: pr_files = fetch_easyconfigs_from_pr(from_pr) ec_files = [path for path in pr_files if path.endswith('.eb')] elif ec_files and easyconfigs_pkg_paths: # look for easyconfigs with relative paths in easybuild-easyconfigs package, # unless they were found at the given relative paths # determine which easyconfigs files need to be found, if any ecs_to_find = [] for idx, ec_file in enumerate(ec_files): if ec_file == os.path.basename(ec_file) and not os.path.exists(ec_file): ecs_to_find.append((idx, ec_file)) _log.debug("List of easyconfig files to find: %s" % ecs_to_find) # find missing easyconfigs by walking paths with installed easyconfig files for path in easyconfigs_pkg_paths: _log.debug("Looking for missing easyconfig files (%d left) in %s..." % (len(ecs_to_find), path)) for (subpath, dirnames, filenames) in os.walk(path, topdown=True): for idx, orig_path in ecs_to_find[:]: if orig_path in filenames: full_path = os.path.join(subpath, orig_path) _log.info("Found %s in %s: %s" % (orig_path, path, full_path)) ec_files[idx] = full_path # if file was found, stop looking for it (first hit wins) ecs_to_find.remove((idx, orig_path)) # stop os.walk insanity as soon as we have all we need (os.walk loop) if not ecs_to_find: break # ignore subdirs specified to be ignored by replacing items in dirnames list used by os.walk dirnames[:] = [d for d in dirnames if not d in build_option('ignore_dirs')] # stop os.walk insanity as soon as we have all we need (outer loop) if not ecs_to_find: break # indicate that specified paths do not contain generated easyconfig files return [(ec_file, False) for ec_file in ec_files]
def det_easyconfig_paths(orig_paths): """ Determine paths to easyconfig files. :param orig_paths: list of original easyconfig paths :return: list of paths to easyconfig files """ try: from_prs = [int(x) for x in build_option('from_pr')] except ValueError: raise EasyBuildError( "Argument to --from-pr must be a comma separated list of PR #s.") robot_path = build_option('robot_path') # list of specified easyconfig files ec_files = orig_paths[:] if from_prs: pr_files = [] for pr in from_prs: # path to where easyconfig files should be downloaded is determined via 'pr_paths' build option, # which corresponds to the list of PR paths returned by alt_easyconfig_paths pr_files.extend(fetch_easyconfigs_from_pr(pr)) if ec_files: # replace paths for specified easyconfigs that are touched in PR for i, ec_file in enumerate(ec_files): for pr_file in pr_files: if ec_file == os.path.basename(pr_file): ec_files[i] = pr_file else: # if no easyconfigs are specified, use all the ones touched in the PR ec_files = [path for path in pr_files if path.endswith('.eb')] filter_ecs = build_option('filter_ecs') if filter_ecs: ec_files = [ ec for ec in ec_files if not any( fnmatch.fnmatch(ec, filter_spec) for filter_spec in filter_ecs) ] if ec_files and robot_path: ignore_subdirs = build_option('ignore_dirs') if not build_option('consider_archived_easyconfigs'): ignore_subdirs.append(EASYCONFIGS_ARCHIVE_DIR) ec_files = locate_files(ec_files, robot_path, ignore_subdirs=ignore_subdirs) return ec_files
def test_fetch_easyconfigs_from_pr(self): """Test fetch_easyconfigs_from_pr function.""" tmpdir = tempfile.mkdtemp() # PR for ictce/6.2.5, see https://github.com/hpcugent/easybuild-easyconfigs/pull/726/files all_ecs = ['gzip-1.6-ictce-6.2.5.eb', 'icc-2013_sp1.2.144.eb', 'ictce-6.2.5.eb', 'ifort-2013_sp1.2.144.eb', 'imkl-11.1.2.144.eb', 'impi-4.1.3.049.eb'] ec_files = fetch_easyconfigs_from_pr(726, path=tmpdir, github_user=GITHUB_TEST_ACCOUNT) self.assertEqual(all_ecs, sorted([os.path.basename(f) for f in ec_files])) self.assertEqual(all_ecs, sorted(os.listdir(tmpdir))) shutil.rmtree(tmpdir) # PR for EasyBuild v1.13.0 release (250+ commits, 218 files changed) err_msg = "PR #897 contains more than .* commits, can't obtain last commit" self.assertErrorRegex(EasyBuildError, err_msg, fetch_easyconfigs_from_pr, 897, github_user=GITHUB_TEST_ACCOUNT)
def test_fetch_easyconfigs_from_pr(self): """Test fetch_easyconfigs_from_pr function.""" if self.github_token is None: print "Skipping test_fetch_easyconfigs_from_pr, no GitHub token available?" return tmpdir = tempfile.mkdtemp() # PR for ictce/6.2.5, see https://github.com/hpcugent/easybuild-easyconfigs/pull/726/files all_ecs = ['gzip-1.6-ictce-6.2.5.eb', 'icc-2013_sp1.2.144.eb', 'ictce-6.2.5.eb', 'ifort-2013_sp1.2.144.eb', 'imkl-11.1.2.144.eb', 'impi-4.1.3.049.eb'] try: ec_files = fetch_easyconfigs_from_pr(726, path=tmpdir, github_user=GITHUB_TEST_ACCOUNT) self.assertEqual(all_ecs, sorted([os.path.basename(f) for f in ec_files])) self.assertEqual(all_ecs, sorted(os.listdir(tmpdir))) # PR for EasyBuild v1.13.0 release (250+ commits, 218 files changed) err_msg = "PR #897 contains more than .* commits, can't obtain last commit" self.assertErrorRegex(EasyBuildError, err_msg, fetch_easyconfigs_from_pr, 897, github_user=GITHUB_TEST_ACCOUNT) except URLError, err: print "Ignoring URLError '%s' in test_fetch_easyconfigs_from_pr" % err
def test_fetch_easyconfigs_from_pr(self): """Test fetch_easyconfigs_from_pr function.""" if self.github_token is None: print "Skipping test_fetch_easyconfigs_from_pr, no GitHub token available?" return # PR for rename of ffmpeg to FFmpeg, # see https://github.com/easybuilders/easybuild-easyconfigs/pull/2481/files all_ecs_pr2481 = [ 'FFmpeg-2.4-intel-2014.06.eb', 'FFmpeg-2.4-intel-2014b.eb', 'FFmpeg-2.8-intel-2015b.eb', 'OpenCV-2.4.9-intel-2014.06.eb', 'OpenCV-2.4.9-intel-2014b.eb', 'animation-2.4-intel-2015b-R-3.2.1.eb', ] # PR where also files are patched in test/ # see https://github.com/easybuilders/easybuild-easyconfigs/pull/6587/files all_ecs_pr6587 = [ 'WIEN2k-18.1-foss-2018a.eb', 'WIEN2k-18.1-gimkl-2017a.eb', 'WIEN2k-18.1-intel-2018a.eb', 'libxc-4.2.3-foss-2018a.eb', 'libxc-4.2.3-gimkl-2017a.eb', 'libxc-4.2.3-intel-2018a.eb', ] for pr, all_ecs in [(2481, all_ecs_pr2481), (6587, all_ecs_pr6587)]: try: tmpdir = os.path.join(self.test_prefix, 'pr%s' % pr) ec_files = gh.fetch_easyconfigs_from_pr( pr, path=tmpdir, github_user=GITHUB_TEST_ACCOUNT) self.assertEqual( sorted(all_ecs), sorted([os.path.basename(f) for f in ec_files])) except URLError, err: print "Ignoring URLError '%s' in test_fetch_easyconfigs_from_pr" % err
def test_fetch_easyconfigs_from_pr(self): """Test fetch_easyconfigs_from_pr function.""" tmpdir = tempfile.mkdtemp() # PR for ictce/6.2.5, see https://github.com/hpcugent/easybuild-easyconfigs/pull/726/files all_ecs = [ 'gzip-1.6-ictce-6.2.5.eb', 'icc-2013_sp1.2.144.eb', 'ictce-6.2.5.eb', 'ifort-2013_sp1.2.144.eb', 'imkl-11.1.2.144.eb', 'impi-4.1.3.049.eb' ] ec_files = fetch_easyconfigs_from_pr(726, path=tmpdir, github_user=GITHUB_TEST_ACCOUNT) self.assertEqual(all_ecs, sorted([os.path.basename(f) for f in ec_files])) self.assertEqual(all_ecs, sorted(os.listdir(tmpdir))) shutil.rmtree(tmpdir) # PR for EasyBuild v1.13.0 release (250+ commits, 218 files changed) err_msg = "PR #897 contains more than .* commits, can't obtain last commit" self.assertErrorRegex(EasyBuildError, err_msg, fetch_easyconfigs_from_pr, 897, github_user=GITHUB_TEST_ACCOUNT)
def review_pr(pr, colored=True, branch="develop"): """ Print multi-diff overview between easyconfigs in specified PR and specified branch. @param pr: pull request number in easybuild-easyconfigs repo to review @param colored: boolean indicating whether a colored multi-diff should be generated @param branch: easybuild-easyconfigs branch to compare with """ tmpdir = tempfile.mkdtemp() download_repo_path = download_repo(branch=branch, path=tmpdir) repo_path = os.path.join(download_repo_path, "easybuild", "easyconfigs") pr_files = [path for path in fetch_easyconfigs_from_pr(pr) if path.endswith(".eb")] lines = [] ecs, _ = parse_easyconfigs([(fp, False) for fp in pr_files], validate=False) for ec in ecs: files = find_related_easyconfigs(repo_path, ec["ec"]) _log.debug("File in PR#%s %s has these related easyconfigs: %s" % (pr, ec["spec"], files)) if files: lines.append(multidiff(ec["spec"], files, colored=colored)) else: lines.extend(["", "(no related easyconfigs found for %s)\n" % os.path.basename(ec["spec"])]) return "\n".join(lines)
def main(testing_data=(None, None, None)): """ Main function: @arg options: a tuple: (options, paths, logger, logfile, hn) as defined in parse_options This function will: - read easyconfig - build software """ # purposely session state very early, to avoid modules loaded by EasyBuild meddling in init_session_state = session_state() # disallow running EasyBuild as root if os.getuid() == 0: sys.stderr.write("ERROR: You seem to be running EasyBuild with root privileges.\n" "That's not wise, so let's end this here.\n" "Exiting.\n") sys.exit(1) # steer behavior when testing main testing = testing_data[0] is not None args, logfile, do_build = testing_data # initialise options eb_go = eboptions.parse_options(args=args) options = eb_go.options orig_paths = eb_go.args eb_config = eb_go.generate_cmd_line(add_default=True) init_session_state.update({'easybuild_configuration': eb_config}) # set umask (as early as possible) if options.umask is not None: new_umask = int(options.umask, 8) old_umask = os.umask(new_umask) # set temporary directory to use eb_tmpdir = set_tmpdir(options.tmpdir) # initialise logging for main if options.logtostdout: fancylogger.logToScreen(enable=True, stdout=True) else: if logfile is None: # mkstemp returns (fd,filename), fd is from os.open, not regular open! fd, logfile = tempfile.mkstemp(suffix='.log', prefix='easybuild-') os.close(fd) fancylogger.logToFile(logfile) print_msg('temporary log file in case of crash %s' % (logfile), log=None, silent=testing) global _log _log = fancylogger.getLogger(fname=False) if options.umask is not None: _log.info("umask set to '%s' (used to be '%s')" % (oct(new_umask), oct(old_umask))) # hello world! _log.info(this_is_easybuild()) # how was EB called? eb_command_line = eb_go.generate_cmd_line() + eb_go.args _log.info("Command line: %s" % (" ".join(eb_command_line))) _log.info("Using %s as temporary directory" % eb_tmpdir) if not options.robot is None: if options.robot: _log.info("Using robot path(s): %s" % options.robot) else: _log.error("No robot paths specified, and unable to determine easybuild-easyconfigs install path.") # do not pass options.robot, it's not a list instance (and it shouldn't be modified) robot_path = [] if options.robot: robot_path = list(options.robot) # determine easybuild-easyconfigs package install path easyconfigs_paths = get_paths_for("easyconfigs", robot_path=robot_path) # keep track of paths for install easyconfigs, so we can obtain find specified easyconfigs easyconfigs_pkg_full_paths = easyconfigs_paths[:] if not easyconfigs_paths: _log.warning("Failed to determine install path for easybuild-easyconfigs package.") # process software build specifications (if any), i.e. # software name/version, toolchain name/version, extra patches, ... (try_to_generate, build_specs) = process_software_build_specs(options) # specified robot paths are preferred over installed easyconfig files # --try-X and --dep-graph both require --robot, so enable it with path of installed easyconfigs if robot_path or try_to_generate or options.dep_graph: robot_path.extend(easyconfigs_paths) easyconfigs_paths = robot_path[:] _log.info("Extended list of robot paths with paths for installed easyconfigs: %s" % robot_path) # prepend robot path with location where tweaked easyconfigs will be placed tweaked_ecs_path = None if try_to_generate and build_specs: tweaked_ecs_path = os.path.join(eb_tmpdir, 'tweaked_easyconfigs') robot_path.insert(0, tweaked_ecs_path) # initialise the easybuild configuration config.init(options, eb_go.get_options_by_section('config')) # building a dependency graph implies force, so that all dependencies are retained # and also skips validation of easyconfigs (e.g. checking os dependencies) retain_all_deps = False if options.dep_graph: _log.info("Enabling force to generate dependency graph.") options.force = True retain_all_deps = True if options.dep_graph or options.dry_run or options.dry_run_short: options.ignore_osdeps = True pr_path = None if options.from_pr: # extend robot search path with location where files touch in PR will be downloaded to pr_path = os.path.join(eb_tmpdir, "files_pr%s" % options.from_pr) robot_path.insert(0, pr_path) _log.info("Prepended list of robot search paths with %s: %s" % (pr_path, robot_path)) config.init_build_options({ 'aggregate_regtest': options.aggregate_regtest, 'allow_modules_tool_mismatch': options.allow_modules_tool_mismatch, 'check_osdeps': not options.ignore_osdeps, 'filter_deps': options.filter_deps, 'cleanup_builddir': options.cleanup_builddir, 'command_line': eb_command_line, 'debug': options.debug, 'dry_run': options.dry_run or options.dry_run_short, 'easyblock': options.easyblock, 'experimental': options.experimental, 'force': options.force, 'github_user': options.github_user, 'group': options.group, 'hidden': options.hidden, 'ignore_dirs': options.ignore_dirs, 'modules_footer': options.modules_footer, 'only_blocks': options.only_blocks, 'optarch': options.optarch, 'recursive_mod_unload': options.recursive_module_unload, 'regtest_output_dir': options.regtest_output_dir, 'retain_all_deps': retain_all_deps, 'robot_path': robot_path, 'sequential': options.sequential, 'silent': testing, 'set_gid_bit': options.set_gid_bit, 'skip': options.skip, 'skip_test_cases': options.skip_test_cases, 'sticky_bit': options.sticky_bit, 'stop': options.stop, 'suffix_modules_path': options.suffix_modules_path, 'test_report_env_filter': options.test_report_env_filter, 'umask': options.umask, 'valid_module_classes': module_classes(), 'valid_stops': [x[0] for x in EasyBlock.get_steps()], 'validate': not options.force, }) # obtain list of loaded modules, build options must be initialized first modlist = session_module_list(testing=testing) init_session_state.update({'module_list': modlist}) _log.debug("Initial session state: %s" % init_session_state) # search for easyconfigs if options.search or options.search_short: search_path = [os.getcwd()] if easyconfigs_paths: search_path = easyconfigs_paths query = options.search or options.search_short ignore_dirs = config.build_option('ignore_dirs') silent = config.build_option('silent') search_file(search_path, query, short=not options.search, ignore_dirs=ignore_dirs, silent=silent) paths = [] if len(orig_paths) == 0: if options.from_pr: pr_files = fetch_easyconfigs_from_pr(options.from_pr, path=pr_path, github_user=options.github_user) paths = [(path, False) for path in pr_files if path.endswith('.eb')] elif 'name' in build_specs: paths = [obtain_path(build_specs, easyconfigs_paths, try_to_generate=try_to_generate, exit_on_error=not testing)] elif not any([options.aggregate_regtest, options.search, options.search_short, options.regtest]): print_error(("Please provide one or multiple easyconfig files, or use software build " "options to make EasyBuild search for easyconfigs"), log=_log, opt_parser=eb_go.parser, exit_on_error=not testing) else: # look for easyconfigs with relative paths in easybuild-easyconfigs package, # unless they were found at the given relative paths if easyconfigs_pkg_full_paths: # determine which easyconfigs files need to be found, if any ecs_to_find = [] for idx, orig_path in enumerate(orig_paths): if orig_path == os.path.basename(orig_path) and not os.path.exists(orig_path): ecs_to_find.append((idx, orig_path)) _log.debug("List of easyconfig files to find: %s" % ecs_to_find) # find missing easyconfigs by walking paths with installed easyconfig files for path in easyconfigs_pkg_full_paths: _log.debug("Looking for missing easyconfig files (%d left) in %s..." % (len(ecs_to_find), path)) for (subpath, dirnames, filenames) in os.walk(path, topdown=True): for idx, orig_path in ecs_to_find[:]: if orig_path in filenames: full_path = os.path.join(subpath, orig_path) _log.info("Found %s in %s: %s" % (orig_path, path, full_path)) orig_paths[idx] = full_path # if file was found, stop looking for it (first hit wins) ecs_to_find.remove((idx, orig_path)) # stop os.walk insanity as soon as we have all we need (os.walk loop) if len(ecs_to_find) == 0: break # ignore subdirs specified to be ignored by replacing items in dirnames list used by os.walk dirnames[:] = [d for d in dirnames if not d in options.ignore_dirs] # stop os.walk insanity as soon as we have all we need (paths loop) if len(ecs_to_find) == 0: break # indicate that specified paths do not contain generated easyconfig files paths = [(path, False) for path in orig_paths] _log.debug("Paths: %s" % paths) # run regtest if options.regtest or options.aggregate_regtest: _log.info("Running regression test") if paths: ec_paths = [path[0] for path in paths] else: # fallback: easybuild-easyconfigs install path ec_paths = easyconfigs_pkg_full_paths regtest_ok = regtest(ec_paths) if not regtest_ok: _log.info("Regression test failed (partially)!") sys.exit(31) # exit -> 3x1t -> 31 # read easyconfig files easyconfigs = [] generated_ecs = False for (path, generated) in paths: path = os.path.abspath(path) # keep track of whether any files were generated generated_ecs |= generated if not os.path.exists(path): print_error("Can't find path %s" % path) try: ec_files = find_easyconfigs(path, ignore_dirs=options.ignore_dirs) for ec_file in ec_files: # only pass build specs when not generating easyconfig files if try_to_generate: ecs = process_easyconfig(ec_file) else: ecs = process_easyconfig(ec_file, build_specs=build_specs) easyconfigs.extend(ecs) except IOError, err: _log.error("Processing easyconfigs in path %s failed: %s" % (path, err))
def review_pr(paths=None, pr=None, colored=True, branch='develop', testing=False): """ Print multi-diff overview between specified easyconfigs or PR and specified branch. :param pr: pull request number in easybuild-easyconfigs repo to review :param paths: path tuples (path, generated) of easyconfigs to review :param colored: boolean indicating whether a colored multi-diff should be generated :param branch: easybuild-easyconfigs branch to compare with :param testing: whether to ignore PR labels (used in test_review_pr) """ pr_target_repo = build_option('pr_target_repo') or GITHUB_EASYCONFIGS_REPO if pr_target_repo != GITHUB_EASYCONFIGS_REPO: raise EasyBuildError( "Reviewing PRs for repositories other than easyconfigs hasn't been implemented yet" ) tmpdir = tempfile.mkdtemp() download_repo_path = download_repo(branch=branch, path=tmpdir) repo_path = os.path.join(download_repo_path, 'easybuild', 'easyconfigs') if pr: pr_files = [ path for path in fetch_easyconfigs_from_pr(pr) if path.endswith('.eb') ] elif paths: pr_files = paths else: raise EasyBuildError("No PR # or easyconfig path specified") lines = [] ecs, _ = parse_easyconfigs([(fp, False) for fp in pr_files], validate=False) for ec in ecs: files = find_related_easyconfigs(repo_path, ec['ec']) if pr: pr_msg = "PR#%s" % pr else: pr_msg = "new PR" _log.debug("File in %s %s has these related easyconfigs: %s" % (pr_msg, ec['spec'], files)) if files: lines.append(multidiff(ec['spec'], files, colored=colored)) else: lines.extend([ '', "(no related easyconfigs found for %s)\n" % os.path.basename(ec['spec']) ]) if pr: file_info = det_file_info(pr_files, download_repo_path) pr_target_account = build_option('pr_target_account') github_user = build_option('github_user') pr_data, _ = fetch_pr_data(pr, pr_target_account, pr_target_repo, github_user) pr_labels = [label['name'] for label in pr_data['labels']] if not testing else [] expected_labels = det_pr_labels(file_info, pr_target_repo) missing_labels = [ label for label in expected_labels if label not in pr_labels ] if missing_labels: lines.extend([ '', "This PR should be labelled with %s" % ', '.join(["'%s'" % ml for ml in missing_labels]) ]) if not pr_data['milestone']: lines.extend(['', "This PR should be associated with a milestone"]) elif '.x' in pr_data['milestone']['title']: lines.extend([ '', "This PR is associated with a generic '.x' milestone, " "it should be associated to the next release milestone once merged" ]) return '\n'.join(lines)
def main(testing_data=(None, None, None)): """ Main function: @arg options: a tuple: (options, paths, logger, logfile, hn) as defined in parse_options This function will: - read easyconfig - build software """ # purposely session state very early, to avoid modules loaded by EasyBuild meddling in init_session_state = session_state() # disallow running EasyBuild as root if os.getuid() == 0: sys.stderr.write( "ERROR: You seem to be running EasyBuild with root privileges.\n" "That's not wise, so let's end this here.\n" "Exiting.\n") sys.exit(1) # steer behavior when testing main testing = testing_data[0] is not None args, logfile, do_build = testing_data # initialise options eb_go = eboptions.parse_options(args=args) options = eb_go.options orig_paths = eb_go.args eb_config = eb_go.generate_cmd_line(add_default=True) init_session_state.update({'easybuild_configuration': eb_config}) # set umask (as early as possible) if options.umask is not None: new_umask = int(options.umask, 8) old_umask = os.umask(new_umask) # set temporary directory to use eb_tmpdir = set_tmpdir(options.tmpdir) # initialise logging for main if options.logtostdout: fancylogger.logToScreen(enable=True, stdout=True) else: if logfile is None: # mkstemp returns (fd,filename), fd is from os.open, not regular open! fd, logfile = tempfile.mkstemp(suffix='.log', prefix='easybuild-') os.close(fd) fancylogger.logToFile(logfile) print_msg('temporary log file in case of crash %s' % (logfile), log=None, silent=testing) global _log _log = fancylogger.getLogger(fname=False) if options.umask is not None: _log.info("umask set to '%s' (used to be '%s')" % (oct(new_umask), oct(old_umask))) # hello world! _log.info(this_is_easybuild()) # how was EB called? eb_command_line = eb_go.generate_cmd_line() + eb_go.args _log.info("Command line: %s" % (" ".join(eb_command_line))) _log.info("Using %s as temporary directory" % eb_tmpdir) if not options.robot is None: if options.robot: _log.info("Using robot path(s): %s" % options.robot) else: _log.error( "No robot paths specified, and unable to determine easybuild-easyconfigs install path." ) # do not pass options.robot, it's not a list instance (and it shouldn't be modified) robot_path = None if options.robot: robot_path = list(options.robot) # determine easybuild-easyconfigs package install path easyconfigs_paths = get_paths_for("easyconfigs", robot_path=robot_path) # keep track of paths for install easyconfigs, so we can obtain find specified easyconfigs easyconfigs_pkg_full_paths = easyconfigs_paths[:] if not easyconfigs_paths: _log.warning( "Failed to determine install path for easybuild-easyconfigs package." ) # process software build specifications (if any), i.e. # software name/version, toolchain name/version, extra patches, ... (try_to_generate, build_specs) = process_software_build_specs(options) # specified robot paths are preferred over installed easyconfig files # --try-X and --dep-graph both require --robot, so enable it with path of installed easyconfigs if robot_path or try_to_generate or options.dep_graph: if robot_path is None: robot_path = [] robot_path.extend(easyconfigs_paths) easyconfigs_paths = robot_path[:] _log.info( "Extended list of robot paths with paths for installed easyconfigs: %s" % robot_path) # initialise the easybuild configuration config.init(options, eb_go.get_options_by_section('config')) # building a dependency graph implies force, so that all dependencies are retained # and also skips validation of easyconfigs (e.g. checking os dependencies) retain_all_deps = False if options.dep_graph: _log.info("Enabling force to generate dependency graph.") options.force = True retain_all_deps = True config.init_build_options({ 'aggregate_regtest': options.aggregate_regtest, 'allow_modules_tool_mismatch': options.allow_modules_tool_mismatch, 'check_osdeps': not options.ignore_osdeps, 'cleanup_builddir': options.cleanup_builddir, 'command_line': eb_command_line, 'debug': options.debug, 'dry_run': options.dry_run, 'easyblock': options.easyblock, 'experimental': options.experimental, 'force': options.force, 'github_user': options.github_user, 'group': options.group, 'ignore_dirs': options.ignore_dirs, 'modules_footer': options.modules_footer, 'only_blocks': options.only_blocks, 'recursive_mod_unload': options.recursive_module_unload, 'regtest_output_dir': options.regtest_output_dir, 'retain_all_deps': retain_all_deps, 'robot_path': robot_path, 'sequential': options.sequential, 'silent': testing, 'set_gid_bit': options.set_gid_bit, 'skip': options.skip, 'skip_test_cases': options.skip_test_cases, 'sticky_bit': options.sticky_bit, 'stop': options.stop, 'umask': options.umask, 'valid_module_classes': module_classes(), 'valid_stops': [x[0] for x in EasyBlock.get_steps()], 'validate': not options.force, }) # obtain list of loaded modules, build options must be initialized first modlist = session_module_list() init_session_state.update({'module_list': modlist}) _log.debug("Initial session state: %s" % init_session_state) # search for easyconfigs if options.search or options.search_short: search_path = [os.getcwd()] if easyconfigs_paths: search_path = easyconfigs_paths query = options.search or options.search_short ignore_dirs = config.build_option('ignore_dirs') silent = config.build_option('silent') search_file(search_path, query, short=not options.search, ignore_dirs=ignore_dirs, silent=silent) paths = [] if len(orig_paths) == 0: if options.from_pr: pr_path = os.path.join(eb_tmpdir, "files_pr%s" % options.from_pr) pr_files = fetch_easyconfigs_from_pr( options.from_pr, path=pr_path, github_user=options.github_user) paths = [(path, False) for path in pr_files if path.endswith('.eb')] elif 'name' in build_specs: paths = [ obtain_path(build_specs, easyconfigs_paths, try_to_generate=try_to_generate, exit_on_error=not testing) ] elif not any([ options.aggregate_regtest, options.search, options.search_short, options.regtest ]): print_error(( "Please provide one or multiple easyconfig files, or use software build " "options to make EasyBuild search for easyconfigs"), log=_log, opt_parser=eb_go.parser, exit_on_error=not testing) else: # look for easyconfigs with relative paths in easybuild-easyconfigs package, # unless they were found at the given relative paths if easyconfigs_pkg_full_paths: # determine which easyconfigs files need to be found, if any ecs_to_find = [] for idx, orig_path in enumerate(orig_paths): if orig_path == os.path.basename( orig_path) and not os.path.exists(orig_path): ecs_to_find.append((idx, orig_path)) _log.debug("List of easyconfig files to find: %s" % ecs_to_find) # find missing easyconfigs by walking paths with installed easyconfig files for path in easyconfigs_pkg_full_paths: _log.debug( "Looking for missing easyconfig files (%d left) in %s..." % (len(ecs_to_find), path)) for (subpath, dirnames, filenames) in os.walk(path, topdown=True): for idx, orig_path in ecs_to_find[:]: if orig_path in filenames: full_path = os.path.join(subpath, orig_path) _log.info("Found %s in %s: %s" % (orig_path, path, full_path)) orig_paths[idx] = full_path # if file was found, stop looking for it (first hit wins) ecs_to_find.remove((idx, orig_path)) # stop os.walk insanity as soon as we have all we need (os.walk loop) if len(ecs_to_find) == 0: break # ignore subdirs specified to be ignored by replacing items in dirnames list used by os.walk dirnames[:] = [ d for d in dirnames if not d in options.ignore_dirs ] # stop os.walk insanity as soon as we have all we need (paths loop) if len(ecs_to_find) == 0: break # indicate that specified paths do not contain generated easyconfig files paths = [(path, False) for path in orig_paths] _log.debug("Paths: %s" % paths) # run regtest if options.regtest or options.aggregate_regtest: _log.info("Running regression test") if paths: ec_paths = [path[0] for path in paths] else: # fallback: easybuild-easyconfigs install path ec_paths = easyconfigs_pkg_full_paths regtest_ok = regtest(ec_paths) if not regtest_ok: _log.info("Regression test failed (partially)!") sys.exit(31) # exit -> 3x1t -> 31 # read easyconfig files easyconfigs = [] for (path, generated) in paths: path = os.path.abspath(path) if not os.path.exists(path): print_error("Can't find path %s" % path) try: ec_files = find_easyconfigs(path, ignore_dirs=options.ignore_dirs) for ec_file in ec_files: # only pass build specs when not generating easyconfig files if try_to_generate: ecs = process_easyconfig(ec_file) else: ecs = process_easyconfig(ec_file, build_specs=build_specs) easyconfigs.extend(ecs) except IOError, err: _log.error("Processing easyconfigs in path %s failed: %s" % (path, err))