def test_fetch_files_from_pr_cache(self):
"""Test caching for fetch_files_from_pr."""
if self.skip_github_tests:
print("Skipping test_fetch_files_from_pr_cache, no GitHub token available?")
return
init_config(build_options={
'pr_target_account': gh.GITHUB_EB_MAIN,
})
# clear cache first, to make sure we start with a clean slate
gh.fetch_files_from_pr.clear_cache()
self.assertFalse(gh.fetch_files_from_pr._cache)
pr7159_filenames = [
'DOLFIN-2018.1.0.post1-foss-2018a-Python-3.6.4.eb',
'OpenFOAM-5.0-20180108-foss-2018a.eb',
'OpenFOAM-5.0-20180108-intel-2018a.eb',
'OpenFOAM-6-foss-2018b.eb',
'OpenFOAM-6-intel-2018a.eb',
'OpenFOAM-v1806-foss-2018b.eb',
'PETSc-3.9.3-foss-2018a.eb',
'SCOTCH-6.0.6-foss-2018a.eb',
'SCOTCH-6.0.6-foss-2018b.eb',
'SCOTCH-6.0.6-intel-2018a.eb',
'Trilinos-12.12.1-foss-2018a-Python-3.6.4.eb'
]
pr7159_files = gh.fetch_easyconfigs_from_pr(7159, path=self.test_prefix, github_user=GITHUB_TEST_ACCOUNT)
self.assertEqual(sorted(pr7159_filenames), sorted(os.path.basename(f) for f in pr7159_files))
# check that cache has been populated for PR 7159
self.assertEqual(len(gh.fetch_files_from_pr._cache.keys()), 1)
# github_account value is None (results in using default 'easybuilders')
cache_key = (7159, None, 'easybuild-easyconfigs', self.test_prefix)
self.assertTrue(cache_key in gh.fetch_files_from_pr._cache.keys())
cache_entry = gh.fetch_files_from_pr._cache[cache_key]
self.assertEqual(sorted([os.path.basename(f) for f in cache_entry]), sorted(pr7159_filenames))
# same query should return result from cache entry
res = gh.fetch_easyconfigs_from_pr(7159, path=self.test_prefix, github_user=GITHUB_TEST_ACCOUNT)
self.assertEqual(res, pr7159_files)
# inject entry in cache and check result of matching query
pr_id = 12345
tmpdir = os.path.join(self.test_prefix, 'easyblocks-pr-12345')
pr12345_files = [
os.path.join(tmpdir, 'foo.py'),
os.path.join(tmpdir, 'bar.py'),
]
for fp in pr12345_files:
write_file(fp, '')
# github_account value is None (results in using default 'easybuilders')
cache_key = (pr_id, None, 'easybuild-easyblocks', tmpdir)
gh.fetch_files_from_pr.update_cache({cache_key: pr12345_files})
res = gh.fetch_easyblocks_from_pr(12345, tmpdir)
self.assertEqual(sorted(pr12345_files), sorted(res))
def test_fetch_easyconfigs_from_pr(self):
"""Test fetch_easyconfigs_from_pr function."""
if self.github_token is None:
print "Skipping test_fetch_easyconfigs_from_pr, no GitHub token available?"
return
tmpdir = tempfile.mkdtemp()
# PR for ictce/6.2.5, see https://github.com/hpcugent/easybuild-easyconfigs/pull/726/files
all_ecs = [
'gzip-1.6-ictce-6.2.5.eb', 'icc-2013_sp1.2.144.eb',
'ictce-6.2.5.eb', 'ifort-2013_sp1.2.144.eb', 'imkl-11.1.2.144.eb',
'impi-4.1.3.049.eb'
]
try:
ec_files = gh.fetch_easyconfigs_from_pr(
726, path=tmpdir, github_user=GITHUB_TEST_ACCOUNT)
self.assertEqual(all_ecs,
sorted([os.path.basename(f) for f in ec_files]))
self.assertEqual(all_ecs, sorted(os.listdir(tmpdir)))
# PR for EasyBuild v1.13.0 release (250+ commits, 218 files changed)
err_msg = "PR #897 contains more than .* commits, can't obtain last commit"
self.assertErrorRegex(EasyBuildError,
err_msg,
gh.fetch_easyconfigs_from_pr,
897,
github_user=GITHUB_TEST_ACCOUNT)
except URLError, err:
print "Ignoring URLError '%s' in test_fetch_easyconfigs_from_pr" % err
def det_easyconfig_paths(orig_paths):
"""
Determine paths to easyconfig files.
:param orig_paths: list of original easyconfig paths
:return: list of paths to easyconfig files
"""
from_pr = build_option('from_pr')
robot_path = build_option('robot_path')
# list of specified easyconfig files
ec_files = orig_paths[:]
if from_pr is not None:
pr_files = fetch_easyconfigs_from_pr(from_pr)
if ec_files:
# replace paths for specified easyconfigs that are touched in PR
for i, ec_file in enumerate(ec_files):
for pr_file in pr_files:
if ec_file == os.path.basename(pr_file):
ec_files[i] = pr_file
else:
# if no easyconfigs are specified, use all the ones touched in the PR
ec_files = [path for path in pr_files if path.endswith('.eb')]
if ec_files and robot_path:
ignore_subdirs = build_option('ignore_dirs')
if not build_option('consider_archived_easyconfigs'):
ignore_subdirs.append(EASYCONFIGS_ARCHIVE_DIR)
ec_files = locate_files(ec_files,
robot_path,
ignore_subdirs=ignore_subdirs)
return ec_files
def det_easyconfig_paths(orig_paths):
"""
Determine paths to easyconfig files.
@param orig_paths: list of original easyconfig paths
@return: list of paths to easyconfig files
"""
from_pr = build_option('from_pr')
robot_path = build_option('robot_path')
# list of specified easyconfig files
ec_files = orig_paths[:]
if from_pr is not None:
pr_files = fetch_easyconfigs_from_pr(from_pr)
if ec_files:
# replace paths for specified easyconfigs that are touched in PR
for i, ec_file in enumerate(ec_files):
for pr_file in pr_files:
if ec_file == os.path.basename(pr_file):
ec_files[i] = pr_file
else:
# if no easyconfigs are specified, use all the ones touched in the PR
ec_files = [path for path in pr_files if path.endswith('.eb')]
if ec_files and robot_path:
# look for easyconfigs with relative paths in robot search path,
# unless they were found at the given relative paths
# determine which easyconfigs files need to be found, if any
ecs_to_find = []
for idx, ec_file in enumerate(ec_files):
if ec_file == os.path.basename(ec_file) and not os.path.exists(ec_file):
ecs_to_find.append((idx, ec_file))
_log.debug("List of easyconfig files to find: %s" % ecs_to_find)
# find missing easyconfigs by walking paths in robot search path
for path in robot_path:
_log.debug("Looking for missing easyconfig files (%d left) in %s..." % (len(ecs_to_find), path))
for (subpath, dirnames, filenames) in os.walk(path, topdown=True):
for idx, orig_path in ecs_to_find[:]:
if orig_path in filenames:
full_path = os.path.join(subpath, orig_path)
_log.info("Found %s in %s: %s" % (orig_path, path, full_path))
ec_files[idx] = full_path
# if file was found, stop looking for it (first hit wins)
ecs_to_find.remove((idx, orig_path))
# stop os.walk insanity as soon as we have all we need (os.walk loop)
if not ecs_to_find:
break
# ignore subdirs specified to be ignored by replacing items in dirnames list used by os.walk
dirnames[:] = [d for d in dirnames if d not in build_option('ignore_dirs')]
# stop os.walk insanity as soon as we have all we need (outer loop)
if not ecs_to_find:
break
return ec_files
def review_pr(pr, colored=True, branch='develop'):
"""
Print multi-diff overview between easyconfigs in specified PR and specified branch.
:param pr: pull request number in easybuild-easyconfigs repo to review
:param colored: boolean indicating whether a colored multi-diff should be generated
:param branch: easybuild-easyconfigs branch to compare with
"""
tmpdir = tempfile.mkdtemp()
download_repo_path = download_repo(branch=branch, path=tmpdir)
repo_path = os.path.join(download_repo_path, 'easybuild', 'easyconfigs')
pr_files = [
path for path in fetch_easyconfigs_from_pr(pr) if path.endswith('.eb')
]
lines = []
ecs, _ = parse_easyconfigs([(fp, False) for fp in pr_files],
validate=False)
for ec in ecs:
files = find_related_easyconfigs(repo_path, ec['ec'])
_log.debug("File in PR#%s %s has these related easyconfigs: %s" %
(pr, ec['spec'], files))
if files:
lines.append(multidiff(ec['spec'], files, colored=colored))
else:
lines.extend([
'',
"(no related easyconfigs found for %s)\n" %
os.path.basename(ec['spec'])
])
return '\n'.join(lines)
def test_fetch_easyconfigs_from_pr(self):
"""Test fetch_easyconfigs_from_pr function."""
if self.github_token is None:
print "Skipping test_fetch_easyconfigs_from_pr, no GitHub token available?"
return
tmpdir = tempfile.mkdtemp()
# PR for rename of ffmpeg to FFmpeg, see https://github.com/hpcugent/easybuild-easyconfigs/pull/2481/files
all_ecs = [
'FFmpeg-2.4-intel-2014.06.eb',
'FFmpeg-2.4-intel-2014b.eb',
'FFmpeg-2.8-intel-2015b.eb',
'OpenCV-2.4.9-intel-2014.06.eb',
'OpenCV-2.4.9-intel-2014b.eb',
'animation-2.4-intel-2015b-R-3.2.1.eb',
]
try:
ec_files = gh.fetch_easyconfigs_from_pr(2481, path=tmpdir, github_user=GITHUB_TEST_ACCOUNT)
self.assertEqual(all_ecs, sorted([os.path.basename(f) for f in ec_files]))
self.assertEqual(all_ecs, sorted([os.path.basename(f) for f in glob.glob(os.path.join(tmpdir, '*', '*'))]))
# PR for EasyBuild v1.13.0 release (250+ commits, 218 files changed)
err_msg = "PR #897 contains more than .* commits, can't obtain last commit"
self.assertErrorRegex(EasyBuildError, err_msg, gh.fetch_easyconfigs_from_pr, 897,
github_user=GITHUB_TEST_ACCOUNT)
except URLError, err:
print "Ignoring URLError '%s' in test_fetch_easyconfigs_from_pr" % err
def review_pr(paths=None, pr=None, colored=True, branch='develop'):
"""
Print multi-diff overview between specified easyconfigs or PR and specified branch.
:param pr: pull request number in easybuild-easyconfigs repo to review
:param paths: path tuples (path, generated) of easyconfigs to review
:param colored: boolean indicating whether a colored multi-diff should be generated
:param branch: easybuild-easyconfigs branch to compare with
"""
tmpdir = tempfile.mkdtemp()
download_repo_path = download_repo(branch=branch, path=tmpdir)
repo_path = os.path.join(download_repo_path, 'easybuild', 'easyconfigs')
if pr:
pr_files = [path for path in fetch_easyconfigs_from_pr(pr) if path.endswith('.eb')]
elif paths:
pr_files = paths
else:
raise EasyBuildError("No PR # or easyconfig path specified")
lines = []
ecs, _ = parse_easyconfigs([(fp, False) for fp in pr_files], validate=False)
for ec in ecs:
files = find_related_easyconfigs(repo_path, ec['ec'])
if pr:
pr_msg = "PR#%s" % pr
else:
pr_msg = "new PR"
_log.debug("File in %s %s has these related easyconfigs: %s" % (pr_msg, ec['spec'], files))
if files:
lines.append(multidiff(ec['spec'], files, colored=colored))
else:
lines.extend(['', "(no related easyconfigs found for %s)\n" % os.path.basename(ec['spec'])])
return '\n'.join(lines)
def det_easyconfig_paths(orig_paths):
"""
Determine paths to easyconfig files.
@param orig_paths: list of original easyconfig paths
@return: list of paths to easyconfig files
"""
from_pr = build_option('from_pr')
robot_path = build_option('robot_path')
# list of specified easyconfig files
ec_files = orig_paths[:]
if from_pr is not None:
pr_files = fetch_easyconfigs_from_pr(from_pr)
if ec_files:
# replace paths for specified easyconfigs that are touched in PR
for i, ec_file in enumerate(ec_files):
for pr_file in pr_files:
if ec_file == os.path.basename(pr_file):
ec_files[i] = pr_file
else:
# if no easyconfigs are specified, use all the ones touched in the PR
ec_files = [path for path in pr_files if path.endswith('.eb')]
if ec_files and robot_path:
# look for easyconfigs with relative paths in robot search path,
# unless they were found at the given relative paths
# determine which easyconfigs files need to be found, if any
ecs_to_find = []
for idx, ec_file in enumerate(ec_files):
if ec_file == os.path.basename(ec_file) and not os.path.exists(ec_file):
ecs_to_find.append((idx, ec_file))
_log.debug("List of easyconfig files to find: %s" % ecs_to_find)
# find missing easyconfigs by walking paths in robot search path
for path in robot_path:
_log.debug("Looking for missing easyconfig files (%d left) in %s..." % (len(ecs_to_find), path))
for (subpath, dirnames, filenames) in os.walk(path, topdown=True):
for idx, orig_path in ecs_to_find[:]:
if orig_path in filenames:
full_path = os.path.join(subpath, orig_path)
_log.info("Found %s in %s: %s" % (orig_path, path, full_path))
ec_files[idx] = full_path
# if file was found, stop looking for it (first hit wins)
ecs_to_find.remove((idx, orig_path))
# stop os.walk insanity as soon as we have all we need (os.walk loop)
if not ecs_to_find:
break
# ignore subdirs specified to be ignored by replacing items in dirnames list used by os.walk
dirnames[:] = [d for d in dirnames if d not in build_option('ignore_dirs')]
# stop os.walk insanity as soon as we have all we need (outer loop)
if not ecs_to_find:
break
return ec_files
def test_fetch_easyconfigs_from_pr(self):
"""Test fetch_easyconfigs_from_pr function."""
if self.github_token is None:
print "Skipping test_fetch_easyconfigs_from_pr, no GitHub token available?"
return
tmpdir = tempfile.mkdtemp()
# PR for rename of ffmpeg to FFmpeg, see https://github.com/easybuilders/easybuild-easyconfigs/pull/2481/files
all_ecs = [
'FFmpeg-2.4-intel-2014.06.eb',
'FFmpeg-2.4-intel-2014b.eb',
'FFmpeg-2.8-intel-2015b.eb',
'OpenCV-2.4.9-intel-2014.06.eb',
'OpenCV-2.4.9-intel-2014b.eb',
'animation-2.4-intel-2015b-R-3.2.1.eb',
]
try:
ec_files = gh.fetch_easyconfigs_from_pr(2481, path=tmpdir, github_user=GITHUB_TEST_ACCOUNT)
self.assertEqual(all_ecs, sorted([os.path.basename(f) for f in ec_files]))
# PR for EasyBuild v1.13.0 release (250+ commits, 218 files changed)
err_msg = "PR #897 contains more than .* commits, can't obtain last commit"
self.assertErrorRegex(EasyBuildError, err_msg, gh.fetch_easyconfigs_from_pr, 897,
github_user=GITHUB_TEST_ACCOUNT)
except URLError, err:
print "Ignoring URLError '%s' in test_fetch_easyconfigs_from_pr" % err
def test_fetch_easyconfigs_from_pr(self):
"""Test fetch_easyconfigs_from_pr function."""
if self.skip_github_tests:
print(
"Skipping test_fetch_easyconfigs_from_pr, no GitHub token available?"
)
return
init_config(build_options={
'pr_target_account': gh.GITHUB_EB_MAIN,
})
# PR for rename of arrow to Arrow,
# see https://github.com/easybuilders/easybuild-easyconfigs/pull/8007/files
all_ecs_pr8007 = [
'Arrow-0.7.1-intel-2017b-Python-3.6.3.eb',
'bat-0.3.3-fix-pyspark.patch',
'bat-0.3.3-intel-2017b-Python-3.6.3.eb',
]
# PR where also files are patched in test/
# see https://github.com/easybuilders/easybuild-easyconfigs/pull/6587/files
all_ecs_pr6587 = [
'WIEN2k-18.1-foss-2018a.eb',
'WIEN2k-18.1-gimkl-2017a.eb',
'WIEN2k-18.1-intel-2018a.eb',
'libxc-4.2.3-foss-2018a.eb',
'libxc-4.2.3-gimkl-2017a.eb',
'libxc-4.2.3-intel-2018a.eb',
]
# PR where files are renamed
# see https://github.com/easybuilders/easybuild-easyconfigs/pull/7159/files
all_ecs_pr7159 = [
'DOLFIN-2018.1.0.post1-foss-2018a-Python-3.6.4.eb',
'OpenFOAM-5.0-20180108-foss-2018a.eb',
'OpenFOAM-5.0-20180108-intel-2018a.eb', 'OpenFOAM-6-foss-2018b.eb',
'OpenFOAM-6-intel-2018a.eb', 'OpenFOAM-v1806-foss-2018b.eb',
'PETSc-3.9.3-foss-2018a.eb', 'SCOTCH-6.0.6-foss-2018a.eb',
'SCOTCH-6.0.6-foss-2018b.eb', 'SCOTCH-6.0.6-intel-2018a.eb',
'Trilinos-12.12.1-foss-2018a-Python-3.6.4.eb'
]
for pr, all_ecs in [(8007, all_ecs_pr8007), (6587, all_ecs_pr6587),
(7159, all_ecs_pr7159)]:
try:
tmpdir = os.path.join(self.test_prefix, 'pr%s' % pr)
ec_files = gh.fetch_easyconfigs_from_pr(
pr, path=tmpdir, github_user=GITHUB_TEST_ACCOUNT)
self.assertEqual(
sorted(all_ecs),
sorted([os.path.basename(f) for f in ec_files]))
except URLError as err:
print(
"Ignoring URLError '%s' in test_fetch_easyconfigs_from_pr"
% err)
def test_fetch_easyconfigs_from_pr(self):
"""Test fetch_easyconfigs_from_pr function."""
if self.skip_github_tests:
print "Skipping test_fetch_easyconfigs_from_pr, no GitHub token available?"
return
init_config(build_options={
'pr_target_account': gh.GITHUB_EB_MAIN,
})
# PR for rename of ffmpeg to FFmpeg,
# see https://github.com/easybuilders/easybuild-easyconfigs/pull/2481/files
all_ecs_pr2481 = [
'FFmpeg-2.4-intel-2014.06.eb',
'FFmpeg-2.4-intel-2014b.eb',
'FFmpeg-2.8-intel-2015b.eb',
'OpenCV-2.4.9-intel-2014.06.eb',
'OpenCV-2.4.9-intel-2014b.eb',
'animation-2.4-intel-2015b-R-3.2.1.eb',
]
# PR where also files are patched in test/
# see https://github.com/easybuilders/easybuild-easyconfigs/pull/6587/files
all_ecs_pr6587 = [
'WIEN2k-18.1-foss-2018a.eb',
'WIEN2k-18.1-gimkl-2017a.eb',
'WIEN2k-18.1-intel-2018a.eb',
'libxc-4.2.3-foss-2018a.eb',
'libxc-4.2.3-gimkl-2017a.eb',
'libxc-4.2.3-intel-2018a.eb',
]
# PR where files are renamed
# see https://github.com/easybuilders/easybuild-easyconfigs/pull/7159/files
all_ecs_pr7159 = [
'DOLFIN-2018.1.0.post1-foss-2018a-Python-3.6.4.eb',
'OpenFOAM-5.0-20180108-foss-2018a.eb',
'OpenFOAM-5.0-20180108-intel-2018a.eb',
'OpenFOAM-6-foss-2018b.eb',
'OpenFOAM-6-intel-2018a.eb',
'OpenFOAM-v1806-foss-2018b.eb',
'PETSc-3.9.3-foss-2018a.eb',
'SCOTCH-6.0.6-foss-2018a.eb',
'SCOTCH-6.0.6-foss-2018b.eb',
'SCOTCH-6.0.6-intel-2018a.eb',
'Trilinos-12.12.1-foss-2018a-Python-3.6.4.eb'
]
for pr, all_ecs in [(2481, all_ecs_pr2481), (6587, all_ecs_pr6587), (7159, all_ecs_pr7159)]:
try:
tmpdir = os.path.join(self.test_prefix, 'pr%s' % pr)
ec_files = gh.fetch_easyconfigs_from_pr(pr, path=tmpdir, github_user=GITHUB_TEST_ACCOUNT)
self.assertEqual(sorted(all_ecs), sorted([os.path.basename(f) for f in ec_files]))
except URLError, err:
print "Ignoring URLError '%s' in test_fetch_easyconfigs_from_pr" % err
def det_easyconfig_paths(orig_paths, from_pr=None, easyconfigs_pkg_paths=None):
"""
Determine paths to easyconfig files.
@param orig_paths: list of original easyconfig paths
@param from_pr: pull request number to fetch easyconfigs from
@param easyconfigs_pkg_paths: paths to installed easyconfigs package
"""
if easyconfigs_pkg_paths is None:
easyconfigs_pkg_paths = []
ec_files = orig_paths[:]
if not ec_files and from_pr:
pr_files = fetch_easyconfigs_from_pr(from_pr)
ec_files = [path for path in pr_files if path.endswith('.eb')]
elif ec_files and easyconfigs_pkg_paths:
# look for easyconfigs with relative paths in easybuild-easyconfigs package,
# unless they were found at the given relative paths
# determine which easyconfigs files need to be found, if any
ecs_to_find = []
for idx, ec_file in enumerate(ec_files):
if ec_file == os.path.basename(ec_file) and not os.path.exists(ec_file):
ecs_to_find.append((idx, ec_file))
_log.debug("List of easyconfig files to find: %s" % ecs_to_find)
# find missing easyconfigs by walking paths with installed easyconfig files
for path in easyconfigs_pkg_paths:
_log.debug("Looking for missing easyconfig files (%d left) in %s..." % (len(ecs_to_find), path))
for (subpath, dirnames, filenames) in os.walk(path, topdown=True):
for idx, orig_path in ecs_to_find[:]:
if orig_path in filenames:
full_path = os.path.join(subpath, orig_path)
_log.info("Found %s in %s: %s" % (orig_path, path, full_path))
ec_files[idx] = full_path
# if file was found, stop looking for it (first hit wins)
ecs_to_find.remove((idx, orig_path))
# stop os.walk insanity as soon as we have all we need (os.walk loop)
if not ecs_to_find:
break
# ignore subdirs specified to be ignored by replacing items in dirnames list used by os.walk
dirnames[:] = [d for d in dirnames if not d in build_option('ignore_dirs')]
# stop os.walk insanity as soon as we have all we need (outer loop)
if not ecs_to_find:
break
# indicate that specified paths do not contain generated easyconfig files
return [(ec_file, False) for ec_file in ec_files]
def det_easyconfig_paths(orig_paths):
"""
Determine paths to easyconfig files.
:param orig_paths: list of original easyconfig paths
:return: list of paths to easyconfig files
"""
try:
from_prs = [int(x) for x in build_option('from_pr')]
except ValueError:
raise EasyBuildError(
"Argument to --from-pr must be a comma separated list of PR #s.")
robot_path = build_option('robot_path')
# list of specified easyconfig files
ec_files = orig_paths[:]
if from_prs:
pr_files = []
for pr in from_prs:
# path to where easyconfig files should be downloaded is determined via 'pr_paths' build option,
# which corresponds to the list of PR paths returned by alt_easyconfig_paths
pr_files.extend(fetch_easyconfigs_from_pr(pr))
if ec_files:
# replace paths for specified easyconfigs that are touched in PR
for i, ec_file in enumerate(ec_files):
for pr_file in pr_files:
if ec_file == os.path.basename(pr_file):
ec_files[i] = pr_file
else:
# if no easyconfigs are specified, use all the ones touched in the PR
ec_files = [path for path in pr_files if path.endswith('.eb')]
filter_ecs = build_option('filter_ecs')
if filter_ecs:
ec_files = [
ec for ec in ec_files if not any(
fnmatch.fnmatch(ec, filter_spec) for filter_spec in filter_ecs)
]
if ec_files and robot_path:
ignore_subdirs = build_option('ignore_dirs')
if not build_option('consider_archived_easyconfigs'):
ignore_subdirs.append(EASYCONFIGS_ARCHIVE_DIR)
ec_files = locate_files(ec_files,
robot_path,
ignore_subdirs=ignore_subdirs)
return ec_files
def test_fetch_easyconfigs_from_pr(self):
"""Test fetch_easyconfigs_from_pr function."""
tmpdir = tempfile.mkdtemp()
# PR for ictce/6.2.5, see https://github.com/hpcugent/easybuild-easyconfigs/pull/726/files
all_ecs = ['gzip-1.6-ictce-6.2.5.eb', 'icc-2013_sp1.2.144.eb', 'ictce-6.2.5.eb', 'ifort-2013_sp1.2.144.eb',
'imkl-11.1.2.144.eb', 'impi-4.1.3.049.eb']
ec_files = fetch_easyconfigs_from_pr(726, path=tmpdir, github_user=GITHUB_TEST_ACCOUNT)
self.assertEqual(all_ecs, sorted([os.path.basename(f) for f in ec_files]))
self.assertEqual(all_ecs, sorted(os.listdir(tmpdir)))
shutil.rmtree(tmpdir)
# PR for EasyBuild v1.13.0 release (250+ commits, 218 files changed)
err_msg = "PR #897 contains more than .* commits, can't obtain last commit"
self.assertErrorRegex(EasyBuildError, err_msg, fetch_easyconfigs_from_pr, 897, github_user=GITHUB_TEST_ACCOUNT)
def test_fetch_easyconfigs_from_pr(self):
"""Test fetch_easyconfigs_from_pr function."""
if self.github_token is None:
print "Skipping test_fetch_easyconfigs_from_pr, no GitHub token available?"
return
tmpdir = tempfile.mkdtemp()
# PR for ictce/6.2.5, see https://github.com/hpcugent/easybuild-easyconfigs/pull/726/files
all_ecs = ['gzip-1.6-ictce-6.2.5.eb', 'icc-2013_sp1.2.144.eb', 'ictce-6.2.5.eb', 'ifort-2013_sp1.2.144.eb',
'imkl-11.1.2.144.eb', 'impi-4.1.3.049.eb']
try:
ec_files = fetch_easyconfigs_from_pr(726, path=tmpdir, github_user=GITHUB_TEST_ACCOUNT)
self.assertEqual(all_ecs, sorted([os.path.basename(f) for f in ec_files]))
self.assertEqual(all_ecs, sorted(os.listdir(tmpdir)))
# PR for EasyBuild v1.13.0 release (250+ commits, 218 files changed)
err_msg = "PR #897 contains more than .* commits, can't obtain last commit"
self.assertErrorRegex(EasyBuildError, err_msg, fetch_easyconfigs_from_pr, 897, github_user=GITHUB_TEST_ACCOUNT)
except URLError, err:
print "Ignoring URLError '%s' in test_fetch_easyconfigs_from_pr" % err
def test_fetch_easyconfigs_from_pr(self):
"""Test fetch_easyconfigs_from_pr function."""
if self.github_token is None:
print "Skipping test_fetch_easyconfigs_from_pr, no GitHub token available?"
return
# PR for rename of ffmpeg to FFmpeg,
# see https://github.com/easybuilders/easybuild-easyconfigs/pull/2481/files
all_ecs_pr2481 = [
'FFmpeg-2.4-intel-2014.06.eb',
'FFmpeg-2.4-intel-2014b.eb',
'FFmpeg-2.8-intel-2015b.eb',
'OpenCV-2.4.9-intel-2014.06.eb',
'OpenCV-2.4.9-intel-2014b.eb',
'animation-2.4-intel-2015b-R-3.2.1.eb',
]
# PR where also files are patched in test/
# see https://github.com/easybuilders/easybuild-easyconfigs/pull/6587/files
all_ecs_pr6587 = [
'WIEN2k-18.1-foss-2018a.eb',
'WIEN2k-18.1-gimkl-2017a.eb',
'WIEN2k-18.1-intel-2018a.eb',
'libxc-4.2.3-foss-2018a.eb',
'libxc-4.2.3-gimkl-2017a.eb',
'libxc-4.2.3-intel-2018a.eb',
]
for pr, all_ecs in [(2481, all_ecs_pr2481), (6587, all_ecs_pr6587)]:
try:
tmpdir = os.path.join(self.test_prefix, 'pr%s' % pr)
ec_files = gh.fetch_easyconfigs_from_pr(
pr, path=tmpdir, github_user=GITHUB_TEST_ACCOUNT)
self.assertEqual(
sorted(all_ecs),
sorted([os.path.basename(f) for f in ec_files]))
except URLError, err:
print "Ignoring URLError '%s' in test_fetch_easyconfigs_from_pr" % err
def test_fetch_easyconfigs_from_pr(self):
"""Test fetch_easyconfigs_from_pr function."""
tmpdir = tempfile.mkdtemp()
# PR for ictce/6.2.5, see https://github.com/hpcugent/easybuild-easyconfigs/pull/726/files
all_ecs = [
'gzip-1.6-ictce-6.2.5.eb', 'icc-2013_sp1.2.144.eb',
'ictce-6.2.5.eb', 'ifort-2013_sp1.2.144.eb', 'imkl-11.1.2.144.eb',
'impi-4.1.3.049.eb'
]
ec_files = fetch_easyconfigs_from_pr(726,
path=tmpdir,
github_user=GITHUB_TEST_ACCOUNT)
self.assertEqual(all_ecs,
sorted([os.path.basename(f) for f in ec_files]))
self.assertEqual(all_ecs, sorted(os.listdir(tmpdir)))
shutil.rmtree(tmpdir)
# PR for EasyBuild v1.13.0 release (250+ commits, 218 files changed)
err_msg = "PR #897 contains more than .* commits, can't obtain last commit"
self.assertErrorRegex(EasyBuildError,
err_msg,
fetch_easyconfigs_from_pr,
897,
github_user=GITHUB_TEST_ACCOUNT)
def review_pr(pr, colored=True, branch="develop"):
"""
Print multi-diff overview between easyconfigs in specified PR and specified branch.
@param pr: pull request number in easybuild-easyconfigs repo to review
@param colored: boolean indicating whether a colored multi-diff should be generated
@param branch: easybuild-easyconfigs branch to compare with
"""
tmpdir = tempfile.mkdtemp()
download_repo_path = download_repo(branch=branch, path=tmpdir)
repo_path = os.path.join(download_repo_path, "easybuild", "easyconfigs")
pr_files = [path for path in fetch_easyconfigs_from_pr(pr) if path.endswith(".eb")]
lines = []
ecs, _ = parse_easyconfigs([(fp, False) for fp in pr_files], validate=False)
for ec in ecs:
files = find_related_easyconfigs(repo_path, ec["ec"])
_log.debug("File in PR#%s %s has these related easyconfigs: %s" % (pr, ec["spec"], files))
if files:
lines.append(multidiff(ec["spec"], files, colored=colored))
else:
lines.extend(["", "(no related easyconfigs found for %s)\n" % os.path.basename(ec["spec"])])
return "\n".join(lines)
def main(testing_data=(None, None, None)):
"""
Main function:
@arg options: a tuple: (options, paths, logger, logfile, hn) as defined in parse_options
This function will:
- read easyconfig
- build software
"""
# purposely session state very early, to avoid modules loaded by EasyBuild meddling in
init_session_state = session_state()
# disallow running EasyBuild as root
if os.getuid() == 0:
sys.stderr.write("ERROR: You seem to be running EasyBuild with root privileges.\n"
"That's not wise, so let's end this here.\n"
"Exiting.\n")
sys.exit(1)
# steer behavior when testing main
testing = testing_data[0] is not None
args, logfile, do_build = testing_data
# initialise options
eb_go = eboptions.parse_options(args=args)
options = eb_go.options
orig_paths = eb_go.args
eb_config = eb_go.generate_cmd_line(add_default=True)
init_session_state.update({'easybuild_configuration': eb_config})
# set umask (as early as possible)
if options.umask is not None:
new_umask = int(options.umask, 8)
old_umask = os.umask(new_umask)
# set temporary directory to use
eb_tmpdir = set_tmpdir(options.tmpdir)
# initialise logging for main
if options.logtostdout:
fancylogger.logToScreen(enable=True, stdout=True)
else:
if logfile is None:
# mkstemp returns (fd,filename), fd is from os.open, not regular open!
fd, logfile = tempfile.mkstemp(suffix='.log', prefix='easybuild-')
os.close(fd)
fancylogger.logToFile(logfile)
print_msg('temporary log file in case of crash %s' % (logfile), log=None, silent=testing)
global _log
_log = fancylogger.getLogger(fname=False)
if options.umask is not None:
_log.info("umask set to '%s' (used to be '%s')" % (oct(new_umask), oct(old_umask)))
# hello world!
_log.info(this_is_easybuild())
# how was EB called?
eb_command_line = eb_go.generate_cmd_line() + eb_go.args
_log.info("Command line: %s" % (" ".join(eb_command_line)))
_log.info("Using %s as temporary directory" % eb_tmpdir)
if not options.robot is None:
if options.robot:
_log.info("Using robot path(s): %s" % options.robot)
else:
_log.error("No robot paths specified, and unable to determine easybuild-easyconfigs install path.")
# do not pass options.robot, it's not a list instance (and it shouldn't be modified)
robot_path = []
if options.robot:
robot_path = list(options.robot)
# determine easybuild-easyconfigs package install path
easyconfigs_paths = get_paths_for("easyconfigs", robot_path=robot_path)
# keep track of paths for install easyconfigs, so we can obtain find specified easyconfigs
easyconfigs_pkg_full_paths = easyconfigs_paths[:]
if not easyconfigs_paths:
_log.warning("Failed to determine install path for easybuild-easyconfigs package.")
# process software build specifications (if any), i.e.
# software name/version, toolchain name/version, extra patches, ...
(try_to_generate, build_specs) = process_software_build_specs(options)
# specified robot paths are preferred over installed easyconfig files
# --try-X and --dep-graph both require --robot, so enable it with path of installed easyconfigs
if robot_path or try_to_generate or options.dep_graph:
robot_path.extend(easyconfigs_paths)
easyconfigs_paths = robot_path[:]
_log.info("Extended list of robot paths with paths for installed easyconfigs: %s" % robot_path)
# prepend robot path with location where tweaked easyconfigs will be placed
tweaked_ecs_path = None
if try_to_generate and build_specs:
tweaked_ecs_path = os.path.join(eb_tmpdir, 'tweaked_easyconfigs')
robot_path.insert(0, tweaked_ecs_path)
# initialise the easybuild configuration
config.init(options, eb_go.get_options_by_section('config'))
# building a dependency graph implies force, so that all dependencies are retained
# and also skips validation of easyconfigs (e.g. checking os dependencies)
retain_all_deps = False
if options.dep_graph:
_log.info("Enabling force to generate dependency graph.")
options.force = True
retain_all_deps = True
if options.dep_graph or options.dry_run or options.dry_run_short:
options.ignore_osdeps = True
pr_path = None
if options.from_pr:
# extend robot search path with location where files touch in PR will be downloaded to
pr_path = os.path.join(eb_tmpdir, "files_pr%s" % options.from_pr)
robot_path.insert(0, pr_path)
_log.info("Prepended list of robot search paths with %s: %s" % (pr_path, robot_path))
config.init_build_options({
'aggregate_regtest': options.aggregate_regtest,
'allow_modules_tool_mismatch': options.allow_modules_tool_mismatch,
'check_osdeps': not options.ignore_osdeps,
'filter_deps': options.filter_deps,
'cleanup_builddir': options.cleanup_builddir,
'command_line': eb_command_line,
'debug': options.debug,
'dry_run': options.dry_run or options.dry_run_short,
'easyblock': options.easyblock,
'experimental': options.experimental,
'force': options.force,
'github_user': options.github_user,
'group': options.group,
'hidden': options.hidden,
'ignore_dirs': options.ignore_dirs,
'modules_footer': options.modules_footer,
'only_blocks': options.only_blocks,
'optarch': options.optarch,
'recursive_mod_unload': options.recursive_module_unload,
'regtest_output_dir': options.regtest_output_dir,
'retain_all_deps': retain_all_deps,
'robot_path': robot_path,
'sequential': options.sequential,
'silent': testing,
'set_gid_bit': options.set_gid_bit,
'skip': options.skip,
'skip_test_cases': options.skip_test_cases,
'sticky_bit': options.sticky_bit,
'stop': options.stop,
'suffix_modules_path': options.suffix_modules_path,
'test_report_env_filter': options.test_report_env_filter,
'umask': options.umask,
'valid_module_classes': module_classes(),
'valid_stops': [x[0] for x in EasyBlock.get_steps()],
'validate': not options.force,
})
# obtain list of loaded modules, build options must be initialized first
modlist = session_module_list(testing=testing)
init_session_state.update({'module_list': modlist})
_log.debug("Initial session state: %s" % init_session_state)
# search for easyconfigs
if options.search or options.search_short:
search_path = [os.getcwd()]
if easyconfigs_paths:
search_path = easyconfigs_paths
query = options.search or options.search_short
ignore_dirs = config.build_option('ignore_dirs')
silent = config.build_option('silent')
search_file(search_path, query, short=not options.search, ignore_dirs=ignore_dirs, silent=silent)
paths = []
if len(orig_paths) == 0:
if options.from_pr:
pr_files = fetch_easyconfigs_from_pr(options.from_pr, path=pr_path, github_user=options.github_user)
paths = [(path, False) for path in pr_files if path.endswith('.eb')]
elif 'name' in build_specs:
paths = [obtain_path(build_specs, easyconfigs_paths, try_to_generate=try_to_generate,
exit_on_error=not testing)]
elif not any([options.aggregate_regtest, options.search, options.search_short, options.regtest]):
print_error(("Please provide one or multiple easyconfig files, or use software build "
"options to make EasyBuild search for easyconfigs"),
log=_log, opt_parser=eb_go.parser, exit_on_error=not testing)
else:
# look for easyconfigs with relative paths in easybuild-easyconfigs package,
# unless they were found at the given relative paths
if easyconfigs_pkg_full_paths:
# determine which easyconfigs files need to be found, if any
ecs_to_find = []
for idx, orig_path in enumerate(orig_paths):
if orig_path == os.path.basename(orig_path) and not os.path.exists(orig_path):
ecs_to_find.append((idx, orig_path))
_log.debug("List of easyconfig files to find: %s" % ecs_to_find)
# find missing easyconfigs by walking paths with installed easyconfig files
for path in easyconfigs_pkg_full_paths:
_log.debug("Looking for missing easyconfig files (%d left) in %s..." % (len(ecs_to_find), path))
for (subpath, dirnames, filenames) in os.walk(path, topdown=True):
for idx, orig_path in ecs_to_find[:]:
if orig_path in filenames:
full_path = os.path.join(subpath, orig_path)
_log.info("Found %s in %s: %s" % (orig_path, path, full_path))
orig_paths[idx] = full_path
# if file was found, stop looking for it (first hit wins)
ecs_to_find.remove((idx, orig_path))
# stop os.walk insanity as soon as we have all we need (os.walk loop)
if len(ecs_to_find) == 0:
break
# ignore subdirs specified to be ignored by replacing items in dirnames list used by os.walk
dirnames[:] = [d for d in dirnames if not d in options.ignore_dirs]
# stop os.walk insanity as soon as we have all we need (paths loop)
if len(ecs_to_find) == 0:
break
# indicate that specified paths do not contain generated easyconfig files
paths = [(path, False) for path in orig_paths]
_log.debug("Paths: %s" % paths)
# run regtest
if options.regtest or options.aggregate_regtest:
_log.info("Running regression test")
if paths:
ec_paths = [path[0] for path in paths]
else: # fallback: easybuild-easyconfigs install path
ec_paths = easyconfigs_pkg_full_paths
regtest_ok = regtest(ec_paths)
if not regtest_ok:
_log.info("Regression test failed (partially)!")
sys.exit(31) # exit -> 3x1t -> 31
# read easyconfig files
easyconfigs = []
generated_ecs = False
for (path, generated) in paths:
path = os.path.abspath(path)
# keep track of whether any files were generated
generated_ecs |= generated
if not os.path.exists(path):
print_error("Can't find path %s" % path)
try:
ec_files = find_easyconfigs(path, ignore_dirs=options.ignore_dirs)
for ec_file in ec_files:
# only pass build specs when not generating easyconfig files
if try_to_generate:
ecs = process_easyconfig(ec_file)
else:
ecs = process_easyconfig(ec_file, build_specs=build_specs)
easyconfigs.extend(ecs)
except IOError, err:
_log.error("Processing easyconfigs in path %s failed: %s" % (path, err))
def review_pr(paths=None,
pr=None,
colored=True,
branch='develop',
testing=False):
"""
Print multi-diff overview between specified easyconfigs or PR and specified branch.
:param pr: pull request number in easybuild-easyconfigs repo to review
:param paths: path tuples (path, generated) of easyconfigs to review
:param colored: boolean indicating whether a colored multi-diff should be generated
:param branch: easybuild-easyconfigs branch to compare with
:param testing: whether to ignore PR labels (used in test_review_pr)
"""
pr_target_repo = build_option('pr_target_repo') or GITHUB_EASYCONFIGS_REPO
if pr_target_repo != GITHUB_EASYCONFIGS_REPO:
raise EasyBuildError(
"Reviewing PRs for repositories other than easyconfigs hasn't been implemented yet"
)
tmpdir = tempfile.mkdtemp()
download_repo_path = download_repo(branch=branch, path=tmpdir)
repo_path = os.path.join(download_repo_path, 'easybuild', 'easyconfigs')
if pr:
pr_files = [
path for path in fetch_easyconfigs_from_pr(pr)
if path.endswith('.eb')
]
elif paths:
pr_files = paths
else:
raise EasyBuildError("No PR # or easyconfig path specified")
lines = []
ecs, _ = parse_easyconfigs([(fp, False) for fp in pr_files],
validate=False)
for ec in ecs:
files = find_related_easyconfigs(repo_path, ec['ec'])
if pr:
pr_msg = "PR#%s" % pr
else:
pr_msg = "new PR"
_log.debug("File in %s %s has these related easyconfigs: %s" %
(pr_msg, ec['spec'], files))
if files:
lines.append(multidiff(ec['spec'], files, colored=colored))
else:
lines.extend([
'',
"(no related easyconfigs found for %s)\n" %
os.path.basename(ec['spec'])
])
if pr:
file_info = det_file_info(pr_files, download_repo_path)
pr_target_account = build_option('pr_target_account')
github_user = build_option('github_user')
pr_data, _ = fetch_pr_data(pr, pr_target_account, pr_target_repo,
github_user)
pr_labels = [label['name']
for label in pr_data['labels']] if not testing else []
expected_labels = det_pr_labels(file_info, pr_target_repo)
missing_labels = [
label for label in expected_labels if label not in pr_labels
]
if missing_labels:
lines.extend([
'',
"This PR should be labelled with %s" %
', '.join(["'%s'" % ml for ml in missing_labels])
])
if not pr_data['milestone']:
lines.extend(['', "This PR should be associated with a milestone"])
elif '.x' in pr_data['milestone']['title']:
lines.extend([
'', "This PR is associated with a generic '.x' milestone, "
"it should be associated to the next release milestone once merged"
])
return '\n'.join(lines)
def main(testing_data=(None, None, None)):
"""
Main function:
@arg options: a tuple: (options, paths, logger, logfile, hn) as defined in parse_options
This function will:
- read easyconfig
- build software
"""
# purposely session state very early, to avoid modules loaded by EasyBuild meddling in
init_session_state = session_state()
# disallow running EasyBuild as root
if os.getuid() == 0:
sys.stderr.write(
"ERROR: You seem to be running EasyBuild with root privileges.\n"
"That's not wise, so let's end this here.\n"
"Exiting.\n")
sys.exit(1)
# steer behavior when testing main
testing = testing_data[0] is not None
args, logfile, do_build = testing_data
# initialise options
eb_go = eboptions.parse_options(args=args)
options = eb_go.options
orig_paths = eb_go.args
eb_config = eb_go.generate_cmd_line(add_default=True)
init_session_state.update({'easybuild_configuration': eb_config})
# set umask (as early as possible)
if options.umask is not None:
new_umask = int(options.umask, 8)
old_umask = os.umask(new_umask)
# set temporary directory to use
eb_tmpdir = set_tmpdir(options.tmpdir)
# initialise logging for main
if options.logtostdout:
fancylogger.logToScreen(enable=True, stdout=True)
else:
if logfile is None:
# mkstemp returns (fd,filename), fd is from os.open, not regular open!
fd, logfile = tempfile.mkstemp(suffix='.log', prefix='easybuild-')
os.close(fd)
fancylogger.logToFile(logfile)
print_msg('temporary log file in case of crash %s' % (logfile),
log=None,
silent=testing)
global _log
_log = fancylogger.getLogger(fname=False)
if options.umask is not None:
_log.info("umask set to '%s' (used to be '%s')" %
(oct(new_umask), oct(old_umask)))
# hello world!
_log.info(this_is_easybuild())
# how was EB called?
eb_command_line = eb_go.generate_cmd_line() + eb_go.args
_log.info("Command line: %s" % (" ".join(eb_command_line)))
_log.info("Using %s as temporary directory" % eb_tmpdir)
if not options.robot is None:
if options.robot:
_log.info("Using robot path(s): %s" % options.robot)
else:
_log.error(
"No robot paths specified, and unable to determine easybuild-easyconfigs install path."
)
# do not pass options.robot, it's not a list instance (and it shouldn't be modified)
robot_path = None
if options.robot:
robot_path = list(options.robot)
# determine easybuild-easyconfigs package install path
easyconfigs_paths = get_paths_for("easyconfigs", robot_path=robot_path)
# keep track of paths for install easyconfigs, so we can obtain find specified easyconfigs
easyconfigs_pkg_full_paths = easyconfigs_paths[:]
if not easyconfigs_paths:
_log.warning(
"Failed to determine install path for easybuild-easyconfigs package."
)
# process software build specifications (if any), i.e.
# software name/version, toolchain name/version, extra patches, ...
(try_to_generate, build_specs) = process_software_build_specs(options)
# specified robot paths are preferred over installed easyconfig files
# --try-X and --dep-graph both require --robot, so enable it with path of installed easyconfigs
if robot_path or try_to_generate or options.dep_graph:
if robot_path is None:
robot_path = []
robot_path.extend(easyconfigs_paths)
easyconfigs_paths = robot_path[:]
_log.info(
"Extended list of robot paths with paths for installed easyconfigs: %s"
% robot_path)
# initialise the easybuild configuration
config.init(options, eb_go.get_options_by_section('config'))
# building a dependency graph implies force, so that all dependencies are retained
# and also skips validation of easyconfigs (e.g. checking os dependencies)
retain_all_deps = False
if options.dep_graph:
_log.info("Enabling force to generate dependency graph.")
options.force = True
retain_all_deps = True
config.init_build_options({
'aggregate_regtest':
options.aggregate_regtest,
'allow_modules_tool_mismatch':
options.allow_modules_tool_mismatch,
'check_osdeps':
not options.ignore_osdeps,
'cleanup_builddir':
options.cleanup_builddir,
'command_line':
eb_command_line,
'debug':
options.debug,
'dry_run':
options.dry_run,
'easyblock':
options.easyblock,
'experimental':
options.experimental,
'force':
options.force,
'github_user':
options.github_user,
'group':
options.group,
'ignore_dirs':
options.ignore_dirs,
'modules_footer':
options.modules_footer,
'only_blocks':
options.only_blocks,
'recursive_mod_unload':
options.recursive_module_unload,
'regtest_output_dir':
options.regtest_output_dir,
'retain_all_deps':
retain_all_deps,
'robot_path':
robot_path,
'sequential':
options.sequential,
'silent':
testing,
'set_gid_bit':
options.set_gid_bit,
'skip':
options.skip,
'skip_test_cases':
options.skip_test_cases,
'sticky_bit':
options.sticky_bit,
'stop':
options.stop,
'umask':
options.umask,
'valid_module_classes':
module_classes(),
'valid_stops': [x[0] for x in EasyBlock.get_steps()],
'validate':
not options.force,
})
# obtain list of loaded modules, build options must be initialized first
modlist = session_module_list()
init_session_state.update({'module_list': modlist})
_log.debug("Initial session state: %s" % init_session_state)
# search for easyconfigs
if options.search or options.search_short:
search_path = [os.getcwd()]
if easyconfigs_paths:
search_path = easyconfigs_paths
query = options.search or options.search_short
ignore_dirs = config.build_option('ignore_dirs')
silent = config.build_option('silent')
search_file(search_path,
query,
short=not options.search,
ignore_dirs=ignore_dirs,
silent=silent)
paths = []
if len(orig_paths) == 0:
if options.from_pr:
pr_path = os.path.join(eb_tmpdir, "files_pr%s" % options.from_pr)
pr_files = fetch_easyconfigs_from_pr(
options.from_pr, path=pr_path, github_user=options.github_user)
paths = [(path, False) for path in pr_files
if path.endswith('.eb')]
elif 'name' in build_specs:
paths = [
obtain_path(build_specs,
easyconfigs_paths,
try_to_generate=try_to_generate,
exit_on_error=not testing)
]
elif not any([
options.aggregate_regtest, options.search,
options.search_short, options.regtest
]):
print_error((
"Please provide one or multiple easyconfig files, or use software build "
"options to make EasyBuild search for easyconfigs"),
log=_log,
opt_parser=eb_go.parser,
exit_on_error=not testing)
else:
# look for easyconfigs with relative paths in easybuild-easyconfigs package,
# unless they were found at the given relative paths
if easyconfigs_pkg_full_paths:
# determine which easyconfigs files need to be found, if any
ecs_to_find = []
for idx, orig_path in enumerate(orig_paths):
if orig_path == os.path.basename(
orig_path) and not os.path.exists(orig_path):
ecs_to_find.append((idx, orig_path))
_log.debug("List of easyconfig files to find: %s" % ecs_to_find)
# find missing easyconfigs by walking paths with installed easyconfig files
for path in easyconfigs_pkg_full_paths:
_log.debug(
"Looking for missing easyconfig files (%d left) in %s..." %
(len(ecs_to_find), path))
for (subpath, dirnames, filenames) in os.walk(path,
topdown=True):
for idx, orig_path in ecs_to_find[:]:
if orig_path in filenames:
full_path = os.path.join(subpath, orig_path)
_log.info("Found %s in %s: %s" %
(orig_path, path, full_path))
orig_paths[idx] = full_path
# if file was found, stop looking for it (first hit wins)
ecs_to_find.remove((idx, orig_path))
# stop os.walk insanity as soon as we have all we need (os.walk loop)
if len(ecs_to_find) == 0:
break
# ignore subdirs specified to be ignored by replacing items in dirnames list used by os.walk
dirnames[:] = [
d for d in dirnames if not d in options.ignore_dirs
]
# stop os.walk insanity as soon as we have all we need (paths loop)
if len(ecs_to_find) == 0:
break
# indicate that specified paths do not contain generated easyconfig files
paths = [(path, False) for path in orig_paths]
_log.debug("Paths: %s" % paths)
# run regtest
if options.regtest or options.aggregate_regtest:
_log.info("Running regression test")
if paths:
ec_paths = [path[0] for path in paths]
else: # fallback: easybuild-easyconfigs install path
ec_paths = easyconfigs_pkg_full_paths
regtest_ok = regtest(ec_paths)
if not regtest_ok:
_log.info("Regression test failed (partially)!")
sys.exit(31) # exit -> 3x1t -> 31
# read easyconfig files
easyconfigs = []
for (path, generated) in paths:
path = os.path.abspath(path)
if not os.path.exists(path):
print_error("Can't find path %s" % path)
try:
ec_files = find_easyconfigs(path, ignore_dirs=options.ignore_dirs)
for ec_file in ec_files:
# only pass build specs when not generating easyconfig files
if try_to_generate:
ecs = process_easyconfig(ec_file)
else:
ecs = process_easyconfig(ec_file, build_specs=build_specs)
easyconfigs.extend(ecs)
except IOError, err:
_log.error("Processing easyconfigs in path %s failed: %s" %
(path, err))
