def test_package(self):
"""Test package function."""
init_config(build_options={'silent': True})
topdir = os.path.dirname(os.path.abspath(__file__))
test_easyconfigs = os.path.join(topdir, 'easyconfigs', 'test_ecs')
ec = EasyConfig(os.path.join(test_easyconfigs, 't', 'toy',
'toy-0.0-gompi-1.3.12-test.eb'),
validate=False)
mock_fpm(self.test_prefix)
# import needs to be done here, since test easyblocks are only included later
from easybuild.easyblocks.toy import EB_toy
easyblock = EB_toy(ec)
# build & install first
easyblock.run_all_steps(False)
# write a dummy log and report file to make sure they don't get packaged
logfile = os.path.join(easyblock.installdir, log_path(), "logfile.log")
write_file(logfile, "I'm a logfile")
reportfile = os.path.join(easyblock.installdir, log_path(),
"report.md")
write_file(reportfile, "I'm a reportfile")
# package using default packaging configuration (FPM to build RPM packages)
pkgdir = package(easyblock)
pkgfile = os.path.join(
pkgdir,
'toy-0.0-gompi-1.3.12-test-eb-%s.1.rpm' % EASYBUILD_VERSION)
self.assertTrue(os.path.isfile(pkgfile), "Found %s" % pkgfile)
pkgtxt = read_file(pkgfile)
pkgtxt_regex = re.compile("STARTCONTENTS of installdir %s" %
easyblock.installdir)
self.assertTrue(
pkgtxt_regex.search(pkgtxt),
"Pattern '%s' found in: %s" % (pkgtxt_regex.pattern, pkgtxt))
no_logfiles_regex = re.compile(
r'STARTCONTENTS.*\.(log|md)$.*ENDCONTENTS',
re.DOTALL | re.MULTILINE)
self.assertFalse(
no_logfiles_regex.search(pkgtxt), "Pattern not '%s' found in: %s" %
(no_logfiles_regex.pattern, pkgtxt))
if DEBUG:
print "The FPM script debug output"
print read_file(os.path.join(self.test_prefix, DEBUG_FPM_FILE))
print "The Package File"
print read_file(pkgfile)
def __init__(self, *args, **kwargs):
"""Constructor for Python easyblock."""
super(EB_Python, self).__init__(*args, **kwargs)
self.pyshortver = '.'.join(self.version.split('.')[:2])
self.pythonpath = None
if self.cfg['ebpythonprefixes']:
easybuild_subdir = log_path()
self.pythonpath = os.path.join(easybuild_subdir, 'python')
ext_defaults = {
# Use PYPI_SOURCE as the default for source_urls of extensions.
'source_urls': [url for name, url, _ in TEMPLATE_CONSTANTS if name == 'PYPI_SOURCE'],
# We should enable this (by default) for all extensions because the only installed packages at this point
# (i.e. those in the site-packages folder) are the default installed ones, e.g. pip & setuptools.
# And we must upgrade them cleanly, i.e. uninstall them first. This also applies to any other package
# which is voluntarily or accidentally installed multiple times.
# Example: Upgrading to a higher version after installing new dependencies.
'pip_ignore_installed': False,
# Python installations must be clean. Requires pip >= 9
'sanity_pip_check': LooseVersion(self._get_pip_ext_version() or '0.0') >= LooseVersion('9.0'),
}
exts_default_options = self.cfg.get_ref('exts_default_options')
for key in ext_defaults:
if key not in exts_default_options:
exts_default_options[key] = ext_defaults[key]
self.log.debug("exts_default_options: %s", self.cfg['exts_default_options'])
self.install_pip = self.cfg['install_pip']
if self.install_pip:
if not self._has_ensure_pip():
raise EasyBuildError("The ensurepip module required to install pip (requested by install_pip=True) "
"is not available in Python %s", self.version)
def test_log_path(self):
"""Test for log_path()."""
# default
self.assertEqual(log_path(), 'easybuild')
# providing template values doesn't affect the default
ec = {'name': 'foo', 'version': '1.2.3'}
res = log_path(ec=ec)
self.assertEqual(res, 'easybuild')
# reconfigure with value for log directory that includes templates
init_config(args=['--logfile-format=easybuild-%(name)s-%(version)s-%(date)s-%(time)s,log.txt'])
regex = re.compile(r'^easybuild-foo-1\.2\.3-[0-9-]{8}-[0-9]{6}$')
res = log_path(ec=ec)
self.assertTrue(regex.match(res), "Pattern '%s' matches '%s'" % (regex.pattern, res))
self.assertEqual(log_file_format(), 'log.txt')
def test_log_path(self):
"""Test for log_path()."""
# default
self.assertEqual(log_path(), 'easybuild')
# providing template values doesn't affect the default
ec = {'name': 'foo', 'version': '1.2.3'}
res = log_path(ec=ec)
self.assertEqual(res, 'easybuild')
# reconfigure with value for log directory that includes templates
init_config(args=['--logfile-format=easybuild-%(name)s-%(version)s-%(date)s-%(time)s,log.txt'])
regex = re.compile(r'^easybuild-foo-1\.2\.3-[0-9-]{8}-[0-9]{6}$')
res = log_path(ec=ec)
self.assertTrue(regex.match(res), "Pattern '%s' matches '%s'" % (regex.pattern, res))
self.assertEqual(log_file_format(), 'log.txt')
def __init__(self, *args, **kwargs):
"""Constructor for Python easyblock."""
super(EB_Python, self).__init__(*args, **kwargs)
self.pyshortver = '.'.join(self.version.split('.')[:2])
self.pythonpath = None
if self.cfg['ebpythonprefixes']:
easybuild_subdir = log_path()
self.pythonpath = os.path.join(easybuild_subdir, 'python')
def __init__(self, *args, **kwargs):
"""Constructor for Python easyblock."""
super(EB_Python, self).__init__(*args, **kwargs)
self.pyshortver = '.'.join(self.version.split('.')[:2])
self.pythonpath = None
if self.cfg['ebpythonprefixes']:
easybuild_subdir = log_path()
self.pythonpath = os.path.join(easybuild_subdir, 'python')
def test_package(self):
"""Test package function."""
init_config(build_options={'silent': True})
topdir = os.path.dirname(os.path.abspath(__file__))
test_easyconfigs = os.path.join(topdir, 'easyconfigs', 'test_ecs')
ec = EasyConfig(os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0-gompi-1.3.12-test.eb'), validate=False)
mock_fpm(self.test_prefix)
# import needs to be done here, since test easyblocks are only included later
from easybuild.easyblocks.toy import EB_toy
easyblock = EB_toy(ec)
# build & install first
easyblock.run_all_steps(False)
# write a dummy log and report file to make sure they don't get packaged
logfile = os.path.join(easyblock.installdir, log_path(), "logfile.log")
write_file(logfile, "I'm a logfile")
reportfile = os.path.join(easyblock.installdir, log_path(), "report.md")
write_file(reportfile, "I'm a reportfile")
# package using default packaging configuration (FPM to build RPM packages)
pkgdir = package(easyblock)
pkgfile = os.path.join(pkgdir, 'toy-0.0-gompi-1.3.12-test-eb-%s.1.rpm' % EASYBUILD_VERSION)
self.assertTrue(os.path.isfile(pkgfile), "Found %s" % pkgfile)
pkgtxt = read_file(pkgfile)
pkgtxt_regex = re.compile("STARTCONTENTS of installdir %s" % easyblock.installdir)
self.assertTrue(pkgtxt_regex.search(pkgtxt), "Pattern '%s' found in: %s" % (pkgtxt_regex.pattern, pkgtxt))
no_logfiles_regex = re.compile(r'STARTCONTENTS.*\.(log|md)$.*ENDCONTENTS', re.DOTALL|re.MULTILINE)
self.assertFalse(no_logfiles_regex.search(pkgtxt), "Pattern not '%s' found in: %s" % (no_logfiles_regex.pattern, pkgtxt))
if DEBUG:
print "The FPM script debug output"
print read_file(os.path.join(self.test_prefix, DEBUG_FPM_FILE))
print "The Package File"
print read_file(pkgfile)
def test_cases_step(self):
"""Run provided list of test cases, or provided examples is no test cases were specified."""
# run all examples if no test cases were specified
# order and grouping is important for some of these tests (e.g., [o]h3tr*
# Some of the examples are deleted
# missing md parameter files: dna.nw, mache.nw, 18c6NaK.nw, membrane.nw, sdm.nw
# method not implemented (unknown thory) or keyword not found: triplet.nw, C2H6.nw, pspw_MgO.nw
# ccsdt_polar_small.nw, CG.nw
# no convergence: diamond.nw
# Too much memory required: ccsd_polar_big.nw
if isinstance(self.cfg['tests'], bool):
examples = [
('qmd', ['3carbo_dft.nw', '3carbo.nw', 'h2o_scf.nw']),
('pspw',
['C2.nw', 'C6.nw', 'Carbene.nw', 'Na16.nw', 'NaCl.nw']),
('tcepolar', ['ccsd_polar_small.nw']),
('dirdyvtst/h3', ['h3tr1.nw', 'h3tr2.nw']),
('dirdyvtst/h3', ['h3tr3.nw']), ('dirdyvtst/h3', ['h3tr4.nw']),
('dirdyvtst/h3', ['h3tr5.nw']),
('dirdyvtst/oh3', ['oh3tr1.nw', 'oh3tr2.nw']),
('dirdyvtst/oh3', ['oh3tr3.nw']),
('dirdyvtst/oh3', ['oh3tr4.nw']),
('dirdyvtst/oh3', ['oh3tr5.nw']),
('pspw/session1', [
'band.nw', 'si4.linear.nw', 'si4.rhombus.nw',
'S2-drift.nw', 'silicon.nw', 'S2.nw', 'si4.rectangle.nw'
]), ('md/myo', ['myo.nw']), ('md/nak', ['NaK.nw']),
('md/crown', ['crown.nw']), ('md/hrc', ['hrc.nw']),
('md/benzene', ['benzene.nw'])
]
self.cfg['tests'] = [(os.path.join(self.examples_dir, d), l)
for (d, l) in examples]
self.log.info("List of examples to be run as test cases: %s" %
self.cfg['tests'])
try:
# symlink $HOME/.nwchemrc to local copy of default nwchemrc
default_nwchemrc = os.path.join(self.installdir, 'data',
'default.nwchemrc')
# make a local copy of the default .nwchemrc file at a fixed path, so we can symlink to it
# this makes sure that multiple parallel builds can reuse the same symlink, even for different builds
# there is apparently no way to point NWChem to a particular config file other that $HOME/.nwchemrc
try:
local_nwchemrc_dir = os.path.dirname(self.local_nwchemrc)
if not os.path.exists(local_nwchemrc_dir):
os.makedirs(local_nwchemrc_dir)
shutil.copy2(default_nwchemrc, self.local_nwchemrc)
# only try to create symlink if it's not there yet
# we've verified earlier that the symlink is what we expect it to be if it's there
if not os.path.islink(self.home_nwchemrc):
symlink(self.local_nwchemrc, self.home_nwchemrc)
except OSError as err:
raise EasyBuildError("Failed to symlink %s to %s: %s",
self.home_nwchemrc, self.local_nwchemrc,
err)
# run tests, keep track of fail ratio
cwd = os.getcwd()
fail = 0.0
tot = 0.0
success_regexp = re.compile(r"Total times\s*cpu:.*wall:.*")
test_cases_logfn = os.path.join(self.installdir, config.log_path(),
'test_cases.log')
test_cases_log = open(test_cases_logfn, "w")
for (testdir, tests) in self.cfg['tests']:
# run test in a temporary dir
tmpdir = tempfile.mkdtemp(prefix='nwchem_test_')
change_dir(tmpdir)
# copy all files in test case dir
for item in os.listdir(testdir):
test_file = os.path.join(testdir, item)
if os.path.isfile(test_file):
self.log.debug("Copying %s to %s" %
(test_file, tmpdir))
shutil.copy2(test_file, tmpdir)
# run tests
for testx in tests:
cmd = "nwchem %s" % testx
msg = "Running test '%s' (from %s) in %s..." % (
cmd, testdir, tmpdir)
self.log.info(msg)
test_cases_log.write("\n%s\n" % msg)
(out, ec) = run_cmd(cmd,
simple=False,
log_all=False,
log_ok=False,
log_output=True)
# check exit code and output
if ec:
msg = "Test %s failed (exit code: %s)!" % (testx, ec)
self.log.warning(msg)
test_cases_log.write('FAIL: %s' % msg)
fail += 1
else:
if success_regexp.search(out):
msg = "Test %s successful!" % testx
self.log.info(msg)
test_cases_log.write('SUCCESS: %s' % msg)
else:
msg = "No 'Total times' found for test %s (but exit code is %s)!" % (
testx, ec)
self.log.warning(msg)
test_cases_log.write('FAIL: %s' % msg)
fail += 1
test_cases_log.write("\nOUTPUT:\n\n%s\n\n" % out)
tot += 1
# go back
change_dir(cwd)
shutil.rmtree(tmpdir)
fail_ratio = fail / tot
fail_pcnt = fail_ratio * 100
msg = "%d of %d tests failed (%s%%)!" % (fail, tot, fail_pcnt)
self.log.info(msg)
test_cases_log.write('\n\nSUMMARY: %s' % msg)
test_cases_log.close()
self.log.info("Log for test cases saved at %s" % test_cases_logfn)
if fail_ratio > self.cfg['max_fail_ratio']:
max_fail_pcnt = self.cfg['max_fail_ratio'] * 100
raise EasyBuildError(
"Over %s%% of test cases failed, assuming broken build.",
max_fail_pcnt)
# cleanup
try:
shutil.rmtree(self.examples_dir)
shutil.rmtree(local_nwchemrc_dir)
except OSError as err:
raise EasyBuildError("Cleanup failed: %s", err)
# set post msg w.r.t. cleaning up $HOME/.nwchemrc symlink
self.postmsg += "\nRemember to clean up %s after all NWChem builds are finished." % self.home_nwchemrc
except OSError as err:
raise EasyBuildError("Failed to run test cases: %s", err)
def test_cases_step(self):
"""Run provided list of test cases, or provided examples is no test cases were specified."""
# run all examples if no test cases were specified
# order and grouping is important for some of these tests (e.g., [o]h3tr*
# Some of the examples are deleted
# missing md parameter files: dna.nw, mache.nw, 18c6NaK.nw, membrane.nw, sdm.nw
# method not implemented (unknown thory) or keyword not found: triplet.nw, C2H6.nw, pspw_MgO.nw, ccsdt_polar_small.nw, CG.nw
# no convergence: diamond.nw
# Too much memory required: ccsd_polar_big.nw
if type(self.cfg['tests']) is bool:
examples = [('qmd', ['3carbo_dft.nw', '3carbo.nw', 'h2o_scf.nw']),
('pspw', ['C2.nw', 'C6.nw', 'Carbene.nw', 'Na16.nw', 'NaCl.nw']),
('tcepolar', ['ccsd_polar_small.nw']),
('dirdyvtst/h3', ['h3tr1.nw', 'h3tr2.nw']),
('dirdyvtst/h3', ['h3tr3.nw']), ('dirdyvtst/h3', ['h3tr4.nw']), ('dirdyvtst/h3', ['h3tr5.nw']),
('dirdyvtst/oh3', ['oh3tr1.nw', 'oh3tr2.nw']),
('dirdyvtst/oh3', ['oh3tr3.nw']), ('dirdyvtst/oh3', ['oh3tr4.nw']), ('dirdyvtst/oh3', ['oh3tr5.nw']),
('pspw/session1', ['band.nw', 'si4.linear.nw', 'si4.rhombus.nw', 'S2-drift.nw',
'silicon.nw', 'S2.nw', 'si4.rectangle.nw']),
('md/myo', ['myo.nw']), ('md/nak', ['NaK.nw']), ('md/crown', ['crown.nw']), ('md/hrc', ['hrc.nw']),
('md/benzene', ['benzene.nw'])]
self.cfg['tests'] = [(os.path.join(self.examples_dir, d), l) for (d, l) in examples]
self.log.info("List of examples to be run as test cases: %s" % self.cfg['tests'])
try:
# symlink $HOME/.nwchemrc to local copy of default nwchemrc
default_nwchemrc = os.path.join(self.installdir, 'data', 'default.nwchemrc')
# make a local copy of the default .nwchemrc file at a fixed path, so we can symlink to it
# this makes sure that multiple parallel builds can reuse the same symlink, even for different builds
# there is apparently no way to point NWChem to a particular config file other that $HOME/.nwchemrc
try:
local_nwchemrc_dir = os.path.dirname(self.local_nwchemrc)
if not os.path.exists(local_nwchemrc_dir):
os.makedirs(local_nwchemrc_dir)
shutil.copy2(default_nwchemrc, self.local_nwchemrc)
# only try to create symlink if it's not there yet
# we've verified earlier that the symlink is what we expect it to be if it's there
if not os.path.exists(self.home_nwchemrc):
os.symlink(self.local_nwchemrc, self.home_nwchemrc)
except OSError, err:
raise EasyBuildError("Failed to symlink %s to %s: %s", self.home_nwchemrc, self.local_nwchemrc, err)
# run tests, keep track of fail ratio
cwd = os.getcwd()
fail = 0.0
tot = 0.0
success_regexp = re.compile("Total times\s*cpu:.*wall:.*")
test_cases_logfn = os.path.join(self.installdir, config.log_path(), 'test_cases.log')
test_cases_log = open(test_cases_logfn, "w")
for (testdir, tests) in self.cfg['tests']:
# run test in a temporary dir
tmpdir = tempfile.mkdtemp(prefix='nwchem_test_')
os.chdir(tmpdir)
# copy all files in test case dir
for item in os.listdir(testdir):
test_file = os.path.join(testdir, item)
if os.path.isfile(test_file):
self.log.debug("Copying %s to %s" % (test_file, tmpdir))
shutil.copy2(test_file, tmpdir)
# run tests
for testx in tests:
cmd = "nwchem %s" % testx
msg = "Running test '%s' (from %s) in %s..." % (cmd, testdir, tmpdir)
self.log.info(msg)
test_cases_log.write("\n%s\n" % msg)
(out, ec) = run_cmd(cmd, simple=False, log_all=False, log_ok=False, log_output=True)
# check exit code and output
if ec:
msg = "Test %s failed (exit code: %s)!" % (testx, ec)
self.log.warning(msg)
test_cases_log.write('FAIL: %s' % msg)
fail += 1
else:
if success_regexp.search(out):
msg = "Test %s successful!" % testx
self.log.info(msg)
test_cases_log.write('SUCCESS: %s' % msg)
else:
msg = "No 'Total times' found for test %s (but exit code is %s)!" % (testx, ec)
self.log.warning(msg)
test_cases_log.write('FAIL: %s' % msg)
fail += 1
test_cases_log.write("\nOUTPUT:\n\n%s\n\n" % out)
tot += 1
# go back
os.chdir(cwd)
shutil.rmtree(tmpdir)
fail_ratio = fail / tot
fail_pcnt = fail_ratio * 100
msg = "%d of %d tests failed (%s%%)!" % (fail, tot, fail_pcnt)
self.log.info(msg)
test_cases_log.write('\n\nSUMMARY: %s' % msg)
test_cases_log.close()
self.log.info("Log for test cases saved at %s" % test_cases_logfn)
if fail_ratio > self.cfg['max_fail_ratio']:
max_fail_pcnt = self.cfg['max_fail_ratio'] * 100
raise EasyBuildError("Over %s%% of test cases failed, assuming broken build.", max_fail_pcnt)
# cleanup
try:
shutil.rmtree(self.examples_dir)
shutil.rmtree(local_nwchemrc_dir)
except OSError, err:
raise EasyBuildError("Cleanup failed: %s", err)
def package_with_fpm(easyblock):
"""
This function will build a package using fpm and return the directory where the packages are
"""
workdir = tempfile.mkdtemp(prefix='eb-pkgs-')
pkgtype = build_option('package_type')
_log.info("Will be creating %s package(s) in %s", pkgtype, workdir)
origdir = change_dir(workdir)
package_naming_scheme = ActivePNS()
pkgname = package_naming_scheme.name(easyblock.cfg)
pkgver = package_naming_scheme.version(easyblock.cfg)
pkgrel = package_naming_scheme.release(easyblock.cfg)
_log.debug("Got the PNS values name: %s version: %s release: %s", pkgname,
pkgver, pkgrel)
cmdlist = [
PKG_TOOL_FPM,
'--workdir',
workdir,
'--name',
pkgname,
'--provides',
pkgname,
'-t',
pkgtype, # target
'-s',
'dir', # source
'--version',
pkgver,
'--iteration',
pkgrel,
'--description',
easyblock.cfg["description"],
'--url',
easyblock.cfg["homepage"],
]
extra_pkg_options = build_option('package_tool_options')
if extra_pkg_options:
cmdlist.extend(extra_pkg_options.split(' '))
if build_option('debug'):
cmdlist.append('--debug')
deps = []
if not easyblock.toolchain.is_system_toolchain():
toolchain_dict = easyblock.toolchain.as_dict()
deps.extend([toolchain_dict])
deps.extend(easyblock.cfg.dependencies())
_log.debug(
"The dependencies to be added to the package are: %s",
pprint.pformat([easyblock.toolchain.as_dict()] +
easyblock.cfg.dependencies()))
for dep in deps:
if dep.get('external_module', False):
_log.debug("Skipping dep marked as external module: %s",
dep['name'])
else:
_log.debug("The dep added looks like %s ", dep)
dep_pkgname = package_naming_scheme.name(dep)
cmdlist.extend(["--depends", dep_pkgname])
# Excluding the EasyBuild logs and test reports that might be in the installdir
exclude_files_globs = [
os.path.join(log_path(ec=easyblock.cfg), "*.log"),
os.path.join(log_path(ec=easyblock.cfg), "*.md"),
]
# stripping off leading / to match expected glob in fpm
for exclude_files_glob in exclude_files_globs:
cmdlist.extend([
'--exclude',
os.path.join(easyblock.installdir.lstrip(os.sep),
exclude_files_glob)
])
cmdlist.extend([
easyblock.installdir,
easyblock.module_generator.get_module_filepath(),
])
cmd = ' '.join(cmdlist)
_log.debug("The flattened cmdlist looks like: %s", cmd)
run_cmd(cmdlist, log_all=True, simple=True, shell=False)
_log.info("Created %s package(s) in %s", pkgtype, workdir)
change_dir(origdir)
return workdir
def package_with_fpm(easyblock):
"""
This function will build a package using fpm and return the directory where the packages are
"""
workdir = tempfile.mkdtemp(prefix='eb-pkgs-')
pkgtype = build_option('package_type')
_log.info("Will be creating %s package(s) in %s", pkgtype, workdir)
origdir = change_dir(workdir)
package_naming_scheme = ActivePNS()
pkgname = package_naming_scheme.name(easyblock.cfg)
pkgver = package_naming_scheme.version(easyblock.cfg)
pkgrel = package_naming_scheme.release(easyblock.cfg)
_log.debug("Got the PNS values name: %s version: %s release: %s", pkgname, pkgver, pkgrel)
cmdlist = [
PKG_TOOL_FPM,
'--workdir', workdir,
'--name', pkgname,
'--provides', pkgname,
'-t', pkgtype, # target
'-s', 'dir', # source
'--version', pkgver,
'--iteration', pkgrel,
'--description', easyblock.cfg["description"],
'--url', easyblock.cfg["homepage"],
]
extra_pkg_options = build_option('package_tool_options')
if extra_pkg_options:
cmdlist.extend(extra_pkg_options.split(' '))
if build_option('debug'):
cmdlist.append('--debug')
deps = []
if easyblock.toolchain.name != DUMMY_TOOLCHAIN_NAME:
toolchain_dict = easyblock.toolchain.as_dict()
deps.extend([toolchain_dict])
deps.extend(easyblock.cfg.dependencies())
_log.debug("The dependencies to be added to the package are: %s",
pprint.pformat([easyblock.toolchain.as_dict()] + easyblock.cfg.dependencies()))
for dep in deps:
if dep.get('external_module', False):
_log.debug("Skipping dep marked as external module: %s", dep['name'])
else:
_log.debug("The dep added looks like %s ", dep)
dep_pkgname = package_naming_scheme.name(dep)
cmdlist.extend(["--depends", dep_pkgname])
# Excluding the EasyBuild logs and test reports that might be in the installdir
exclude_files_globs = [
os.path.join(log_path(), "*.log"),
os.path.join(log_path(), "*.md"),
]
# stripping off leading / to match expected glob in fpm
for exclude_files_glob in exclude_files_globs:
cmdlist.extend(['--exclude', os.path.join(easyblock.installdir.lstrip(os.sep), exclude_files_glob)])
cmdlist.extend([
easyblock.installdir,
easyblock.module_generator.get_module_filepath(),
])
cmd = ' '.join(cmdlist)
_log.debug("The flattened cmdlist looks like: %s", cmd)
run_cmd(cmdlist, log_all=True, simple=True, shell=False)
_log.info("Created %s package(s) in %s", pkgtype, workdir)
change_dir(origdir)
return workdir
ended = "ended"
# successful build
if result:
# collect build stats
_log.info("Collecting build stats...")
currentbuildstats = app.cfg['buildstats']
buildstats = get_build_stats(app, starttime)
_log.debug("Build stats: %s" % buildstats)
if app.cfg['stop']:
ended = "STOPPED"
newLogDir = os.path.join(app.builddir, config.log_path())
else:
newLogDir = os.path.join(app.installdir, config.log_path())
try:
# upload spec to central repository
repo = init_repository(get_repository(), get_repositorypath())
if 'originalSpec' in module:
repo.add_easyconfig(module['originalSpec'], app.name, app.get_installversion() + ".block", buildstats, currentbuildstats)
repo.add_easyconfig(spec, app.name, app.get_installversion(), buildstats, currentbuildstats)
repo.commit("Built %s/%s" % (app.name, app.get_installversion()))
del repo
except EasyBuildError, err:
_log.warn("Unable to commit easyconfig to repository: %s", err)
exitCode = 0
def test_package(self):
"""Test package function."""
build_options = {
'package_tool_options': '--foo bar',
'silent': True,
}
init_config(build_options=build_options)
topdir = os.path.dirname(os.path.abspath(__file__))
test_easyconfigs = os.path.join(topdir, 'easyconfigs', 'test_ecs')
ec = EasyConfig(os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb'), validate=False)
mock_fpm(self.test_prefix)
# import needs to be done here, since test easyblocks are only included later
from easybuild.easyblocks.toy import EB_toy
easyblock = EB_toy(ec)
# build & install first
easyblock.run_all_steps(False)
# write a dummy log and report file to make sure they don't get packaged
logfile = os.path.join(easyblock.installdir, log_path(), "logfile.log")
write_file(logfile, "I'm a logfile")
reportfile = os.path.join(easyblock.installdir, log_path(), "report.md")
write_file(reportfile, "I'm a reportfile")
# package using default packaging configuration (FPM to build RPM packages)
pkgdir = package(easyblock)
pkgfile = os.path.join(pkgdir, 'toy-0.0-gompi-2018a-test-eb-%s.1.rpm' % EASYBUILD_VERSION)
fpm_output = read_file(os.path.join(self.test_prefix, FPM_OUTPUT_FILE))
pkgtxt = read_file(pkgfile)
self.assertTrue(os.path.isfile(pkgfile), "Found %s" % pkgfile)
# check whether extra packaging options were passed down
regex = re.compile("^got an unhandled option: --foo bar$", re.M)
self.assertTrue(regex.search(fpm_output), "Pattern '%s' found in: %s" % (regex.pattern, fpm_output))
pkgtxt = read_file(pkgfile)
pkgtxt_regex = re.compile("STARTCONTENTS of installdir %s" % easyblock.installdir)
self.assertTrue(pkgtxt_regex.search(pkgtxt), "Pattern '%s' found in: %s" % (pkgtxt_regex.pattern, pkgtxt))
no_logfiles_regex = re.compile(r'STARTCONTENTS.*\.(log|md)$.*ENDCONTENTS', re.DOTALL | re.MULTILINE)
res = no_logfiles_regex.search(pkgtxt)
self.assertFalse(res, "Pattern not '%s' found in: %s" % (no_logfiles_regex.pattern, pkgtxt))
toy_txt = read_file(os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb'))
replace_str = '''description = """Toy C program, 100% toy. Now with `backticks'\n'''
replace_str += '''and newlines"""'''
toy_txt = re.sub('description = .*', replace_str, toy_txt)
toy_file = os.path.join(self.test_prefix, 'toy-test-description.eb')
write_file(toy_file, toy_txt)
regex = re.compile(r"""`backticks'""")
self.assertTrue(regex.search(toy_txt), "Pattern '%s' found in: %s" % (regex.pattern, toy_txt))
ec_desc = EasyConfig(toy_file, validate=False)
easyblock_desc = EB_toy(ec_desc)
easyblock_desc.run_all_steps(False)
pkgdir = package(easyblock_desc)
pkgfile = os.path.join(pkgdir, 'toy-0.0-gompi-2018a-test-eb-%s.1.rpm' % EASYBUILD_VERSION)
self.assertTrue(os.path.isfile(pkgfile))
pkgtxt = read_file(pkgfile)
regex_pkg = re.compile(r"""DESCRIPTION:.*`backticks'.*""")
self.assertTrue(regex_pkg.search(pkgtxt), "Pattern '%s' not found in: %s" % (regex_pkg.pattern, pkgtxt))
regex_pkg = re.compile(r"""DESCRIPTION:.*\nand newlines""", re.MULTILINE)
self.assertTrue(regex_pkg.search(pkgtxt), "Pattern '%s' not found in: %s" % (regex_pkg.pattern, pkgtxt))
def test_cases_step(self):
"""Run provided list of test cases, or provided examples is no test cases were specified."""
# run all examples if no test cases were specified
# order and grouping is important for some of these tests (e.g., [o]h3tr*
if type(self.cfg['tests']) is bool:
examples = [('qmd', ['3carbo_dft.nw', '3carbo.nw', 'h2o_scf.nw']),
('pspw', ['C2.nw', 'C6.nw', 'CG.nw', 'Carbene.nw', 'Na16.nw', 'NaCl.nw']),
('tcepolar', ['ccsdt_polar_small.nw', 'ccsd_polar_big.nw', 'ccsd_polar_small.nw']),
('dirdyvtst/h3', ['h3tr1.nw', 'h3tr2.nw', 'h3tr3.nw', 'h3tr4.nw', 'h3tr5.nw']),
('dirdyvtst/oh3', ['oh3tr1.nw', 'oh3tr2.nw', 'oh3tr3.nw', 'oh3tr4.nw', 'oh3tr5.nw']),
('pspw/session1', ['band.nw', 'si4.linear.nw', 'si4.rhombus.nw', 'S2-drift.nw', 'diamond.nw',
'silicon.nw', 'S2.nw', 'si4.rectangle.nw']),
('pspw/MgO+Cu', ['pspw_MgO.nw']), ('pspw/C2H6', ['C2H6.nw']), ('pspw/Carbene', ['triplet.nw']),
('md/dna', ['dna.nw']), ('md/ache', ['mache.nw']), ('md/myo', ['myo.nw']),
('md/nak', ['NaK.nw']), ('md/nak', ['18c6NaK.nw']), ('md/membrane', ['membrane.nw']),
('md/sdm', ['sdm.nw']), ('md/crown', ['crown.nw']), ('md/hrc', ['hrc.nw']),
('md/benzene', ['benzene.nw'])]
self.cfg['tests'] = [(os.path.join(self.examples_dir, d), l) for (d, l) in examples]
self.log.info("List of examples to be run as test cases: %s" % self.cfg['tests'])
try:
# symlink $HOME/.nwchemrc to local copy of default nwchemrc
default_nwchemrc = os.path.join(self.installdir, 'data', 'default.nwchemrc')
home_nwchemrc = os.path.join(os.getenv('HOME'), '.nwchemrc')
local_nwchemrc = os.path.join(tempfile.gettempdir(), os.getenv('USER'), 'easybuild_nwchem', '.nwchemrc')
local_nwchemrc_dir = os.path.dirname(local_nwchemrc)
if not os.path.exists(local_nwchemrc_dir):
os.makedirs(local_nwchemrc_dir)
shutil.copy2(default_nwchemrc, local_nwchemrc)
symlink_ok = False
if os.path.exists(home_nwchemrc):
if not os.path.islink(home_nwchemrc) or not os.path.samefile(home_nwchemrc, local_nwchemrc):
self.log.error("File %s is present, but is not a symlink to %s" % (home_nwchemrc, local_nwchemrc))
else:
symlink_ok = True
if not symlink_ok:
os.symlink(local_nwchemrc, home_nwchemrc)
# run tests, keep track of fail ratio
cwd = os.getcwd()
fail = 0.0
tot = 0.0
success_regexp = re.compile("Total times\s*cpu:.*wall:.*")
test_cases_logfn = os.path.join(self.installdir, config.log_path(), 'test_cases.log')
test_cases_log = open(test_cases_logfn, "w")
for (testdir, tests) in self.cfg['tests']:
# run test in a temporary dir
tmpdir = tempfile.mkdtemp(prefix='nwchem_test_')
os.chdir(tmpdir)
# copy all files in test case dir
for item in os.listdir(testdir):
test_file = os.path.join(testdir, item)
if os.path.isfile(test_file):
self.log.debug("Copying %s to %s" % (test_file, tmpdir))
shutil.copy2(test_file, tmpdir)
# run tests
for testx in tests:
cmd = "nwchem %s" % testx
msg = "Running test '%s' (from %s) in %s..." % (cmd, testdir, tmpdir)
self.log.info(msg)
test_cases_log.write("\n%s\n" % msg)
(out, ec) = run_cmd(cmd, simple=False, log_all=False, log_ok=False, log_output=True)
# check exit code and output
if ec:
msg = "Test %s failed (exit code: %s)!" % (testx, ec)
self.log.warning(msg)
test_cases_log.write('FAIL: %s' % msg)
fail += 1
else:
if success_regexp.search(out):
msg = "Test %s successful!" % testx
self.log.info(msg)
test_cases_log.write('SUCCESS: %s' % msg)
else:
msg = "No 'Total times' found for test %s (but exit code is %s)!" % (testx, ec)
self.log.warning(msg)
test_cases_log.write('FAIL: %s' % msg)
fail += 1
test_cases_log.write("\nOUTPUT:\n\n%s\n\n" % out)
tot += 1
# go back
os.chdir(cwd)
shutil.rmtree(tmpdir)
fail_ratio = fail / tot
fail_pcnt = fail_ratio * 100
msg = "%d of %d tests failed (%s%%)!" % (fail, tot, fail_pcnt)
self.log.info(msg)
test_cases_log.write('\n\nSUMMARY: %s' % msg)
test_cases_log.close()
self.log.info("Log for test cases saved at %s" % test_cases_logfn)
if fail_ratio > self.cfg['max_fail_ratio']:
max_fail_pcnt = self.cfg['max_fail_ratio'] * 100
self.log.error("Over %s%% of test cases failed, assuming broken build." % max_fail_pcnt)
shutil.rmtree(self.examples_dir)
shutil.rmtree(local_nwchemrc_dir)
except OSError, err:
self.log.error("Failed to run test cases: %s" % err)
ended = "ended"
# successful build
if result:
# collect build stats
_log.info("Collecting build stats...")
currentbuildstats = app.cfg["buildstats"]
buildstats = get_build_stats(app, starttime)
_log.debug("Build stats: %s" % buildstats)
if app.cfg["stop"]:
ended = "STOPPED"
newLogDir = os.path.join(app.builddir, config.log_path())
else:
newLogDir = os.path.join(app.installdir, config.log_path())
try:
# upload spec to central repository
repo = init_repository(get_repository(), get_repositorypath())
if "originalSpec" in module:
repo.add_easyconfig(
module["originalSpec"],
app.name,
app.get_installversion() + ".block",
buildstats,
currentbuildstats,
)
repo.add_easyconfig(spec, app.name, app.get_installversion(), buildstats, currentbuildstats)
deps.extend(easyblock.cfg.dependencies())
_log.debug("The dependencies to be added to the package are: %s",
pprint.pformat([easyblock.toolchain.as_dict()] + easyblock.cfg.dependencies()))
depstring = ''
for dep in deps:
if dep.get('external_module', False):
_log.debug("Skipping dep marked as external module: %s", dep['name'])
else:
_log.debug("The dep added looks like %s ", dep)
dep_pkgname = package_naming_scheme.name(dep)
depstring += " --depends %s" % quote_str(dep_pkgname)
# Excluding the EasyBuild logs and test reports that might be in the installdir
exclude_files_glob = [
os.path.join(log_path(), "*.log"),
os.path.join(log_path(), "*.md"),
]
# stripping off leading / to match expected glob in fpm
exclude_files_glob = [
'--exclude %s' % quote_str(os.path.join(easyblock.installdir.lstrip(os.sep), x))
for x in exclude_files_glob
]
_log.debug("The list of excluded files passed to fpm: %s", exclude_files_glob)
cmdlist = [
PKG_TOOL_FPM,
'--workdir', workdir,
'--name', quote_str(pkgname),
'--provides', quote_str(pkgname),
'-t', pkgtype, # target
'-s', 'dir', # source
def test_cases_step(self):
"""Run provided list of test cases, or provided examples is no test cases were specified."""
# run all examples if no test cases were specified
# order and grouping is important for some of these tests (e.g., [o]h3tr*
if type(self.cfg['tests']) is bool:
examples = [
('qmd', ['3carbo_dft.nw', '3carbo.nw', 'h2o_scf.nw']),
('pspw', [
'C2.nw', 'C6.nw', 'CG.nw', 'Carbene.nw', 'Na16.nw',
'NaCl.nw'
]),
('tcepolar', [
'ccsdt_polar_small.nw', 'ccsd_polar_big.nw',
'ccsd_polar_small.nw'
]),
('dirdyvtst/h3',
['h3tr1.nw', 'h3tr2.nw', 'h3tr3.nw', 'h3tr4.nw', 'h3tr5.nw']),
('dirdyvtst/oh3', [
'oh3tr1.nw', 'oh3tr2.nw', 'oh3tr3.nw', 'oh3tr4.nw',
'oh3tr5.nw'
]),
('pspw/session1', [
'band.nw', 'si4.linear.nw', 'si4.rhombus.nw',
'S2-drift.nw', 'diamond.nw', 'silicon.nw', 'S2.nw',
'si4.rectangle.nw'
]), ('pspw/MgO+Cu', ['pspw_MgO.nw']),
('pspw/C2H6', ['C2H6.nw']), ('pspw/Carbene', ['triplet.nw']),
('md/dna', ['dna.nw']), ('md/ache', ['mache.nw']),
('md/myo', ['myo.nw']), ('md/nak', ['NaK.nw']),
('md/nak', ['18c6NaK.nw']), ('md/membrane', ['membrane.nw']),
('md/sdm', ['sdm.nw']), ('md/crown', ['crown.nw']),
('md/hrc', ['hrc.nw']), ('md/benzene', ['benzene.nw'])
]
self.cfg['tests'] = [(os.path.join(self.examples_dir, d), l)
for (d, l) in examples]
self.log.info("List of examples to be run as test cases: %s" %
self.cfg['tests'])
try:
# symlink $HOME/.nwchemrc to local copy of default nwchemrc
default_nwchemrc = os.path.join(self.installdir, 'data',
'default.nwchemrc')
home_nwchemrc = os.path.join(os.getenv('HOME'), '.nwchemrc')
local_nwchemrc = os.path.join(tempfile.gettempdir(),
os.getenv('USER'),
'easybuild_nwchem', '.nwchemrc')
local_nwchemrc_dir = os.path.dirname(local_nwchemrc)
if not os.path.exists(local_nwchemrc_dir):
os.makedirs(local_nwchemrc_dir)
shutil.copy2(default_nwchemrc, local_nwchemrc)
symlink_ok = False
if os.path.exists(home_nwchemrc):
if not os.path.islink(home_nwchemrc) or not os.path.samefile(
home_nwchemrc, local_nwchemrc):
self.log.error(
"File %s is present, but is not a symlink to %s" %
(home_nwchemrc, local_nwchemrc))
else:
symlink_ok = True
if not symlink_ok:
os.symlink(local_nwchemrc, home_nwchemrc)
# run tests, keep track of fail ratio
cwd = os.getcwd()
fail = 0.0
tot = 0.0
success_regexp = re.compile("Total times\s*cpu:.*wall:.*")
test_cases_logfn = os.path.join(self.installdir, config.log_path(),
'test_cases.log')
test_cases_log = open(test_cases_logfn, "w")
for (testdir, tests) in self.cfg['tests']:
# run test in a temporary dir
tmpdir = tempfile.mkdtemp(prefix='nwchem_test_')
os.chdir(tmpdir)
# copy all files in test case dir
for item in os.listdir(testdir):
test_file = os.path.join(testdir, item)
if os.path.isfile(test_file):
self.log.debug("Copying %s to %s" %
(test_file, tmpdir))
shutil.copy2(test_file, tmpdir)
# run tests
for testx in tests:
cmd = "nwchem %s" % testx
msg = "Running test '%s' (from %s) in %s..." % (
cmd, testdir, tmpdir)
self.log.info(msg)
test_cases_log.write("\n%s\n" % msg)
(out, ec) = run_cmd(cmd,
simple=False,
log_all=False,
log_ok=False,
log_output=True)
# check exit code and output
if ec:
msg = "Test %s failed (exit code: %s)!" % (testx, ec)
self.log.warning(msg)
test_cases_log.write('FAIL: %s' % msg)
fail += 1
else:
if success_regexp.search(out):
msg = "Test %s successful!" % testx
self.log.info(msg)
test_cases_log.write('SUCCESS: %s' % msg)
else:
msg = "No 'Total times' found for test %s (but exit code is %s)!" % (
testx, ec)
self.log.warning(msg)
test_cases_log.write('FAIL: %s' % msg)
fail += 1
test_cases_log.write("\nOUTPUT:\n\n%s\n\n" % out)
tot += 1
# go back
os.chdir(cwd)
shutil.rmtree(tmpdir)
fail_ratio = fail / tot
fail_pcnt = fail_ratio * 100
msg = "%d of %d tests failed (%s%%)!" % (fail, tot, fail_pcnt)
self.log.info(msg)
test_cases_log.write('\n\nSUMMARY: %s' % msg)
test_cases_log.close()
self.log.info("Log for test cases saved at %s" % test_cases_logfn)
if fail_ratio > self.cfg['max_fail_ratio']:
max_fail_pcnt = self.cfg['max_fail_ratio'] * 100
self.log.error(
"Over %s%% of test cases failed, assuming broken build." %
max_fail_pcnt)
shutil.rmtree(self.examples_dir)
shutil.rmtree(local_nwchemrc_dir)
except OSError, err:
self.log.error("Failed to run test cases: %s" % err)
"The dependencies to be added to the package are: %s",
pprint.pformat([easyblock.toolchain.as_dict()] +
easyblock.cfg.dependencies()))
depstring = ''
for dep in deps:
if dep.get('external_module', False):
_log.debug("Skipping dep marked as external module: %s",
dep['name'])
else:
_log.debug("The dep added looks like %s ", dep)
dep_pkgname = package_naming_scheme.name(dep)
depstring += " --depends %s" % quote_str(dep_pkgname)
# Excluding the EasyBuild logs and test reports that might be in the installdir
exclude_files_glob = [
os.path.join(log_path(), "*.log"),
os.path.join(log_path(), "*.md"),
]
# stripping off leading / to match expected glob in fpm
exclude_files_glob = [
'--exclude %s' %
quote_str(os.path.join(easyblock.installdir.lstrip(os.sep), x))
for x in exclude_files_glob
]
_log.debug("The list of excluded files passed to fpm: %s",
exclude_files_glob)
cmdlist = [
PKG_TOOL_FPM,
'--workdir',
workdir,
'--name',
def test_package(self):
"""Test package function."""
build_options = {
'package_tool_options': '--foo bar',
'silent': True,
}
init_config(build_options=build_options)
topdir = os.path.dirname(os.path.abspath(__file__))
test_easyconfigs = os.path.join(topdir, 'easyconfigs', 'test_ecs')
ec = EasyConfig(os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb'), validate=False)
mock_fpm(self.test_prefix)
# import needs to be done here, since test easyblocks are only included later
from easybuild.easyblocks.toy import EB_toy
easyblock = EB_toy(ec)
# build & install first
easyblock.run_all_steps(False)
# write a dummy log and report file to make sure they don't get packaged
logfile = os.path.join(easyblock.installdir, log_path(), "logfile.log")
write_file(logfile, "I'm a logfile")
reportfile = os.path.join(easyblock.installdir, log_path(), "report.md")
write_file(reportfile, "I'm a reportfile")
# package using default packaging configuration (FPM to build RPM packages)
pkgdir = package(easyblock)
pkgfile = os.path.join(pkgdir, 'toy-0.0-gompi-2018a-test-eb-%s.1.rpm' % EASYBUILD_VERSION)
fpm_output = read_file(os.path.join(self.test_prefix, FPM_OUTPUT_FILE))
pkgtxt = read_file(pkgfile)
self.assertTrue(os.path.isfile(pkgfile), "Found %s" % pkgfile)
# check whether extra packaging options were passed down
regex = re.compile("^got an unhandled option: --foo bar$", re.M)
self.assertTrue(regex.search(fpm_output), "Pattern '%s' found in: %s" % (regex.pattern, fpm_output))
pkgtxt = read_file(pkgfile)
pkgtxt_regex = re.compile("STARTCONTENTS of installdir %s" % easyblock.installdir)
self.assertTrue(pkgtxt_regex.search(pkgtxt), "Pattern '%s' found in: %s" % (pkgtxt_regex.pattern, pkgtxt))
no_logfiles_regex = re.compile(r'STARTCONTENTS.*\.(log|md)$.*ENDCONTENTS', re.DOTALL | re.MULTILINE)
res = no_logfiles_regex.search(pkgtxt)
self.assertFalse(res, "Pattern not '%s' found in: %s" % (no_logfiles_regex.pattern, pkgtxt))
toy_txt = read_file(os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb'))
replace_str = '''description = """Toy C program, 100% toy. Now with `backticks'\n'''
replace_str += '''and newlines"""'''
toy_txt = re.sub('description = .*', replace_str, toy_txt)
toy_file = os.path.join(self.test_prefix, 'toy-test-description.eb')
write_file(toy_file, toy_txt)
regex = re.compile(r"""`backticks'""")
self.assertTrue(regex.search(toy_txt), "Pattern '%s' found in: %s" % (regex.pattern, toy_txt))
ec_desc = EasyConfig(toy_file, validate=False)
easyblock_desc = EB_toy(ec_desc)
easyblock_desc.run_all_steps(False)
pkgdir = package(easyblock_desc)
pkgfile = os.path.join(pkgdir, 'toy-0.0-gompi-2018a-test-eb-%s.1.rpm' % EASYBUILD_VERSION)
self.assertTrue(os.path.isfile(pkgfile))
pkgtxt = read_file(pkgfile)
regex_pkg = re.compile(r"""DESCRIPTION:.*`backticks'.*""")
self.assertTrue(regex_pkg.search(pkgtxt), "Pattern '%s' not found in: %s" % (regex_pkg.pattern, pkgtxt))
regex_pkg = re.compile(r"""DESCRIPTION:.*\nand newlines""", re.MULTILINE)
self.assertTrue(regex_pkg.search(pkgtxt), "Pattern '%s' not found in: %s" % (regex_pkg.pattern, pkgtxt))
