def test_build_easyconfigs_in_parallel_gc3pie(self):
"""Test build_easyconfigs_in_parallel(), using GC3Pie with local config as backend for --job."""
try:
import gc3libs # noqa (ignore unused import)
except ImportError:
print "GC3Pie not available, skipping test"
return
# put GC3Pie config in place to use local host and fork/exec
resourcedir = os.path.join(self.test_prefix, 'gc3pie')
gc3pie_cfgfile = os.path.join(self.test_prefix, 'gc3pie_local.ini')
gc3pie_cfgtxt = GC3PIE_LOCAL_CONFIGURATION % {
'resourcedir': resourcedir,
'time': which('time'),
}
write_file(gc3pie_cfgfile, gc3pie_cfgtxt)
output_dir = os.path.join(self.test_prefix, 'subdir', 'gc3pie_output_dir')
# purposely pre-create output dir, and put a file in it (to check whether GC3Pie tries to rename the output dir)
mkdir(output_dir, parents=True)
write_file(os.path.join(output_dir, 'foo'), 'bar')
# remove write permissions on parent dir of specified output dir,
# to check that GC3Pie does not try to rename the (already existing) output directory...
adjust_permissions(os.path.dirname(output_dir), stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH,
add=False, recursive=False)
topdir = os.path.dirname(os.path.abspath(__file__))
build_options = {
'job_backend_config': gc3pie_cfgfile,
'job_max_walltime': 24,
'job_output_dir': output_dir,
'job_polling_interval': 0.2, # quick polling
'job_target_resource': 'ebtestlocalhost',
'robot_path': os.path.join(topdir, 'easyconfigs', 'test_ecs'),
'silent': True,
'valid_module_classes': config.module_classes(),
'validate': False,
}
init_config(args=['--job-backend=GC3Pie'], build_options=build_options)
ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb')
easyconfigs = process_easyconfig(ec_file)
ordered_ecs = resolve_dependencies(easyconfigs, self.modtool)
topdir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
test_easyblocks_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sandbox')
cmd = "PYTHONPATH=%s:%s:$PYTHONPATH eb %%(spec)s -df" % (topdir, test_easyblocks_path)
build_easyconfigs_in_parallel(cmd, ordered_ecs, prepare_first=False)
self.assertTrue(os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0'))
self.assertTrue(os.path.join(self.test_installpath, 'software', 'toy', '0.0', 'bin', 'toy'))
def test_build_easyconfigs_in_parallel(self):
"""Basic test for build_easyconfigs_in_parallel function."""
easyconfig_file = os.path.join(os.path.dirname(__file__), "easyconfigs", "gzip-1.5-goolf-1.4.10.eb")
easyconfigs = process_easyconfig(easyconfig_file)
ordered_ecs = resolve_dependencies(easyconfigs)
jobs = build_easyconfigs_in_parallel("echo %(spec)s", ordered_ecs, prepare_first=False)
self.assertEqual(len(jobs), 8)
def test_build_easyconfigs_in_parallel_slurm(self):
"""Test build_easyconfigs_in_parallel(), using (mocked) Slurm as backend for --job."""
# install mocked versions of 'sbatch' and 'scontrol' commands
sbatch = os.path.join(self.test_prefix, 'bin', 'sbatch')
write_file(sbatch, MOCKED_SBATCH)
adjust_permissions(sbatch, stat.S_IXUSR, add=True)
scontrol = os.path.join(self.test_prefix, 'bin', 'scontrol')
write_file(scontrol, MOCKED_SCONTROL)
adjust_permissions(scontrol, stat.S_IXUSR, add=True)
os.environ['PATH'] = os.path.pathsep.join([os.path.join(self.test_prefix, 'bin'), os.getenv('PATH')])
topdir = os.path.dirname(os.path.abspath(__file__))
test_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'g', 'gzip', 'gzip-1.5-foss-2018a.eb')
foss_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'f', 'foss', 'foss-2018a.eb')
build_options = {
'external_modules_metadata': {},
'robot_path': os.path.join(topdir, 'easyconfigs', 'test_ecs'),
'valid_module_classes': config.module_classes(),
'validate': False,
'job_cores': 3,
'job_max_walltime': 5,
'force': True,
}
init_config(args=['--job-backend=Slurm'], build_options=build_options)
easyconfigs = process_easyconfig(test_ec) + process_easyconfig(foss_ec)
ordered_ecs = resolve_dependencies(easyconfigs, self.modtool)
self.mock_stdout(True)
jobs = build_easyconfigs_in_parallel("echo '%(spec)s'", ordered_ecs, prepare_first=False)
self.mock_stdout(False)
# jobs are submitted for foss & gzip (listed easyconfigs)
self.assertEqual(len(jobs), 2)
# last job (gzip) has a dependency on second-to-last job (foss)
self.assertEqual(jobs[0].job_specs['job-name'], 'foss-2018a')
expected = {
'dependency': 'afterok:%s' % jobs[0].jobid,
'hold': True,
'job-name': 'gzip-1.5-foss-2018a',
'nodes': 1,
'ntasks': 3,
'ntasks-per-node': 3,
'output': '%x-%j.out',
'time': 300, # 60*5 (unit is minutes)
'wrap': "echo '%s'" % test_ec,
}
self.assertEqual(jobs[1].job_specs, expected)
def test_build_easyconfigs_in_parallel_pbs_python(self):
"""Test build_easyconfigs_in_parallel(), using (mocked) pbs_python as backend for --job."""
# put mocked functions in place
PbsPython__init__ = PbsPython.__init__
PbsPython_check_version = PbsPython._check_version
PbsPython_complete = PbsPython.complete
PbsPython_connect_to_server = PbsPython.connect_to_server
PbsPython_ppn = PbsPython.ppn
pbs_python_PbsJob = pbs_python.PbsJob
PbsPython.__init__ = lambda self: PbsPython__init__(self, pbs_server='localhost')
PbsPython._check_version = lambda _: True
PbsPython.complete = mock
PbsPython.connect_to_server = mock
PbsPython.ppn = mock
pbs_python.PbsJob = MockPbsJob
build_options = {
'robot_path': os.path.join(os.path.dirname(__file__), 'easyconfigs'),
'valid_module_classes': config.module_classes(),
'validate': False,
}
init_config(args=['--job-backend=PbsPython'], build_options=build_options)
ec_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'gzip-1.5-goolf-1.4.10.eb')
easyconfigs = process_easyconfig(ec_file)
ordered_ecs = resolve_dependencies(easyconfigs)
jobs = build_easyconfigs_in_parallel("echo %(spec)s", ordered_ecs, prepare_first=False)
self.assertEqual(len(jobs), 8)
# restore mocked stuff
PbsPython.__init__ = PbsPython__init__
PbsPython._check_version = PbsPython_check_version
PbsPython.complete = PbsPython_complete
PbsPython.connect_to_server = PbsPython_connect_to_server
PbsPython.ppn = PbsPython_ppn
pbs_python.PbsJob = pbs_python_PbsJob
def test_build_easyconfigs_in_parallel_pbs_python(self):
"""Test build_easyconfigs_in_parallel(), using (mocked) pbs_python as backend for --job."""
# put mocked functions in place
PbsPython__init__ = PbsPython.__init__
PbsPython_check_version = PbsPython._check_version
PbsPython_complete = PbsPython.complete
PbsPython_connect_to_server = PbsPython.connect_to_server
PbsPython_ppn = PbsPython.ppn
pbs_python_PbsJob = pbs_python.PbsJob
PbsPython.__init__ = lambda self: PbsPython__init__(self, pbs_server='localhost')
PbsPython._check_version = lambda _: True
PbsPython.complete = mock
PbsPython.connect_to_server = mock
PbsPython.ppn = mock
pbs_python.PbsJob = MockPbsJob
topdir = os.path.dirname(os.path.abspath(__file__))
build_options = {
'external_modules_metadata': {},
'robot_path': os.path.join(topdir, 'easyconfigs', 'test_ecs'),
'valid_module_classes': config.module_classes(),
'validate': False,
'job_cores': 3,
}
init_config(args=['--job-backend=PbsPython'], build_options=build_options)
ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'g', 'gzip', 'gzip-1.5-foss-2018a.eb')
easyconfigs = process_easyconfig(ec_file)
ordered_ecs = resolve_dependencies(easyconfigs, self.modtool)
jobs = build_easyconfigs_in_parallel("echo '%(spec)s'", ordered_ecs, prepare_first=False)
# only one job submitted since foss/2018a module is already available
self.assertEqual(len(jobs), 1)
regex = re.compile("echo '.*/gzip-1.5-foss-2018a.eb'")
self.assertTrue(regex.search(jobs[-1].script), "Pattern '%s' found in: %s" % (regex.pattern, jobs[-1].script))
ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'g', 'gzip', 'gzip-1.4-GCC-4.6.3.eb')
ordered_ecs = resolve_dependencies(process_easyconfig(ec_file), self.modtool, retain_all_deps=True)
jobs = submit_jobs(ordered_ecs, '', testing=False, prepare_first=False)
# make sure command is correct, and that --hidden is there when it needs to be
for i, ec in enumerate(ordered_ecs):
if ec['hidden']:
regex = re.compile("eb %s.* --hidden" % ec['spec'])
else:
regex = re.compile("eb %s" % ec['spec'])
self.assertTrue(regex.search(jobs[i].script), "Pattern '%s' found in: %s" % (regex.pattern, jobs[i].script))
for job in jobs:
self.assertEqual(job.cores, build_options['job_cores'])
# no deps for GCC/4.6.3 (toolchain) and intel/2018a (test easyconfig with 'fake' deps)
self.assertEqual(len(jobs[0].deps), 0)
self.assertEqual(len(jobs[1].deps), 0)
# only dependency for toy/0.0-deps is intel/2018a (dep marked as external module is filtered out)
self.assertTrue('toy-0.0-deps.eb' in jobs[2].script)
self.assertEqual(len(jobs[2].deps), 1)
self.assertTrue('intel-2018a.eb' in jobs[2].deps[0].script)
# dependencies for gzip/1.4-GCC-4.6.3: GCC/4.6.3 (toolchain) + toy/.0.0-deps
self.assertTrue('gzip-1.4-GCC-4.6.3.eb' in jobs[3].script)
self.assertEqual(len(jobs[3].deps), 2)
regex = re.compile('toy-0.0-deps.eb\s* --hidden')
self.assertTrue(regex.search(jobs[3].deps[0].script))
self.assertTrue('GCC-4.6.3.eb' in jobs[3].deps[1].script)
# also test use of --pre-create-installdir
ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb')
ordered_ecs = resolve_dependencies(process_easyconfig(ec_file), self.modtool)
# installation directory doesn't exist yet before submission
toy_installdir = os.path.join(self.test_installpath, 'software', 'toy', '0.0')
self.assertFalse(os.path.exists(toy_installdir))
jobs = submit_jobs(ordered_ecs, '', testing=False)
self.assertEqual(len(jobs), 1)
# software install dir is created (by default) as part of job submission process (fetch_step is run)
self.assertTrue(os.path.exists(toy_installdir))
remove_dir(toy_installdir)
remove_dir(os.path.dirname(toy_installdir))
self.assertFalse(os.path.exists(toy_installdir))
# installation directory does *not* get created when --pre-create-installdir is used
build_options['pre_create_installdir'] = False
init_config(args=['--job-backend=PbsPython'], build_options=build_options)
jobs = submit_jobs(ordered_ecs, '', testing=False)
self.assertEqual(len(jobs), 1)
self.assertFalse(os.path.exists(toy_installdir))
# restore mocked stuff
PbsPython.__init__ = PbsPython__init__
PbsPython._check_version = PbsPython_check_version
PbsPython.complete = PbsPython_complete
PbsPython.connect_to_server = PbsPython_connect_to_server
PbsPython.ppn = PbsPython_ppn
pbs_python.PbsJob = pbs_python_PbsJob
if not build_option('force'):
_log.debug("Skipping easyconfigs from %s that already have a module available..." % easyconfigs)
easyconfigs = skip_available(easyconfigs, modtool)
_log.debug("Retained easyconfigs after skipping: %s" % easyconfigs)
if build_option('sequential'):
return build_easyconfigs(easyconfigs, output_dir, test_results)
else:
resolved = resolve_dependencies(easyconfigs, modtool)
cmd = "eb %(spec)s --regtest --sequential -ld --testoutput=%(output_dir)s"
command = "unset TMPDIR && cd %s && %s; " % (cur_dir, cmd)
# retry twice in case of failure, to avoid fluke errors
command += "if [ $? -ne 0 ]; then %(cmd)s --force && %(cmd)s --force; fi" % {'cmd': cmd}
build_easyconfigs_in_parallel(command, resolved, output_dir=output_dir)
_log.info("Submitted regression test as jobs, results in %s" % output_dir)
return True # success
def session_state():
"""Get session state: timestamp, dump of environment, system info."""
return {
'time': gmtime(),
'environment': copy.deepcopy(os.environ),
'system_info': get_system_info(),
}
# the options to ignore (help options can't reach here)
ignore_opts = ['robot', 'job']
# generate_cmd_line returns the options in form --longopt=value
opts = [
x for x in eb_go.generate_cmd_line()
if not x.split('=')[0] in ['--%s' % y for y in ignore_opts]
]
quoted_opts = subprocess.list2cmdline(opts)
command = "unset TMPDIR && cd %s && eb %%(spec)s %s" % (curdir,
quoted_opts)
_log.info("Command template for jobs: %s" % command)
if not testing:
jobs = build_easyconfigs_in_parallel(command, ordered_ecs)
txt = ["List of submitted jobs:"]
txt.extend([
"%s (%s): %s" % (job.name, job.module, job.jobid)
for job in jobs
])
txt.append("(%d jobs submitted)" % len(jobs))
print_msg("Submitted parallel build jobs, exiting now: %s" %
'\n'.join(txt),
log=_log)
cleanup(logfile, eb_tmpdir, testing)
sys.exit(0)
# build software, will exit when errors occurs (except when testing)
exit_on_failure = not options.dump_test_report and not options.upload_test_report
# submit build as job(s) and exit
if options.job:
curdir = os.getcwd()
# the options to ignore (help options can't reach here)
ignore_opts = ['robot', 'job']
# generate_cmd_line returns the options in form --longopt=value
opts = [x for x in eb_go.generate_cmd_line() if not x.split('=')[0] in ['--%s' % y for y in ignore_opts]]
quoted_opts = subprocess.list2cmdline(opts)
command = "unset TMPDIR && cd %s && eb %%(spec)s %s" % (curdir, quoted_opts)
_log.info("Command template for jobs: %s" % command)
if not testing:
jobs = parbuild.build_easyconfigs_in_parallel(command, orderedSpecs, "easybuild-build",
robot_path=options.robot)
txt = ["List of submitted jobs:"]
txt.extend(["%s: %s" % (job.name, job.jobid) for job in jobs])
txt.append("(%d jobs submitted)" % len(jobs))
msg = "\n".join(txt)
_log.info("Submitted parallel build jobs, exiting now (%s)." % msg)
print msg
cleanup_logfile_and_exit(logfile, testing, True)
sys.exit(0)
# build software, will exit when errors occurs (except when regtesting)
correct_built_cnt = 0
all_built_cnt = 0
if options.job:
curdir = os.getcwd()
# the options to ignore (help options can't reach here)
ignore_opts = ["robot", "job"]
# generate_cmd_line returns the options in form --longopt=value
opts = [x for x in eb_go.generate_cmd_line() if not x.split("=")[0] in ["--%s" % y for y in ignore_opts]]
quoted_opts = subprocess.list2cmdline(opts)
command = "unset TMPDIR && cd %s && eb %%(spec)s %s" % (curdir, quoted_opts)
_log.info("Command template for jobs: %s" % command)
if not testing:
jobs = parbuild.build_easyconfigs_in_parallel(
command, orderedSpecs, "easybuild-build", robot_path=options.robot
)
txt = ["List of submitted jobs:"]
txt.extend(["%s: %s" % (job.name, job.jobid) for job in jobs])
txt.append("(%d jobs submitted)" % len(jobs))
msg = "\n".join(txt)
_log.info("Submitted parallel build jobs, exiting now (%s)." % msg)
print msg
cleanup_logfile_and_exit(logfile, testing, True)
sys.exit(0)
# build software, will exit when errors occurs (except when regtesting)
correct_built_cnt = 0
def test_build_easyconfigs_in_parallel_pbs_python(self):
"""Test build_easyconfigs_in_parallel(), using (mocked) pbs_python as backend for --job."""
# put mocked functions in place
PbsPython__init__ = PbsPython.__init__
PbsPython_check_version = PbsPython._check_version
PbsPython_complete = PbsPython.complete
PbsPython_connect_to_server = PbsPython.connect_to_server
PbsPython_ppn = PbsPython.ppn
pbs_python_PbsJob = pbs_python.PbsJob
PbsPython.__init__ = lambda self: PbsPython__init__(self, pbs_server='localhost')
PbsPython._check_version = lambda _: True
PbsPython.complete = mock
PbsPython.connect_to_server = mock
PbsPython.ppn = mock
pbs_python.PbsJob = MockPbsJob
build_options = {
'external_modules_metadata': {},
'robot_path': os.path.join(os.path.dirname(__file__), 'easyconfigs'),
'valid_module_classes': config.module_classes(),
'validate': False,
'job_cores': 3,
}
init_config(args=['--job-backend=PbsPython'], build_options=build_options)
ec_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'gzip-1.5-goolf-1.4.10.eb')
easyconfigs = process_easyconfig(ec_file)
ordered_ecs = resolve_dependencies(easyconfigs)
jobs = build_easyconfigs_in_parallel("echo '%(spec)s'", ordered_ecs, prepare_first=False)
self.assertEqual(len(jobs), 8)
regex = re.compile("echo '.*/gzip-1.5-goolf-1.4.10.eb'")
self.assertTrue(regex.search(jobs[-1].script), "Pattern '%s' found in: %s" % (regex.pattern, jobs[-1].script))
ec_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'gzip-1.4-GCC-4.6.3.eb')
ordered_ecs = resolve_dependencies(process_easyconfig(ec_file), retain_all_deps=True)
jobs = submit_jobs(ordered_ecs, '', testing=False, prepare_first=False)
# make sure command is correct, and that --hidden is there when it needs to be
for i, ec in enumerate(ordered_ecs):
if ec['hidden']:
regex = re.compile("eb %s.* --hidden" % ec['spec'])
else:
regex = re.compile("eb %s" % ec['spec'])
self.assertTrue(regex.search(jobs[i].script), "Pattern '%s' found in: %s" % (regex.pattern, jobs[i].script))
for job in jobs:
self.assertEqual(job.cores, build_options['job_cores'])
# no deps for GCC/4.6.3 (toolchain) and ictce/4.1.13 (test easyconfig with 'fake' deps)
self.assertEqual(len(jobs[0].deps), 0)
self.assertEqual(len(jobs[1].deps), 0)
# only dependency for toy/0.0-deps is ictce/4.1.13 (dep marked as external module is filtered out)
self.assertTrue('toy-0.0-deps.eb' in jobs[2].script)
self.assertEqual(len(jobs[2].deps), 1)
self.assertTrue('ictce-4.1.13.eb' in jobs[2].deps[0].script)
# dependencies for gzip/1.4-GCC-4.6.3: GCC/4.6.3 (toolchain) + toy/.0.0-deps
self.assertTrue('gzip-1.4-GCC-4.6.3.eb' in jobs[3].script)
self.assertEqual(len(jobs[3].deps), 2)
regex = re.compile('toy-0.0-deps.eb\s* --hidden')
self.assertTrue(regex.search(jobs[3].deps[0].script))
self.assertTrue('GCC-4.6.3.eb' in jobs[3].deps[1].script)
# restore mocked stuff
PbsPython.__init__ = PbsPython__init__
PbsPython._check_version = PbsPython_check_version
PbsPython.complete = PbsPython_complete
PbsPython.connect_to_server = PbsPython_connect_to_server
PbsPython.ppn = PbsPython_ppn
pbs_python.PbsJob = pbs_python_PbsJob
def test_build_easyconfigs_in_parallel_gc3pie(self):
"""Test build_easyconfigs_in_parallel(), using GC3Pie with local config as backend for --job."""
try:
import gc3libs # noqa (ignore unused import)
except ImportError:
print("GC3Pie not available, skipping test")
return
# put GC3Pie config in place to use local host and fork/exec
resourcedir = os.path.join(self.test_prefix, 'gc3pie')
gc3pie_cfgfile = os.path.join(self.test_prefix, 'gc3pie_local.ini')
gc3pie_cfgtxt = GC3PIE_LOCAL_CONFIGURATION % {
'resourcedir': resourcedir,
'time': which('time'),
}
write_file(gc3pie_cfgfile, gc3pie_cfgtxt)
output_dir = os.path.join(self.test_prefix, 'subdir',
'gc3pie_output_dir')
# purposely pre-create output dir, and put a file in it (to check whether GC3Pie tries to rename the output dir)
mkdir(output_dir, parents=True)
write_file(os.path.join(output_dir, 'foo'), 'bar')
# remove write permissions on parent dir of specified output dir,
# to check that GC3Pie does not try to rename the (already existing) output directory...
adjust_permissions(os.path.dirname(output_dir),
stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH,
add=False,
recursive=False)
topdir = os.path.dirname(os.path.abspath(__file__))
build_options = {
'job_backend_config': gc3pie_cfgfile,
'job_max_walltime': 24,
'job_output_dir': output_dir,
'job_polling_interval': 0.2, # quick polling
'job_target_resource': 'ebtestlocalhost',
'robot_path': os.path.join(topdir, 'easyconfigs', 'test_ecs'),
'silent': True,
'valid_module_classes': config.module_classes(),
'validate': False,
}
init_config(args=['--job-backend=GC3Pie'], build_options=build_options)
ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy',
'toy-0.0.eb')
easyconfigs = process_easyconfig(ec_file)
ordered_ecs = resolve_dependencies(easyconfigs, self.modtool)
topdir = os.path.dirname(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
test_easyblocks_path = os.path.join(
os.path.dirname(os.path.abspath(__file__)), 'sandbox')
cmd = "PYTHONPATH=%s:%s:$PYTHONPATH eb %%(spec)s -df" % (
topdir, test_easyblocks_path)
build_easyconfigs_in_parallel(cmd, ordered_ecs, prepare_first=False)
toy_modfile = os.path.join(self.test_installpath, 'modules', 'all',
'toy', '0.0')
if get_module_syntax() == 'Lua':
toy_modfile += '.lua'
self.assertTrue(os.path.exists(toy_modfile))
self.assertTrue(
os.path.exists(
os.path.join(self.test_installpath, 'software', 'toy', '0.0',
'bin', 'toy')))
# also check what happens when a job fails (an error should be raised)
test_ecfile = os.path.join(self.test_prefix, 'test.eb')
ectxt = read_file(ec_file)
# use different version, for which no sources are available
regex = re.compile('^version = .*', re.M)
ectxt = regex.sub("version = '1.2.3'", ectxt)
write_file(test_ecfile, ectxt)
ecs = resolve_dependencies(process_easyconfig(test_ecfile),
self.modtool)
error = "1 jobs failed: toy-1.2.3"
self.assertErrorRegex(EasyBuildError,
error,
build_easyconfigs_in_parallel,
cmd,
ecs,
prepare_first=False)
def test_build_easyconfigs_in_parallel_pbs_python(self):
"""Test build_easyconfigs_in_parallel(), using (mocked) pbs_python as backend for --job."""
# put mocked functions in place
PbsPython__init__ = PbsPython.__init__
PbsPython_check_version = PbsPython._check_version
PbsPython_complete = PbsPython.complete
PbsPython_connect_to_server = PbsPython.connect_to_server
PbsPython_ppn = PbsPython.ppn
pbs_python_PbsJob = pbs_python.PbsJob
PbsPython.__init__ = lambda self: PbsPython__init__(
self, pbs_server='localhost')
PbsPython._check_version = lambda _: True
PbsPython.complete = mock
PbsPython.connect_to_server = mock
PbsPython.ppn = mock
pbs_python.PbsJob = MockPbsJob
topdir = os.path.dirname(os.path.abspath(__file__))
build_options = {
'external_modules_metadata': {},
'robot_path': os.path.join(topdir, 'easyconfigs', 'test_ecs'),
'valid_module_classes': config.module_classes(),
'validate': False,
'job_cores': 3,
}
init_config(args=['--job-backend=PbsPython'],
build_options=build_options)
ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'g', 'gzip',
'gzip-1.5-foss-2018a.eb')
easyconfigs = process_easyconfig(ec_file)
ordered_ecs = resolve_dependencies(easyconfigs, self.modtool)
jobs = build_easyconfigs_in_parallel("echo '%(spec)s'",
ordered_ecs,
prepare_first=False)
# only one job submitted since foss/2018a module is already available
self.assertEqual(len(jobs), 1)
regex = re.compile("echo '.*/gzip-1.5-foss-2018a.eb'")
self.assertTrue(
regex.search(jobs[-1].script),
"Pattern '%s' found in: %s" % (regex.pattern, jobs[-1].script))
ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'g', 'gzip',
'gzip-1.4-GCC-4.6.3.eb')
ordered_ecs = resolve_dependencies(process_easyconfig(ec_file),
self.modtool,
retain_all_deps=True)
jobs = submit_jobs(ordered_ecs, '', testing=False, prepare_first=False)
# make sure command is correct, and that --hidden is there when it needs to be
for i, ec in enumerate(ordered_ecs):
if ec['hidden']:
regex = re.compile("eb %s.* --hidden" % ec['spec'])
else:
regex = re.compile("eb %s" % ec['spec'])
self.assertTrue(
regex.search(jobs[i].script),
"Pattern '%s' found in: %s" % (regex.pattern, jobs[i].script))
for job in jobs:
self.assertEqual(job.cores, build_options['job_cores'])
# no deps for GCC/4.6.3 (toolchain) and intel/2018a (test easyconfig with 'fake' deps)
self.assertEqual(len(jobs[0].deps), 0)
self.assertEqual(len(jobs[1].deps), 0)
# only dependency for toy/0.0-deps is intel/2018a (dep marked as external module is filtered out)
self.assertTrue('toy-0.0-deps.eb' in jobs[2].script)
self.assertEqual(len(jobs[2].deps), 1)
self.assertTrue('intel-2018a.eb' in jobs[2].deps[0].script)
# dependencies for gzip/1.4-GCC-4.6.3: GCC/4.6.3 (toolchain) + toy/.0.0-deps
self.assertTrue('gzip-1.4-GCC-4.6.3.eb' in jobs[3].script)
self.assertEqual(len(jobs[3].deps), 2)
regex = re.compile(r'toy-0.0-deps\.eb.* --hidden')
script_txt = jobs[3].deps[0].script
fail_msg = "Pattern '%s' should be found in: %s" % (regex.pattern,
script_txt)
self.assertTrue(regex.search(script_txt), fail_msg)
self.assertTrue('GCC-4.6.3.eb' in jobs[3].deps[1].script)
# also test use of --pre-create-installdir
ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy',
'toy-0.0.eb')
ordered_ecs = resolve_dependencies(process_easyconfig(ec_file),
self.modtool)
# installation directory doesn't exist yet before submission
toy_installdir = os.path.join(self.test_installpath, 'software', 'toy',
'0.0')
self.assertFalse(os.path.exists(toy_installdir))
jobs = submit_jobs(ordered_ecs, '', testing=False)
self.assertEqual(len(jobs), 1)
# software install dir is created (by default) as part of job submission process (fetch_step is run)
self.assertTrue(os.path.exists(toy_installdir))
remove_dir(toy_installdir)
remove_dir(os.path.dirname(toy_installdir))
self.assertFalse(os.path.exists(toy_installdir))
# installation directory does *not* get created when --pre-create-installdir is used
build_options['pre_create_installdir'] = False
init_config(args=['--job-backend=PbsPython'],
build_options=build_options)
jobs = submit_jobs(ordered_ecs, '', testing=False)
self.assertEqual(len(jobs), 1)
self.assertFalse(os.path.exists(toy_installdir))
# restore mocked stuff
PbsPython.__init__ = PbsPython__init__
PbsPython._check_version = PbsPython_check_version
PbsPython.complete = PbsPython_complete
PbsPython.connect_to_server = PbsPython_connect_to_server
PbsPython.ppn = PbsPython_ppn
pbs_python.PbsJob = pbs_python_PbsJob
def test_build_easyconfigs_in_parallel_pbs_python(self):
"""Test build_easyconfigs_in_parallel(), using (mocked) pbs_python as backend for --job."""
# put mocked functions in place
PbsPython__init__ = PbsPython.__init__
PbsPython_check_version = PbsPython._check_version
PbsPython_complete = PbsPython.complete
PbsPython_connect_to_server = PbsPython.connect_to_server
PbsPython_ppn = PbsPython.ppn
pbs_python_PbsJob = pbs_python.PbsJob
PbsPython.__init__ = lambda self: PbsPython__init__(
self, pbs_server='localhost')
PbsPython._check_version = lambda _: True
PbsPython.complete = mock
PbsPython.connect_to_server = mock
PbsPython.ppn = mock
pbs_python.PbsJob = MockPbsJob
topdir = os.path.dirname(os.path.abspath(__file__))
build_options = {
'external_modules_metadata': {},
'robot_path': os.path.join(topdir, 'easyconfigs', 'test_ecs'),
'valid_module_classes': config.module_classes(),
'validate': False,
'job_cores': 3,
}
init_config(args=['--job-backend=PbsPython'],
build_options=build_options)
ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'g', 'gzip',
'gzip-1.5-goolf-1.4.10.eb')
easyconfigs = process_easyconfig(ec_file)
ordered_ecs = resolve_dependencies(easyconfigs, self.modtool)
jobs = build_easyconfigs_in_parallel("echo '%(spec)s'",
ordered_ecs,
prepare_first=False)
self.assertEqual(len(jobs), 8)
regex = re.compile("echo '.*/gzip-1.5-goolf-1.4.10.eb'")
self.assertTrue(
regex.search(jobs[-1].script),
"Pattern '%s' found in: %s" % (regex.pattern, jobs[-1].script))
ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'g', 'gzip',
'gzip-1.4-GCC-4.6.3.eb')
ordered_ecs = resolve_dependencies(process_easyconfig(ec_file),
self.modtool,
retain_all_deps=True)
jobs = submit_jobs(ordered_ecs, '', testing=False, prepare_first=False)
# make sure command is correct, and that --hidden is there when it needs to be
for i, ec in enumerate(ordered_ecs):
if ec['hidden']:
regex = re.compile("eb %s.* --hidden" % ec['spec'])
else:
regex = re.compile("eb %s" % ec['spec'])
self.assertTrue(
regex.search(jobs[i].script),
"Pattern '%s' found in: %s" % (regex.pattern, jobs[i].script))
for job in jobs:
self.assertEqual(job.cores, build_options['job_cores'])
# no deps for GCC/4.6.3 (toolchain) and ictce/4.1.13 (test easyconfig with 'fake' deps)
self.assertEqual(len(jobs[0].deps), 0)
self.assertEqual(len(jobs[1].deps), 0)
# only dependency for toy/0.0-deps is ictce/4.1.13 (dep marked as external module is filtered out)
self.assertTrue('toy-0.0-deps.eb' in jobs[2].script)
self.assertEqual(len(jobs[2].deps), 1)
self.assertTrue('ictce-4.1.13.eb' in jobs[2].deps[0].script)
# dependencies for gzip/1.4-GCC-4.6.3: GCC/4.6.3 (toolchain) + toy/.0.0-deps
self.assertTrue('gzip-1.4-GCC-4.6.3.eb' in jobs[3].script)
self.assertEqual(len(jobs[3].deps), 2)
regex = re.compile('toy-0.0-deps.eb\s* --hidden')
self.assertTrue(regex.search(jobs[3].deps[0].script))
self.assertTrue('GCC-4.6.3.eb' in jobs[3].deps[1].script)
# restore mocked stuff
PbsPython.__init__ = PbsPython__init__
PbsPython._check_version = PbsPython_check_version
PbsPython.complete = PbsPython_complete
PbsPython.connect_to_server = PbsPython_connect_to_server
PbsPython.ppn = PbsPython_ppn
pbs_python.PbsJob = pbs_python_PbsJob
def test_build_easyconfigs_in_parallel_gc3pie(self):
"""Test build_easyconfigs_in_parallel(), using GC3Pie with local config as backend for --job."""
try:
import gc3libs
except ImportError:
print "GC3Pie not available, skipping test"
return
# put GC3Pie config in place to use local host and fork/exec
resourcedir = os.path.join(self.test_prefix, 'gc3pie')
gc3pie_cfgfile = os.path.join(self.test_prefix, 'gc3pie_local.ini')
gc3pie_cfgtxt = GC3PIE_LOCAL_CONFIGURATION % {
'resourcedir': resourcedir,
'time': which('time'),
}
write_file(gc3pie_cfgfile, gc3pie_cfgtxt)
output_dir = os.path.join(self.test_prefix, 'subdir',
'gc3pie_output_dir')
# purposely pre-create output dir, and put a file in it (to check whether GC3Pie tries to rename the output dir)
mkdir(output_dir, parents=True)
write_file(os.path.join(output_dir, 'foo'), 'bar')
# remove write permissions on parent dir of specified output dir,
# to check that GC3Pie does not try to rename the (already existing) output directory...
adjust_permissions(os.path.dirname(output_dir),
stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH,
add=False,
recursive=False)
topdir = os.path.dirname(os.path.abspath(__file__))
build_options = {
'job_backend_config': gc3pie_cfgfile,
'job_max_walltime': 24,
'job_output_dir': output_dir,
'job_polling_interval': 0.2, # quick polling
'job_target_resource': 'ebtestlocalhost',
'robot_path': os.path.join(topdir, 'easyconfigs', 'test_ecs'),
'silent': True,
'valid_module_classes': config.module_classes(),
'validate': False,
}
options = init_config(args=['--job-backend=GC3Pie'],
build_options=build_options)
ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy',
'toy-0.0.eb')
easyconfigs = process_easyconfig(ec_file)
ordered_ecs = resolve_dependencies(easyconfigs, self.modtool)
topdir = os.path.dirname(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
test_easyblocks_path = os.path.join(
os.path.dirname(os.path.abspath(__file__)), 'sandbox')
cmd = "PYTHONPATH=%s:%s:$PYTHONPATH eb %%(spec)s -df" % (
topdir, test_easyblocks_path)
jobs = build_easyconfigs_in_parallel(cmd,
ordered_ecs,
prepare_first=False)
self.assertTrue(
os.path.join(self.test_installpath, 'modules', 'all', 'toy',
'0.0'))
self.assertTrue(
os.path.join(self.test_installpath, 'software', 'toy', '0.0',
'bin', 'toy'))
def regtest(easyconfig_paths, modtool, build_specs=None):
"""
Run regression test, using easyconfigs available in given path
:param easyconfig_paths: path of easyconfigs to run regtest on
:param modtool: ModulesTool instance to use
:param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...)
"""
cur_dir = os.getcwd()
aggregate_regtest = build_option('aggregate_regtest')
if aggregate_regtest is not None:
output_file = os.path.join(
aggregate_regtest,
"%s-aggregate.xml" % os.path.basename(aggregate_regtest))
aggregate_xml_in_dirs(aggregate_regtest, output_file)
_log.info("aggregated xml files inside %s, output written to: %s" %
(aggregate_regtest, output_file))
sys.exit(0)
# create base directory, which is used to place all log files and the test output as xml
regtest_output_dir = build_option('regtest_output_dir')
testoutput = build_option('testoutput')
if regtest_output_dir is not None:
output_dir = regtest_output_dir
elif testoutput is not None:
output_dir = os.path.abspath(testoutput)
else:
# default: current dir + easybuild-test-[timestamp]
dirname = "easybuild-test-%s" % datetime.now().strftime("%Y%m%d%H%M%S")
output_dir = os.path.join(cur_dir, dirname)
mkdir(output_dir, parents=True)
# find all easyconfigs
ecfiles = []
if easyconfig_paths:
for path in easyconfig_paths:
ecfiles += find_easyconfigs(
path, ignore_dirs=build_option('ignore_dirs'))
else:
raise EasyBuildError("No easyconfig paths specified.")
test_results = []
# process all the found easyconfig files
easyconfigs = []
for ecfile in ecfiles:
try:
easyconfigs.extend(
process_easyconfig(ecfile, build_specs=build_specs))
except EasyBuildError as err:
test_results.append((ecfile, 'parsing_easyconfigs',
'easyconfig file error: %s' % err, _log))
# skip easyconfigs for which a module is already available, unless forced
if not build_option('force'):
_log.debug(
"Skipping easyconfigs from %s that already have a module available..."
% easyconfigs)
easyconfigs = skip_available(easyconfigs, modtool)
_log.debug("Retained easyconfigs after skipping: %s" % easyconfigs)
if build_option('sequential'):
return build_easyconfigs(easyconfigs, output_dir, test_results)
else:
resolved = resolve_dependencies(easyconfigs, modtool)
cmd = "eb %(spec)s --regtest --sequential -ld --testoutput=%(output_dir)s"
command = "unset TMPDIR && cd %s && %s; " % (cur_dir, cmd)
# retry twice in case of failure, to avoid fluke errors
command += "if [ $? -ne 0 ]; then %(cmd)s --force && %(cmd)s --force; fi" % {
'cmd': cmd
}
build_easyconfigs_in_parallel(command, resolved, output_dir=output_dir)
_log.info("Submitted regression test as jobs, results in %s" %
output_dir)
return True # success
# submit build as job(s) and exit
if options.job:
curdir = os.getcwd()
# the options to ignore (help options can't reach here)
ignore_opts = ['robot', 'job']
# generate_cmd_line returns the options in form --longopt=value
opts = [x for x in eb_go.generate_cmd_line() if not x.split('=')[0] in ['--%s' % y for y in ignore_opts]]
quoted_opts = subprocess.list2cmdline(opts)
command = "unset TMPDIR && cd %s && eb %%(spec)s %s" % (curdir, quoted_opts)
_log.info("Command template for jobs: %s" % command)
if not testing:
jobs = build_easyconfigs_in_parallel(command, ordered_ecs, build_options=build_options,
build_specs=build_specs)
txt = ["List of submitted jobs:"]
txt.extend(["%s (%s): %s" % (job.name, job.module, job.jobid) for job in jobs])
txt.append("(%d jobs submitted)" % len(jobs))
msg = "\n".join(txt)
_log.info("Submitted parallel build jobs, exiting now (%s)." % msg)
print msg
cleanup(logfile, eb_tmpdir, testing)
sys.exit(0)
# build software, will exit when errors occurs (except when regtesting)
correct_built_cnt = 0
all_built_cnt = 0
def test_build_easyconfigs_in_parallel(self):
"""Basic test for build_easyconfigs_in_parallel function."""
easyconfig_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'gzip-1.5-goolf-1.4.10.eb')
easyconfigs = process_easyconfig(easyconfig_file, validate=False)
ordered_ecs = resolve_dependencies(easyconfigs)
build_easyconfigs_in_parallel("echo %(spec)s", ordered_ecs)
_log.debug("Retained easyconfigs after skipping: %s" % easyconfigs)
if build_option('sequential'):
return build_easyconfigs(easyconfigs, output_dir, test_results)
else:
resolved = resolve_dependencies(easyconfigs, build_specs=build_specs)
cmd = "eb %(spec)s --regtest --sequential -ld --testoutput=%(output_dir)s"
command = "unset TMPDIR && cd %s && %s; " % (cur_dir, cmd)
# retry twice in case of failure, to avoid fluke errors
command += "if [ $? -ne 0 ]; then %(cmd)s --force && %(cmd)s --force; fi" % {
'cmd': cmd
}
jobs = build_easyconfigs_in_parallel(command,
resolved,
output_dir=output_dir)
print "List of submitted jobs:"
for job in jobs:
print "%s: %s" % (job.name, job.jobid)
print "(%d jobs submitted)" % len(jobs)
# determine leaf nodes in dependency graph, and report them
all_deps = set()
for job in jobs:
all_deps = all_deps.union(job.deps)
leaf_nodes = []
for job in jobs:
if job.jobid not in all_deps:
# submit build as job(s) and exit
if options.job:
curdir = os.getcwd()
# the options to ignore (help options can't reach here)
ignore_opts = ['robot', 'job']
# generate_cmd_line returns the options in form --longopt=value
opts = [x for x in eb_go.generate_cmd_line() if not x.split('=')[0] in ['--%s' % y for y in ignore_opts]]
quoted_opts = subprocess.list2cmdline(opts)
command = "unset TMPDIR && cd %s && eb %%(spec)s %s" % (curdir, quoted_opts)
_log.info("Command template for jobs: %s" % command)
if not testing:
jobs = build_easyconfigs_in_parallel(command, ordered_ecs)
txt = ["List of submitted jobs:"]
txt.extend(["%s (%s): %s" % (job.name, job.module, job.jobid) for job in jobs])
txt.append("(%d jobs submitted)" % len(jobs))
msg = "\n".join(txt)
_log.info("Submitted parallel build jobs, exiting now (%s)." % msg)
print msg
cleanup(logfile, eb_tmpdir, testing)
sys.exit(0)
# build software, will exit when errors occurs (except when regtesting)
correct_built_cnt = 0
all_built_cnt = 0
def regtest(easyconfig_paths, modtool, build_specs=None):
"""
Run regression test, using easyconfigs available in given path
:param easyconfig_paths: path of easyconfigs to run regtest on
:param modtool: ModulesTool instance to use
:param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...)
"""
cur_dir = os.getcwd()
aggregate_regtest = build_option('aggregate_regtest')
if aggregate_regtest is not None:
output_file = os.path.join(aggregate_regtest, "%s-aggregate.xml" % os.path.basename(aggregate_regtest))
aggregate_xml_in_dirs(aggregate_regtest, output_file)
_log.info("aggregated xml files inside %s, output written to: %s" % (aggregate_regtest, output_file))
sys.exit(0)
# create base directory, which is used to place all log files and the test output as xml
regtest_output_dir = build_option('regtest_output_dir')
testoutput = build_option('testoutput')
if regtest_output_dir is not None:
output_dir = regtest_output_dir
elif testoutput is not None:
output_dir = os.path.abspath(testoutput)
else:
# default: current dir + easybuild-test-[timestamp]
dirname = "easybuild-test-%s" % datetime.now().strftime("%Y%m%d%H%M%S")
output_dir = os.path.join(cur_dir, dirname)
mkdir(output_dir, parents=True)
# find all easyconfigs
ecfiles = []
if easyconfig_paths:
for path in easyconfig_paths:
ecfiles += find_easyconfigs(path, ignore_dirs=build_option('ignore_dirs'))
else:
raise EasyBuildError("No easyconfig paths specified.")
test_results = []
# process all the found easyconfig files
easyconfigs = []
for ecfile in ecfiles:
try:
easyconfigs.extend(process_easyconfig(ecfile, build_specs=build_specs))
except EasyBuildError as err:
test_results.append((ecfile, 'parsing_easyconfigs', 'easyconfig file error: %s' % err, _log))
# skip easyconfigs for which a module is already available, unless forced
if not build_option('force'):
_log.debug("Skipping easyconfigs from %s that already have a module available..." % easyconfigs)
easyconfigs = skip_available(easyconfigs, modtool)
_log.debug("Retained easyconfigs after skipping: %s" % easyconfigs)
if build_option('sequential'):
return build_easyconfigs(easyconfigs, output_dir, test_results)
else:
resolved = resolve_dependencies(easyconfigs, modtool)
cmd = "eb %(spec)s --regtest --sequential -ld --testoutput=%(output_dir)s"
command = "unset TMPDIR && cd %s && %s; " % (cur_dir, cmd)
# retry twice in case of failure, to avoid fluke errors
command += "if [ $? -ne 0 ]; then %(cmd)s --force && %(cmd)s --force; fi" % {'cmd': cmd}
build_easyconfigs_in_parallel(command, resolved, output_dir=output_dir)
_log.info("Submitted regression test as jobs, results in %s" % output_dir)
return True # success
