def test_build_easyconfigs_in_parallel_gc3pie(self): """Test build_easyconfigs_in_parallel(), using GC3Pie with local config as backend for --job.""" try: import gc3libs # noqa (ignore unused import) except ImportError: print "GC3Pie not available, skipping test" return # put GC3Pie config in place to use local host and fork/exec resourcedir = os.path.join(self.test_prefix, 'gc3pie') gc3pie_cfgfile = os.path.join(self.test_prefix, 'gc3pie_local.ini') gc3pie_cfgtxt = GC3PIE_LOCAL_CONFIGURATION % { 'resourcedir': resourcedir, 'time': which('time'), } write_file(gc3pie_cfgfile, gc3pie_cfgtxt) output_dir = os.path.join(self.test_prefix, 'subdir', 'gc3pie_output_dir') # purposely pre-create output dir, and put a file in it (to check whether GC3Pie tries to rename the output dir) mkdir(output_dir, parents=True) write_file(os.path.join(output_dir, 'foo'), 'bar') # remove write permissions on parent dir of specified output dir, # to check that GC3Pie does not try to rename the (already existing) output directory... adjust_permissions(os.path.dirname(output_dir), stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH, add=False, recursive=False) topdir = os.path.dirname(os.path.abspath(__file__)) build_options = { 'job_backend_config': gc3pie_cfgfile, 'job_max_walltime': 24, 'job_output_dir': output_dir, 'job_polling_interval': 0.2, # quick polling 'job_target_resource': 'ebtestlocalhost', 'robot_path': os.path.join(topdir, 'easyconfigs', 'test_ecs'), 'silent': True, 'valid_module_classes': config.module_classes(), 'validate': False, } init_config(args=['--job-backend=GC3Pie'], build_options=build_options) ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') easyconfigs = process_easyconfig(ec_file) ordered_ecs = resolve_dependencies(easyconfigs, self.modtool) topdir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) test_easyblocks_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sandbox') cmd = "PYTHONPATH=%s:%s:$PYTHONPATH eb %%(spec)s -df" % (topdir, test_easyblocks_path) build_easyconfigs_in_parallel(cmd, ordered_ecs, prepare_first=False) self.assertTrue(os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0')) self.assertTrue(os.path.join(self.test_installpath, 'software', 'toy', '0.0', 'bin', 'toy'))
def test_build_easyconfigs_in_parallel(self): """Basic test for build_easyconfigs_in_parallel function.""" easyconfig_file = os.path.join(os.path.dirname(__file__), "easyconfigs", "gzip-1.5-goolf-1.4.10.eb") easyconfigs = process_easyconfig(easyconfig_file) ordered_ecs = resolve_dependencies(easyconfigs) jobs = build_easyconfigs_in_parallel("echo %(spec)s", ordered_ecs, prepare_first=False) self.assertEqual(len(jobs), 8)
def test_build_easyconfigs_in_parallel_slurm(self): """Test build_easyconfigs_in_parallel(), using (mocked) Slurm as backend for --job.""" # install mocked versions of 'sbatch' and 'scontrol' commands sbatch = os.path.join(self.test_prefix, 'bin', 'sbatch') write_file(sbatch, MOCKED_SBATCH) adjust_permissions(sbatch, stat.S_IXUSR, add=True) scontrol = os.path.join(self.test_prefix, 'bin', 'scontrol') write_file(scontrol, MOCKED_SCONTROL) adjust_permissions(scontrol, stat.S_IXUSR, add=True) os.environ['PATH'] = os.path.pathsep.join([os.path.join(self.test_prefix, 'bin'), os.getenv('PATH')]) topdir = os.path.dirname(os.path.abspath(__file__)) test_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'g', 'gzip', 'gzip-1.5-foss-2018a.eb') foss_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'f', 'foss', 'foss-2018a.eb') build_options = { 'external_modules_metadata': {}, 'robot_path': os.path.join(topdir, 'easyconfigs', 'test_ecs'), 'valid_module_classes': config.module_classes(), 'validate': False, 'job_cores': 3, 'job_max_walltime': 5, 'force': True, } init_config(args=['--job-backend=Slurm'], build_options=build_options) easyconfigs = process_easyconfig(test_ec) + process_easyconfig(foss_ec) ordered_ecs = resolve_dependencies(easyconfigs, self.modtool) self.mock_stdout(True) jobs = build_easyconfigs_in_parallel("echo '%(spec)s'", ordered_ecs, prepare_first=False) self.mock_stdout(False) # jobs are submitted for foss & gzip (listed easyconfigs) self.assertEqual(len(jobs), 2) # last job (gzip) has a dependency on second-to-last job (foss) self.assertEqual(jobs[0].job_specs['job-name'], 'foss-2018a') expected = { 'dependency': 'afterok:%s' % jobs[0].jobid, 'hold': True, 'job-name': 'gzip-1.5-foss-2018a', 'nodes': 1, 'ntasks': 3, 'ntasks-per-node': 3, 'output': '%x-%j.out', 'time': 300, # 60*5 (unit is minutes) 'wrap': "echo '%s'" % test_ec, } self.assertEqual(jobs[1].job_specs, expected)
def test_build_easyconfigs_in_parallel_pbs_python(self): """Test build_easyconfigs_in_parallel(), using (mocked) pbs_python as backend for --job.""" # put mocked functions in place PbsPython__init__ = PbsPython.__init__ PbsPython_check_version = PbsPython._check_version PbsPython_complete = PbsPython.complete PbsPython_connect_to_server = PbsPython.connect_to_server PbsPython_ppn = PbsPython.ppn pbs_python_PbsJob = pbs_python.PbsJob PbsPython.__init__ = lambda self: PbsPython__init__(self, pbs_server='localhost') PbsPython._check_version = lambda _: True PbsPython.complete = mock PbsPython.connect_to_server = mock PbsPython.ppn = mock pbs_python.PbsJob = MockPbsJob build_options = { 'robot_path': os.path.join(os.path.dirname(__file__), 'easyconfigs'), 'valid_module_classes': config.module_classes(), 'validate': False, } init_config(args=['--job-backend=PbsPython'], build_options=build_options) ec_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'gzip-1.5-goolf-1.4.10.eb') easyconfigs = process_easyconfig(ec_file) ordered_ecs = resolve_dependencies(easyconfigs) jobs = build_easyconfigs_in_parallel("echo %(spec)s", ordered_ecs, prepare_first=False) self.assertEqual(len(jobs), 8) # restore mocked stuff PbsPython.__init__ = PbsPython__init__ PbsPython._check_version = PbsPython_check_version PbsPython.complete = PbsPython_complete PbsPython.connect_to_server = PbsPython_connect_to_server PbsPython.ppn = PbsPython_ppn pbs_python.PbsJob = pbs_python_PbsJob
def test_build_easyconfigs_in_parallel_pbs_python(self): """Test build_easyconfigs_in_parallel(), using (mocked) pbs_python as backend for --job.""" # put mocked functions in place PbsPython__init__ = PbsPython.__init__ PbsPython_check_version = PbsPython._check_version PbsPython_complete = PbsPython.complete PbsPython_connect_to_server = PbsPython.connect_to_server PbsPython_ppn = PbsPython.ppn pbs_python_PbsJob = pbs_python.PbsJob PbsPython.__init__ = lambda self: PbsPython__init__(self, pbs_server='localhost') PbsPython._check_version = lambda _: True PbsPython.complete = mock PbsPython.connect_to_server = mock PbsPython.ppn = mock pbs_python.PbsJob = MockPbsJob topdir = os.path.dirname(os.path.abspath(__file__)) build_options = { 'external_modules_metadata': {}, 'robot_path': os.path.join(topdir, 'easyconfigs', 'test_ecs'), 'valid_module_classes': config.module_classes(), 'validate': False, 'job_cores': 3, } init_config(args=['--job-backend=PbsPython'], build_options=build_options) ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'g', 'gzip', 'gzip-1.5-foss-2018a.eb') easyconfigs = process_easyconfig(ec_file) ordered_ecs = resolve_dependencies(easyconfigs, self.modtool) jobs = build_easyconfigs_in_parallel("echo '%(spec)s'", ordered_ecs, prepare_first=False) # only one job submitted since foss/2018a module is already available self.assertEqual(len(jobs), 1) regex = re.compile("echo '.*/gzip-1.5-foss-2018a.eb'") self.assertTrue(regex.search(jobs[-1].script), "Pattern '%s' found in: %s" % (regex.pattern, jobs[-1].script)) ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'g', 'gzip', 'gzip-1.4-GCC-4.6.3.eb') ordered_ecs = resolve_dependencies(process_easyconfig(ec_file), self.modtool, retain_all_deps=True) jobs = submit_jobs(ordered_ecs, '', testing=False, prepare_first=False) # make sure command is correct, and that --hidden is there when it needs to be for i, ec in enumerate(ordered_ecs): if ec['hidden']: regex = re.compile("eb %s.* --hidden" % ec['spec']) else: regex = re.compile("eb %s" % ec['spec']) self.assertTrue(regex.search(jobs[i].script), "Pattern '%s' found in: %s" % (regex.pattern, jobs[i].script)) for job in jobs: self.assertEqual(job.cores, build_options['job_cores']) # no deps for GCC/4.6.3 (toolchain) and intel/2018a (test easyconfig with 'fake' deps) self.assertEqual(len(jobs[0].deps), 0) self.assertEqual(len(jobs[1].deps), 0) # only dependency for toy/0.0-deps is intel/2018a (dep marked as external module is filtered out) self.assertTrue('toy-0.0-deps.eb' in jobs[2].script) self.assertEqual(len(jobs[2].deps), 1) self.assertTrue('intel-2018a.eb' in jobs[2].deps[0].script) # dependencies for gzip/1.4-GCC-4.6.3: GCC/4.6.3 (toolchain) + toy/.0.0-deps self.assertTrue('gzip-1.4-GCC-4.6.3.eb' in jobs[3].script) self.assertEqual(len(jobs[3].deps), 2) regex = re.compile('toy-0.0-deps.eb\s* --hidden') self.assertTrue(regex.search(jobs[3].deps[0].script)) self.assertTrue('GCC-4.6.3.eb' in jobs[3].deps[1].script) # also test use of --pre-create-installdir ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') ordered_ecs = resolve_dependencies(process_easyconfig(ec_file), self.modtool) # installation directory doesn't exist yet before submission toy_installdir = os.path.join(self.test_installpath, 'software', 'toy', '0.0') self.assertFalse(os.path.exists(toy_installdir)) jobs = submit_jobs(ordered_ecs, '', testing=False) self.assertEqual(len(jobs), 1) # software install dir is created (by default) as part of job submission process (fetch_step is run) self.assertTrue(os.path.exists(toy_installdir)) remove_dir(toy_installdir) remove_dir(os.path.dirname(toy_installdir)) self.assertFalse(os.path.exists(toy_installdir)) # installation directory does *not* get created when --pre-create-installdir is used build_options['pre_create_installdir'] = False init_config(args=['--job-backend=PbsPython'], build_options=build_options) jobs = submit_jobs(ordered_ecs, '', testing=False) self.assertEqual(len(jobs), 1) self.assertFalse(os.path.exists(toy_installdir)) # restore mocked stuff PbsPython.__init__ = PbsPython__init__ PbsPython._check_version = PbsPython_check_version PbsPython.complete = PbsPython_complete PbsPython.connect_to_server = PbsPython_connect_to_server PbsPython.ppn = PbsPython_ppn pbs_python.PbsJob = pbs_python_PbsJob
if not build_option('force'): _log.debug("Skipping easyconfigs from %s that already have a module available..." % easyconfigs) easyconfigs = skip_available(easyconfigs, modtool) _log.debug("Retained easyconfigs after skipping: %s" % easyconfigs) if build_option('sequential'): return build_easyconfigs(easyconfigs, output_dir, test_results) else: resolved = resolve_dependencies(easyconfigs, modtool) cmd = "eb %(spec)s --regtest --sequential -ld --testoutput=%(output_dir)s" command = "unset TMPDIR && cd %s && %s; " % (cur_dir, cmd) # retry twice in case of failure, to avoid fluke errors command += "if [ $? -ne 0 ]; then %(cmd)s --force && %(cmd)s --force; fi" % {'cmd': cmd} build_easyconfigs_in_parallel(command, resolved, output_dir=output_dir) _log.info("Submitted regression test as jobs, results in %s" % output_dir) return True # success def session_state(): """Get session state: timestamp, dump of environment, system info.""" return { 'time': gmtime(), 'environment': copy.deepcopy(os.environ), 'system_info': get_system_info(), }
# the options to ignore (help options can't reach here) ignore_opts = ['robot', 'job'] # generate_cmd_line returns the options in form --longopt=value opts = [ x for x in eb_go.generate_cmd_line() if not x.split('=')[0] in ['--%s' % y for y in ignore_opts] ] quoted_opts = subprocess.list2cmdline(opts) command = "unset TMPDIR && cd %s && eb %%(spec)s %s" % (curdir, quoted_opts) _log.info("Command template for jobs: %s" % command) if not testing: jobs = build_easyconfigs_in_parallel(command, ordered_ecs) txt = ["List of submitted jobs:"] txt.extend([ "%s (%s): %s" % (job.name, job.module, job.jobid) for job in jobs ]) txt.append("(%d jobs submitted)" % len(jobs)) print_msg("Submitted parallel build jobs, exiting now: %s" % '\n'.join(txt), log=_log) cleanup(logfile, eb_tmpdir, testing) sys.exit(0) # build software, will exit when errors occurs (except when testing) exit_on_failure = not options.dump_test_report and not options.upload_test_report
# submit build as job(s) and exit if options.job: curdir = os.getcwd() # the options to ignore (help options can't reach here) ignore_opts = ['robot', 'job'] # generate_cmd_line returns the options in form --longopt=value opts = [x for x in eb_go.generate_cmd_line() if not x.split('=')[0] in ['--%s' % y for y in ignore_opts]] quoted_opts = subprocess.list2cmdline(opts) command = "unset TMPDIR && cd %s && eb %%(spec)s %s" % (curdir, quoted_opts) _log.info("Command template for jobs: %s" % command) if not testing: jobs = parbuild.build_easyconfigs_in_parallel(command, orderedSpecs, "easybuild-build", robot_path=options.robot) txt = ["List of submitted jobs:"] txt.extend(["%s: %s" % (job.name, job.jobid) for job in jobs]) txt.append("(%d jobs submitted)" % len(jobs)) msg = "\n".join(txt) _log.info("Submitted parallel build jobs, exiting now (%s)." % msg) print msg cleanup_logfile_and_exit(logfile, testing, True) sys.exit(0) # build software, will exit when errors occurs (except when regtesting) correct_built_cnt = 0 all_built_cnt = 0
if options.job: curdir = os.getcwd() # the options to ignore (help options can't reach here) ignore_opts = ["robot", "job"] # generate_cmd_line returns the options in form --longopt=value opts = [x for x in eb_go.generate_cmd_line() if not x.split("=")[0] in ["--%s" % y for y in ignore_opts]] quoted_opts = subprocess.list2cmdline(opts) command = "unset TMPDIR && cd %s && eb %%(spec)s %s" % (curdir, quoted_opts) _log.info("Command template for jobs: %s" % command) if not testing: jobs = parbuild.build_easyconfigs_in_parallel( command, orderedSpecs, "easybuild-build", robot_path=options.robot ) txt = ["List of submitted jobs:"] txt.extend(["%s: %s" % (job.name, job.jobid) for job in jobs]) txt.append("(%d jobs submitted)" % len(jobs)) msg = "\n".join(txt) _log.info("Submitted parallel build jobs, exiting now (%s)." % msg) print msg cleanup_logfile_and_exit(logfile, testing, True) sys.exit(0) # build software, will exit when errors occurs (except when regtesting) correct_built_cnt = 0
def test_build_easyconfigs_in_parallel_pbs_python(self): """Test build_easyconfigs_in_parallel(), using (mocked) pbs_python as backend for --job.""" # put mocked functions in place PbsPython__init__ = PbsPython.__init__ PbsPython_check_version = PbsPython._check_version PbsPython_complete = PbsPython.complete PbsPython_connect_to_server = PbsPython.connect_to_server PbsPython_ppn = PbsPython.ppn pbs_python_PbsJob = pbs_python.PbsJob PbsPython.__init__ = lambda self: PbsPython__init__(self, pbs_server='localhost') PbsPython._check_version = lambda _: True PbsPython.complete = mock PbsPython.connect_to_server = mock PbsPython.ppn = mock pbs_python.PbsJob = MockPbsJob build_options = { 'external_modules_metadata': {}, 'robot_path': os.path.join(os.path.dirname(__file__), 'easyconfigs'), 'valid_module_classes': config.module_classes(), 'validate': False, 'job_cores': 3, } init_config(args=['--job-backend=PbsPython'], build_options=build_options) ec_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'gzip-1.5-goolf-1.4.10.eb') easyconfigs = process_easyconfig(ec_file) ordered_ecs = resolve_dependencies(easyconfigs) jobs = build_easyconfigs_in_parallel("echo '%(spec)s'", ordered_ecs, prepare_first=False) self.assertEqual(len(jobs), 8) regex = re.compile("echo '.*/gzip-1.5-goolf-1.4.10.eb'") self.assertTrue(regex.search(jobs[-1].script), "Pattern '%s' found in: %s" % (regex.pattern, jobs[-1].script)) ec_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'gzip-1.4-GCC-4.6.3.eb') ordered_ecs = resolve_dependencies(process_easyconfig(ec_file), retain_all_deps=True) jobs = submit_jobs(ordered_ecs, '', testing=False, prepare_first=False) # make sure command is correct, and that --hidden is there when it needs to be for i, ec in enumerate(ordered_ecs): if ec['hidden']: regex = re.compile("eb %s.* --hidden" % ec['spec']) else: regex = re.compile("eb %s" % ec['spec']) self.assertTrue(regex.search(jobs[i].script), "Pattern '%s' found in: %s" % (regex.pattern, jobs[i].script)) for job in jobs: self.assertEqual(job.cores, build_options['job_cores']) # no deps for GCC/4.6.3 (toolchain) and ictce/4.1.13 (test easyconfig with 'fake' deps) self.assertEqual(len(jobs[0].deps), 0) self.assertEqual(len(jobs[1].deps), 0) # only dependency for toy/0.0-deps is ictce/4.1.13 (dep marked as external module is filtered out) self.assertTrue('toy-0.0-deps.eb' in jobs[2].script) self.assertEqual(len(jobs[2].deps), 1) self.assertTrue('ictce-4.1.13.eb' in jobs[2].deps[0].script) # dependencies for gzip/1.4-GCC-4.6.3: GCC/4.6.3 (toolchain) + toy/.0.0-deps self.assertTrue('gzip-1.4-GCC-4.6.3.eb' in jobs[3].script) self.assertEqual(len(jobs[3].deps), 2) regex = re.compile('toy-0.0-deps.eb\s* --hidden') self.assertTrue(regex.search(jobs[3].deps[0].script)) self.assertTrue('GCC-4.6.3.eb' in jobs[3].deps[1].script) # restore mocked stuff PbsPython.__init__ = PbsPython__init__ PbsPython._check_version = PbsPython_check_version PbsPython.complete = PbsPython_complete PbsPython.connect_to_server = PbsPython_connect_to_server PbsPython.ppn = PbsPython_ppn pbs_python.PbsJob = pbs_python_PbsJob
def test_build_easyconfigs_in_parallel_gc3pie(self): """Test build_easyconfigs_in_parallel(), using GC3Pie with local config as backend for --job.""" try: import gc3libs # noqa (ignore unused import) except ImportError: print("GC3Pie not available, skipping test") return # put GC3Pie config in place to use local host and fork/exec resourcedir = os.path.join(self.test_prefix, 'gc3pie') gc3pie_cfgfile = os.path.join(self.test_prefix, 'gc3pie_local.ini') gc3pie_cfgtxt = GC3PIE_LOCAL_CONFIGURATION % { 'resourcedir': resourcedir, 'time': which('time'), } write_file(gc3pie_cfgfile, gc3pie_cfgtxt) output_dir = os.path.join(self.test_prefix, 'subdir', 'gc3pie_output_dir') # purposely pre-create output dir, and put a file in it (to check whether GC3Pie tries to rename the output dir) mkdir(output_dir, parents=True) write_file(os.path.join(output_dir, 'foo'), 'bar') # remove write permissions on parent dir of specified output dir, # to check that GC3Pie does not try to rename the (already existing) output directory... adjust_permissions(os.path.dirname(output_dir), stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH, add=False, recursive=False) topdir = os.path.dirname(os.path.abspath(__file__)) build_options = { 'job_backend_config': gc3pie_cfgfile, 'job_max_walltime': 24, 'job_output_dir': output_dir, 'job_polling_interval': 0.2, # quick polling 'job_target_resource': 'ebtestlocalhost', 'robot_path': os.path.join(topdir, 'easyconfigs', 'test_ecs'), 'silent': True, 'valid_module_classes': config.module_classes(), 'validate': False, } init_config(args=['--job-backend=GC3Pie'], build_options=build_options) ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') easyconfigs = process_easyconfig(ec_file) ordered_ecs = resolve_dependencies(easyconfigs, self.modtool) topdir = os.path.dirname( os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) test_easyblocks_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), 'sandbox') cmd = "PYTHONPATH=%s:%s:$PYTHONPATH eb %%(spec)s -df" % ( topdir, test_easyblocks_path) build_easyconfigs_in_parallel(cmd, ordered_ecs, prepare_first=False) toy_modfile = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0') if get_module_syntax() == 'Lua': toy_modfile += '.lua' self.assertTrue(os.path.exists(toy_modfile)) self.assertTrue( os.path.exists( os.path.join(self.test_installpath, 'software', 'toy', '0.0', 'bin', 'toy'))) # also check what happens when a job fails (an error should be raised) test_ecfile = os.path.join(self.test_prefix, 'test.eb') ectxt = read_file(ec_file) # use different version, for which no sources are available regex = re.compile('^version = .*', re.M) ectxt = regex.sub("version = '1.2.3'", ectxt) write_file(test_ecfile, ectxt) ecs = resolve_dependencies(process_easyconfig(test_ecfile), self.modtool) error = "1 jobs failed: toy-1.2.3" self.assertErrorRegex(EasyBuildError, error, build_easyconfigs_in_parallel, cmd, ecs, prepare_first=False)
def test_build_easyconfigs_in_parallel_pbs_python(self): """Test build_easyconfigs_in_parallel(), using (mocked) pbs_python as backend for --job.""" # put mocked functions in place PbsPython__init__ = PbsPython.__init__ PbsPython_check_version = PbsPython._check_version PbsPython_complete = PbsPython.complete PbsPython_connect_to_server = PbsPython.connect_to_server PbsPython_ppn = PbsPython.ppn pbs_python_PbsJob = pbs_python.PbsJob PbsPython.__init__ = lambda self: PbsPython__init__( self, pbs_server='localhost') PbsPython._check_version = lambda _: True PbsPython.complete = mock PbsPython.connect_to_server = mock PbsPython.ppn = mock pbs_python.PbsJob = MockPbsJob topdir = os.path.dirname(os.path.abspath(__file__)) build_options = { 'external_modules_metadata': {}, 'robot_path': os.path.join(topdir, 'easyconfigs', 'test_ecs'), 'valid_module_classes': config.module_classes(), 'validate': False, 'job_cores': 3, } init_config(args=['--job-backend=PbsPython'], build_options=build_options) ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'g', 'gzip', 'gzip-1.5-foss-2018a.eb') easyconfigs = process_easyconfig(ec_file) ordered_ecs = resolve_dependencies(easyconfigs, self.modtool) jobs = build_easyconfigs_in_parallel("echo '%(spec)s'", ordered_ecs, prepare_first=False) # only one job submitted since foss/2018a module is already available self.assertEqual(len(jobs), 1) regex = re.compile("echo '.*/gzip-1.5-foss-2018a.eb'") self.assertTrue( regex.search(jobs[-1].script), "Pattern '%s' found in: %s" % (regex.pattern, jobs[-1].script)) ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'g', 'gzip', 'gzip-1.4-GCC-4.6.3.eb') ordered_ecs = resolve_dependencies(process_easyconfig(ec_file), self.modtool, retain_all_deps=True) jobs = submit_jobs(ordered_ecs, '', testing=False, prepare_first=False) # make sure command is correct, and that --hidden is there when it needs to be for i, ec in enumerate(ordered_ecs): if ec['hidden']: regex = re.compile("eb %s.* --hidden" % ec['spec']) else: regex = re.compile("eb %s" % ec['spec']) self.assertTrue( regex.search(jobs[i].script), "Pattern '%s' found in: %s" % (regex.pattern, jobs[i].script)) for job in jobs: self.assertEqual(job.cores, build_options['job_cores']) # no deps for GCC/4.6.3 (toolchain) and intel/2018a (test easyconfig with 'fake' deps) self.assertEqual(len(jobs[0].deps), 0) self.assertEqual(len(jobs[1].deps), 0) # only dependency for toy/0.0-deps is intel/2018a (dep marked as external module is filtered out) self.assertTrue('toy-0.0-deps.eb' in jobs[2].script) self.assertEqual(len(jobs[2].deps), 1) self.assertTrue('intel-2018a.eb' in jobs[2].deps[0].script) # dependencies for gzip/1.4-GCC-4.6.3: GCC/4.6.3 (toolchain) + toy/.0.0-deps self.assertTrue('gzip-1.4-GCC-4.6.3.eb' in jobs[3].script) self.assertEqual(len(jobs[3].deps), 2) regex = re.compile(r'toy-0.0-deps\.eb.* --hidden') script_txt = jobs[3].deps[0].script fail_msg = "Pattern '%s' should be found in: %s" % (regex.pattern, script_txt) self.assertTrue(regex.search(script_txt), fail_msg) self.assertTrue('GCC-4.6.3.eb' in jobs[3].deps[1].script) # also test use of --pre-create-installdir ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') ordered_ecs = resolve_dependencies(process_easyconfig(ec_file), self.modtool) # installation directory doesn't exist yet before submission toy_installdir = os.path.join(self.test_installpath, 'software', 'toy', '0.0') self.assertFalse(os.path.exists(toy_installdir)) jobs = submit_jobs(ordered_ecs, '', testing=False) self.assertEqual(len(jobs), 1) # software install dir is created (by default) as part of job submission process (fetch_step is run) self.assertTrue(os.path.exists(toy_installdir)) remove_dir(toy_installdir) remove_dir(os.path.dirname(toy_installdir)) self.assertFalse(os.path.exists(toy_installdir)) # installation directory does *not* get created when --pre-create-installdir is used build_options['pre_create_installdir'] = False init_config(args=['--job-backend=PbsPython'], build_options=build_options) jobs = submit_jobs(ordered_ecs, '', testing=False) self.assertEqual(len(jobs), 1) self.assertFalse(os.path.exists(toy_installdir)) # restore mocked stuff PbsPython.__init__ = PbsPython__init__ PbsPython._check_version = PbsPython_check_version PbsPython.complete = PbsPython_complete PbsPython.connect_to_server = PbsPython_connect_to_server PbsPython.ppn = PbsPython_ppn pbs_python.PbsJob = pbs_python_PbsJob
def test_build_easyconfigs_in_parallel_pbs_python(self): """Test build_easyconfigs_in_parallel(), using (mocked) pbs_python as backend for --job.""" # put mocked functions in place PbsPython__init__ = PbsPython.__init__ PbsPython_check_version = PbsPython._check_version PbsPython_complete = PbsPython.complete PbsPython_connect_to_server = PbsPython.connect_to_server PbsPython_ppn = PbsPython.ppn pbs_python_PbsJob = pbs_python.PbsJob PbsPython.__init__ = lambda self: PbsPython__init__( self, pbs_server='localhost') PbsPython._check_version = lambda _: True PbsPython.complete = mock PbsPython.connect_to_server = mock PbsPython.ppn = mock pbs_python.PbsJob = MockPbsJob topdir = os.path.dirname(os.path.abspath(__file__)) build_options = { 'external_modules_metadata': {}, 'robot_path': os.path.join(topdir, 'easyconfigs', 'test_ecs'), 'valid_module_classes': config.module_classes(), 'validate': False, 'job_cores': 3, } init_config(args=['--job-backend=PbsPython'], build_options=build_options) ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'g', 'gzip', 'gzip-1.5-goolf-1.4.10.eb') easyconfigs = process_easyconfig(ec_file) ordered_ecs = resolve_dependencies(easyconfigs, self.modtool) jobs = build_easyconfigs_in_parallel("echo '%(spec)s'", ordered_ecs, prepare_first=False) self.assertEqual(len(jobs), 8) regex = re.compile("echo '.*/gzip-1.5-goolf-1.4.10.eb'") self.assertTrue( regex.search(jobs[-1].script), "Pattern '%s' found in: %s" % (regex.pattern, jobs[-1].script)) ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'g', 'gzip', 'gzip-1.4-GCC-4.6.3.eb') ordered_ecs = resolve_dependencies(process_easyconfig(ec_file), self.modtool, retain_all_deps=True) jobs = submit_jobs(ordered_ecs, '', testing=False, prepare_first=False) # make sure command is correct, and that --hidden is there when it needs to be for i, ec in enumerate(ordered_ecs): if ec['hidden']: regex = re.compile("eb %s.* --hidden" % ec['spec']) else: regex = re.compile("eb %s" % ec['spec']) self.assertTrue( regex.search(jobs[i].script), "Pattern '%s' found in: %s" % (regex.pattern, jobs[i].script)) for job in jobs: self.assertEqual(job.cores, build_options['job_cores']) # no deps for GCC/4.6.3 (toolchain) and ictce/4.1.13 (test easyconfig with 'fake' deps) self.assertEqual(len(jobs[0].deps), 0) self.assertEqual(len(jobs[1].deps), 0) # only dependency for toy/0.0-deps is ictce/4.1.13 (dep marked as external module is filtered out) self.assertTrue('toy-0.0-deps.eb' in jobs[2].script) self.assertEqual(len(jobs[2].deps), 1) self.assertTrue('ictce-4.1.13.eb' in jobs[2].deps[0].script) # dependencies for gzip/1.4-GCC-4.6.3: GCC/4.6.3 (toolchain) + toy/.0.0-deps self.assertTrue('gzip-1.4-GCC-4.6.3.eb' in jobs[3].script) self.assertEqual(len(jobs[3].deps), 2) regex = re.compile('toy-0.0-deps.eb\s* --hidden') self.assertTrue(regex.search(jobs[3].deps[0].script)) self.assertTrue('GCC-4.6.3.eb' in jobs[3].deps[1].script) # restore mocked stuff PbsPython.__init__ = PbsPython__init__ PbsPython._check_version = PbsPython_check_version PbsPython.complete = PbsPython_complete PbsPython.connect_to_server = PbsPython_connect_to_server PbsPython.ppn = PbsPython_ppn pbs_python.PbsJob = pbs_python_PbsJob
def test_build_easyconfigs_in_parallel_gc3pie(self): """Test build_easyconfigs_in_parallel(), using GC3Pie with local config as backend for --job.""" try: import gc3libs except ImportError: print "GC3Pie not available, skipping test" return # put GC3Pie config in place to use local host and fork/exec resourcedir = os.path.join(self.test_prefix, 'gc3pie') gc3pie_cfgfile = os.path.join(self.test_prefix, 'gc3pie_local.ini') gc3pie_cfgtxt = GC3PIE_LOCAL_CONFIGURATION % { 'resourcedir': resourcedir, 'time': which('time'), } write_file(gc3pie_cfgfile, gc3pie_cfgtxt) output_dir = os.path.join(self.test_prefix, 'subdir', 'gc3pie_output_dir') # purposely pre-create output dir, and put a file in it (to check whether GC3Pie tries to rename the output dir) mkdir(output_dir, parents=True) write_file(os.path.join(output_dir, 'foo'), 'bar') # remove write permissions on parent dir of specified output dir, # to check that GC3Pie does not try to rename the (already existing) output directory... adjust_permissions(os.path.dirname(output_dir), stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH, add=False, recursive=False) topdir = os.path.dirname(os.path.abspath(__file__)) build_options = { 'job_backend_config': gc3pie_cfgfile, 'job_max_walltime': 24, 'job_output_dir': output_dir, 'job_polling_interval': 0.2, # quick polling 'job_target_resource': 'ebtestlocalhost', 'robot_path': os.path.join(topdir, 'easyconfigs', 'test_ecs'), 'silent': True, 'valid_module_classes': config.module_classes(), 'validate': False, } options = init_config(args=['--job-backend=GC3Pie'], build_options=build_options) ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') easyconfigs = process_easyconfig(ec_file) ordered_ecs = resolve_dependencies(easyconfigs, self.modtool) topdir = os.path.dirname( os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) test_easyblocks_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), 'sandbox') cmd = "PYTHONPATH=%s:%s:$PYTHONPATH eb %%(spec)s -df" % ( topdir, test_easyblocks_path) jobs = build_easyconfigs_in_parallel(cmd, ordered_ecs, prepare_first=False) self.assertTrue( os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0')) self.assertTrue( os.path.join(self.test_installpath, 'software', 'toy', '0.0', 'bin', 'toy'))
def regtest(easyconfig_paths, modtool, build_specs=None): """ Run regression test, using easyconfigs available in given path :param easyconfig_paths: path of easyconfigs to run regtest on :param modtool: ModulesTool instance to use :param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...) """ cur_dir = os.getcwd() aggregate_regtest = build_option('aggregate_regtest') if aggregate_regtest is not None: output_file = os.path.join( aggregate_regtest, "%s-aggregate.xml" % os.path.basename(aggregate_regtest)) aggregate_xml_in_dirs(aggregate_regtest, output_file) _log.info("aggregated xml files inside %s, output written to: %s" % (aggregate_regtest, output_file)) sys.exit(0) # create base directory, which is used to place all log files and the test output as xml regtest_output_dir = build_option('regtest_output_dir') testoutput = build_option('testoutput') if regtest_output_dir is not None: output_dir = regtest_output_dir elif testoutput is not None: output_dir = os.path.abspath(testoutput) else: # default: current dir + easybuild-test-[timestamp] dirname = "easybuild-test-%s" % datetime.now().strftime("%Y%m%d%H%M%S") output_dir = os.path.join(cur_dir, dirname) mkdir(output_dir, parents=True) # find all easyconfigs ecfiles = [] if easyconfig_paths: for path in easyconfig_paths: ecfiles += find_easyconfigs( path, ignore_dirs=build_option('ignore_dirs')) else: raise EasyBuildError("No easyconfig paths specified.") test_results = [] # process all the found easyconfig files easyconfigs = [] for ecfile in ecfiles: try: easyconfigs.extend( process_easyconfig(ecfile, build_specs=build_specs)) except EasyBuildError as err: test_results.append((ecfile, 'parsing_easyconfigs', 'easyconfig file error: %s' % err, _log)) # skip easyconfigs for which a module is already available, unless forced if not build_option('force'): _log.debug( "Skipping easyconfigs from %s that already have a module available..." % easyconfigs) easyconfigs = skip_available(easyconfigs, modtool) _log.debug("Retained easyconfigs after skipping: %s" % easyconfigs) if build_option('sequential'): return build_easyconfigs(easyconfigs, output_dir, test_results) else: resolved = resolve_dependencies(easyconfigs, modtool) cmd = "eb %(spec)s --regtest --sequential -ld --testoutput=%(output_dir)s" command = "unset TMPDIR && cd %s && %s; " % (cur_dir, cmd) # retry twice in case of failure, to avoid fluke errors command += "if [ $? -ne 0 ]; then %(cmd)s --force && %(cmd)s --force; fi" % { 'cmd': cmd } build_easyconfigs_in_parallel(command, resolved, output_dir=output_dir) _log.info("Submitted regression test as jobs, results in %s" % output_dir) return True # success
# submit build as job(s) and exit if options.job: curdir = os.getcwd() # the options to ignore (help options can't reach here) ignore_opts = ['robot', 'job'] # generate_cmd_line returns the options in form --longopt=value opts = [x for x in eb_go.generate_cmd_line() if not x.split('=')[0] in ['--%s' % y for y in ignore_opts]] quoted_opts = subprocess.list2cmdline(opts) command = "unset TMPDIR && cd %s && eb %%(spec)s %s" % (curdir, quoted_opts) _log.info("Command template for jobs: %s" % command) if not testing: jobs = build_easyconfigs_in_parallel(command, ordered_ecs, build_options=build_options, build_specs=build_specs) txt = ["List of submitted jobs:"] txt.extend(["%s (%s): %s" % (job.name, job.module, job.jobid) for job in jobs]) txt.append("(%d jobs submitted)" % len(jobs)) msg = "\n".join(txt) _log.info("Submitted parallel build jobs, exiting now (%s)." % msg) print msg cleanup(logfile, eb_tmpdir, testing) sys.exit(0) # build software, will exit when errors occurs (except when regtesting) correct_built_cnt = 0 all_built_cnt = 0
def test_build_easyconfigs_in_parallel(self): """Basic test for build_easyconfigs_in_parallel function.""" easyconfig_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'gzip-1.5-goolf-1.4.10.eb') easyconfigs = process_easyconfig(easyconfig_file, validate=False) ordered_ecs = resolve_dependencies(easyconfigs) build_easyconfigs_in_parallel("echo %(spec)s", ordered_ecs)
_log.debug("Retained easyconfigs after skipping: %s" % easyconfigs) if build_option('sequential'): return build_easyconfigs(easyconfigs, output_dir, test_results) else: resolved = resolve_dependencies(easyconfigs, build_specs=build_specs) cmd = "eb %(spec)s --regtest --sequential -ld --testoutput=%(output_dir)s" command = "unset TMPDIR && cd %s && %s; " % (cur_dir, cmd) # retry twice in case of failure, to avoid fluke errors command += "if [ $? -ne 0 ]; then %(cmd)s --force && %(cmd)s --force; fi" % { 'cmd': cmd } jobs = build_easyconfigs_in_parallel(command, resolved, output_dir=output_dir) print "List of submitted jobs:" for job in jobs: print "%s: %s" % (job.name, job.jobid) print "(%d jobs submitted)" % len(jobs) # determine leaf nodes in dependency graph, and report them all_deps = set() for job in jobs: all_deps = all_deps.union(job.deps) leaf_nodes = [] for job in jobs: if job.jobid not in all_deps:
# submit build as job(s) and exit if options.job: curdir = os.getcwd() # the options to ignore (help options can't reach here) ignore_opts = ['robot', 'job'] # generate_cmd_line returns the options in form --longopt=value opts = [x for x in eb_go.generate_cmd_line() if not x.split('=')[0] in ['--%s' % y for y in ignore_opts]] quoted_opts = subprocess.list2cmdline(opts) command = "unset TMPDIR && cd %s && eb %%(spec)s %s" % (curdir, quoted_opts) _log.info("Command template for jobs: %s" % command) if not testing: jobs = build_easyconfigs_in_parallel(command, ordered_ecs) txt = ["List of submitted jobs:"] txt.extend(["%s (%s): %s" % (job.name, job.module, job.jobid) for job in jobs]) txt.append("(%d jobs submitted)" % len(jobs)) msg = "\n".join(txt) _log.info("Submitted parallel build jobs, exiting now (%s)." % msg) print msg cleanup(logfile, eb_tmpdir, testing) sys.exit(0) # build software, will exit when errors occurs (except when regtesting) correct_built_cnt = 0 all_built_cnt = 0
def regtest(easyconfig_paths, modtool, build_specs=None): """ Run regression test, using easyconfigs available in given path :param easyconfig_paths: path of easyconfigs to run regtest on :param modtool: ModulesTool instance to use :param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...) """ cur_dir = os.getcwd() aggregate_regtest = build_option('aggregate_regtest') if aggregate_regtest is not None: output_file = os.path.join(aggregate_regtest, "%s-aggregate.xml" % os.path.basename(aggregate_regtest)) aggregate_xml_in_dirs(aggregate_regtest, output_file) _log.info("aggregated xml files inside %s, output written to: %s" % (aggregate_regtest, output_file)) sys.exit(0) # create base directory, which is used to place all log files and the test output as xml regtest_output_dir = build_option('regtest_output_dir') testoutput = build_option('testoutput') if regtest_output_dir is not None: output_dir = regtest_output_dir elif testoutput is not None: output_dir = os.path.abspath(testoutput) else: # default: current dir + easybuild-test-[timestamp] dirname = "easybuild-test-%s" % datetime.now().strftime("%Y%m%d%H%M%S") output_dir = os.path.join(cur_dir, dirname) mkdir(output_dir, parents=True) # find all easyconfigs ecfiles = [] if easyconfig_paths: for path in easyconfig_paths: ecfiles += find_easyconfigs(path, ignore_dirs=build_option('ignore_dirs')) else: raise EasyBuildError("No easyconfig paths specified.") test_results = [] # process all the found easyconfig files easyconfigs = [] for ecfile in ecfiles: try: easyconfigs.extend(process_easyconfig(ecfile, build_specs=build_specs)) except EasyBuildError as err: test_results.append((ecfile, 'parsing_easyconfigs', 'easyconfig file error: %s' % err, _log)) # skip easyconfigs for which a module is already available, unless forced if not build_option('force'): _log.debug("Skipping easyconfigs from %s that already have a module available..." % easyconfigs) easyconfigs = skip_available(easyconfigs, modtool) _log.debug("Retained easyconfigs after skipping: %s" % easyconfigs) if build_option('sequential'): return build_easyconfigs(easyconfigs, output_dir, test_results) else: resolved = resolve_dependencies(easyconfigs, modtool) cmd = "eb %(spec)s --regtest --sequential -ld --testoutput=%(output_dir)s" command = "unset TMPDIR && cd %s && %s; " % (cur_dir, cmd) # retry twice in case of failure, to avoid fluke errors command += "if [ $? -ne 0 ]; then %(cmd)s --force && %(cmd)s --force; fi" % {'cmd': cmd} build_easyconfigs_in_parallel(command, resolved, output_dir=output_dir) _log.info("Submitted regression test as jobs, results in %s" % output_dir) return True # success