def suite(): """Return all easyblock --module-only tests.""" # initialize configuration (required for e.g. default modules_tool setting) cleanup() eb_go = eboptions.parse_options(args=['--prefix=%s' % TMPDIR]) config.init(eb_go.options, eb_go.get_options_by_section('config')) build_options = { # enable --force --module-only 'force': True, 'module_only': True, 'silent': True, 'suffix_modules_path': GENERAL_CLASS, 'valid_module_classes': config.module_classes(), 'valid_stops': [x[0] for x in EasyBlock.get_steps()], } config.init_build_options(build_options=build_options) config.set_tmpdir() # dynamically generate a separate test for each of the available easyblocks easyblocks_path = get_paths_for("easyblocks")[0] all_pys = glob.glob('%s/*/*.py' % easyblocks_path) easyblocks = [eb for eb in all_pys if os.path.basename(eb) != '__init__.py' and '/test/' not in eb] # filter out no longer supported easyblocks easyblocks = [e for e in easyblocks if os.path.basename(e) not in ['versionindependendpythonpackage.py']] for easyblock in easyblocks: # dynamically define new inner functions that can be added as class methods to ModuleOnlyTest exec("def innertest(self): template_module_only_test(self, '%s')" % easyblock) innertest.__doc__ = "Test for using --module-only with easyblock %s" % easyblock innertest.__name__ = "test_easyblock_%s" % '_'.join(easyblock.replace('.py', '').split('/')) setattr(ModuleOnlyTest, innertest.__name__, innertest) return TestLoader().loadTestsFromTestCase(ModuleOnlyTest)
def suite(): """Return all easyblock --module-only tests.""" # initialize configuration (required for e.g. default modules_tool setting) cleanup() eb_go = eboptions.parse_options(args=['--prefix=%s' % TMPDIR]) config.init(eb_go.options, eb_go.get_options_by_section('config')) build_options = { # enable --force --module-only 'force': True, 'module_only': True, 'silent': True, 'suffix_modules_path': GENERAL_CLASS, 'valid_module_classes': config.module_classes(), 'valid_stops': [x[0] for x in EasyBlock.get_steps()], } config.init_build_options(build_options=build_options) config.set_tmpdir() # dynamically generate a separate test for each of the available easyblocks easyblocks_path = get_paths_for("easyblocks")[0] all_pys = glob.glob('%s/*/*.py' % easyblocks_path) easyblocks = [ eb for eb in all_pys if os.path.basename(eb) != '__init__.py' and '/test/' not in eb ] # filter out no longer supported easyblocks, or easyblocks that are tested in a different way excluded_easyblocks = ['versionindependendpythonpackage.py'] easyblocks = [ e for e in easyblocks if os.path.basename(e) not in excluded_easyblocks ] for easyblock in easyblocks: # dynamically define new inner functions that can be added as class methods to ModuleOnlyTest if os.path.basename(easyblock) == 'systemcompiler.py': # use GCC as name when testing SystemCompiler easyblock exec( "def innertest(self): template_module_only_test(self, '%s', name='GCC', version='system')" % easyblock) else: exec("def innertest(self): template_module_only_test(self, '%s')" % easyblock) innertest.__doc__ = "Test for using --module-only with easyblock %s" % easyblock innertest.__name__ = "test_easyblock_%s" % '_'.join( easyblock.replace('.py', '').split('/')) setattr(ModuleOnlyTest, innertest.__name__, innertest) return TestLoader().loadTestsFromTestCase(ModuleOnlyTest)
def test_set_tmpdir(self): """Test set_tmpdir config function.""" self.purge_environment() for tmpdir in [None, os.path.join(tempfile.gettempdir(), 'foo')]: parent = tmpdir if parent is None: parent = tempfile.gettempdir() mytmpdir = set_tmpdir(tmpdir=tmpdir) for var in ['TMPDIR', 'TEMP', 'TMP']: self.assertTrue(os.environ[var].startswith( os.path.join(parent, 'easybuild-'))) self.assertEqual(os.environ[var], mytmpdir) self.assertTrue(tempfile.gettempdir().startswith( os.path.join(parent, 'easybuild-'))) tempfile_tmpdir = tempfile.mkdtemp() self.assertTrue( tempfile_tmpdir.startswith(os.path.join(parent, 'easybuild-'))) fd, tempfile_tmpfile = tempfile.mkstemp() self.assertTrue( tempfile_tmpfile.startswith(os.path.join(parent, 'easybuild-'))) # cleanup os.close(fd) shutil.rmtree(mytmpdir) modify_env(os.environ, self.orig_environ) tempfile.tempdir = None
def test_set_tmpdir(self): """Test set_tmpdir config function.""" self.purge_environment() for tmpdir in [None, os.path.join(tempfile.gettempdir(), 'foo')]: parent = tmpdir if parent is None: parent = tempfile.gettempdir() mytmpdir = set_tmpdir(tmpdir=tmpdir) for var in ['TMPDIR', 'TEMP', 'TMP']: self.assertTrue(os.environ[var].startswith(os.path.join(parent, 'easybuild-'))) self.assertEqual(os.environ[var], mytmpdir) self.assertTrue(tempfile.gettempdir().startswith(os.path.join(parent, 'easybuild-'))) tempfile_tmpdir = tempfile.mkdtemp() self.assertTrue(tempfile_tmpdir.startswith(os.path.join(parent, 'easybuild-'))) fd, tempfile_tmpfile = tempfile.mkstemp() self.assertTrue(tempfile_tmpfile.startswith(os.path.join(parent, 'easybuild-'))) # cleanup os.close(fd) shutil.rmtree(mytmpdir) modify_env(os.environ, self.orig_environ) tempfile.tempdir = None
class EasyConfigTest(TestCase): """Baseclass for easyconfig testcases.""" if LooseVersion(sys.version) >= LooseVersion('2.6'): os.environ['EASYBUILD_DEPRECATED'] = '2.0' # initialize configuration (required for e.g. default modules_tool setting) eb_go = eboptions.parse_options() config.init(eb_go.options, eb_go.get_options_by_section('config')) build_options = { 'check_osdeps': False, 'force': True, 'robot_path': get_paths_for("easyconfigs")[0], 'suffix_modules_path': GENERAL_CLASS, 'valid_module_classes': config.module_classes(), 'valid_stops': [x[0] for x in EasyBlock.get_steps()], } config.init_build_options(build_options=build_options) config.set_tmpdir() del eb_go log = fancylogger.getLogger("EasyConfigTest", fname=False) # make sure a logger is present for main main._log = log ordered_specs = None parsed_easyconfigs = [] def process_all_easyconfigs(self): """Process all easyconfigs and resolve inter-easyconfig dependencies.""" # all available easyconfig files easyconfigs_path = get_paths_for("easyconfigs")[0] specs = glob.glob('%s/*/*/*.eb' % easyconfigs_path) # parse all easyconfigs if they haven't been already if not self.parsed_easyconfigs: for spec in specs: self.parsed_easyconfigs.extend(process_easyconfig(spec)) self.ordered_specs = resolve_dependencies(self.parsed_easyconfigs) def test_dep_graph(self): """Unit test that builds a full dependency graph.""" # pygraph dependencies required for constructing dependency graph are not available prior to Python 2.6 if LooseVersion( sys.version) >= LooseVersion('2.6') and single_tests_ok: # temporary file for dep graph (hn, fn) = tempfile.mkstemp(suffix='.dot') os.close(hn) if self.ordered_specs is None: self.process_all_easyconfigs() dep_graph(fn, self.ordered_specs, silent=True) try: os.remove(fn) except OSError, err: log.error("Failed to remove %s: %s" % (fn, err)) else:
def postprocess(self): """Do some postprocessing, in particular print stuff""" build_log.EXPERIMENTAL = self.options.experimental # set strictness of run module if self.options.strict: run.strictness = self.options.strict # override current version of EasyBuild with version specified to --deprecated if self.options.deprecated: build_log.CURRENT_VERSION = LooseVersion(self.options.deprecated) # log to specified value of --unittest-file if self.options.unittest_file: fancylogger.logToFile(self.options.unittest_file) # set tmpdir self.tmpdir = set_tmpdir(self.options.tmpdir) # take --include options into account self._postprocess_include() # prepare for --list/--avail if any([self.options.avail_easyconfig_params, self.options.avail_easyconfig_templates, self.options.list_easyblocks, self.options.list_toolchains, self.options.avail_cfgfile_constants, self.options.avail_easyconfig_constants, self.options.avail_easyconfig_licenses, self.options.avail_repositories, self.options.show_default_moduleclasses, self.options.avail_modules_tools, self.options.avail_module_naming_schemes, self.options.show_default_configfiles, ]): build_easyconfig_constants_dict() # runs the easyconfig constants sanity check self._postprocess_list_avail() # fail early if required dependencies for functionality requiring using GitHub API are not available: if self.options.from_pr or self.options.upload_test_report: if not HAVE_GITHUB_API: raise EasyBuildError("Required support for using GitHub API is not available (see warnings).") if self.options.module_syntax == ModuleGeneratorLua.SYNTAX and self.options.modules_tool != Lmod.__name__: raise EasyBuildError("Generating Lua module files requires Lmod as modules tool.") # make sure a GitHub token is available when it's required if self.options.upload_test_report: if not HAVE_KEYRING: raise EasyBuildError("Python 'keyring' module required for obtaining GitHub token is not available.") if self.options.github_user is None: raise EasyBuildError("No GitHub user name provided, required for fetching GitHub token.") token = fetch_github_token(self.options.github_user) if token is None: raise EasyBuildError("Failed to obtain required GitHub token for user '%s'", self.options.github_user) # make sure autopep8 is available when it needs to be if self.options.dump_autopep8: if not HAVE_AUTOPEP8: raise EasyBuildError("Python 'autopep8' module required to reformat dumped easyconfigs as requested") self._postprocess_external_modules_metadata() self._postprocess_config()
class EasyConfigTest(TestCase): """Baseclass for easyconfig testcases.""" # initialize configuration (required for e.g. default modules_tool setting) eb_go = eboptions.parse_options() config.init(eb_go.options, eb_go.get_options_by_section('config')) config.set_tmpdir() del eb_go log = fancylogger.getLogger("EasyConfigTest", fname=False) name_regex = re.compile("^name\s*=\s*['\"](.*)['\"]$", re.M) easyblock_regex = re.compile(r"^\s*easyblock\s*=['\"](.*)['\"]$", re.M) # make sure a logger is present for main main._log = log ordered_specs = None def process_all_easyconfigs(self): """Process all easyconfigs and resolve inter-easyconfig dependencies.""" # all available easyconfig files easyconfigs_path = get_paths_for("easyconfigs")[0] specs = glob.glob('%s/*/*/*.eb' % easyconfigs_path) # parse all easyconfigs easyconfigs = [] for spec in specs: easyconfigs.extend( process_easyconfig(spec, build_options={'validate': False})) build_options = { 'robot_path': easyconfigs_path, 'force': True, } self.ordered_specs = resolve_dependencies(easyconfigs, build_options=build_options) def test_dep_graph(self): """Unit test that builds a full dependency graph.""" # pygraph dependencies required for constructing dependency graph are not available prior to Python 2.6 if LooseVersion( sys.version) >= LooseVersion('2.6') and single_tests_ok: # temporary file for dep graph (hn, fn) = tempfile.mkstemp(suffix='.dot') os.close(hn) if self.ordered_specs is None: self.process_all_easyconfigs() dep_graph(fn, self.ordered_specs, silent=True) try: os.remove(fn) except OSError, err: log.error("Failed to remove %s: %s" % (fn, err)) else:
def suite(): """Return all easyblock --module-only tests.""" # initialize configuration (required for e.g. default modules_tool setting) cleanup() eb_go = eboptions.parse_options(args=["--prefix=%s" % TMPDIR]) config.init(eb_go.options, eb_go.get_options_by_section("config")) build_options = { # enable --force --module-only "force": True, "module_only": True, "silent": True, "suffix_modules_path": GENERAL_CLASS, "valid_module_classes": config.module_classes(), "valid_stops": [x[0] for x in EasyBlock.get_steps()], } config.init_build_options(build_options=build_options) config.set_tmpdir() # dynamically generate a separate test for each of the available easyblocks easyblocks_path = get_paths_for("easyblocks")[0] all_pys = glob.glob("%s/*/*.py" % easyblocks_path) easyblocks = [eb for eb in all_pys if os.path.basename(eb) != "__init__.py" and "/test/" not in eb] # filter out no longer supported easyblocks, or easyblocks that are tested in a different way excluded_easyblocks = ["versionindependendpythonpackage.py"] easyblocks = [e for e in easyblocks if os.path.basename(e) not in excluded_easyblocks] for easyblock in easyblocks: # dynamically define new inner functions that can be added as class methods to ModuleOnlyTest if os.path.basename(easyblock) == "systemcompiler.py": # use GCC as name when testing SystemCompiler easyblock exec("def innertest(self): template_module_only_test(self, '%s', name='GCC', version='system')" % easyblock) else: exec("def innertest(self): template_module_only_test(self, '%s')" % easyblock) innertest.__doc__ = "Test for using --module-only with easyblock %s" % easyblock innertest.__name__ = "test_easyblock_%s" % "_".join(easyblock.replace(".py", "").split("/")) setattr(ModuleOnlyTest, innertest.__name__, innertest) return TestLoader().loadTestsFromTestCase(ModuleOnlyTest)
class InitTest(TestCase): """ Baseclass for easyblock testcases """ # initialize configuration (required for e.g. default modules_tool setting) eb_go = eboptions.parse_options() config.init(eb_go.options, eb_go.get_options_by_section('config')) build_options = { 'suffix_modules_path': GENERAL_CLASS, 'valid_module_classes': config.module_classes(), 'valid_stops': [x[0] for x in EasyBlock.get_steps()], } config.init_build_options(build_options=build_options) config.set_tmpdir() del eb_go def writeEC(self, easyblock, extratxt=''): """ create temporary easyconfig file """ txt = '\n'.join([ 'easyblock = "%s"', 'name = "foo"', 'version = "1.3.2"', 'homepage = "http://example.com"', 'description = "Dummy easyconfig file."', 'toolchain = {"name": "dummy", "version": "dummy"}', 'sources = []', extratxt, ]) f = open(self.eb_file, "w") f.write(txt % easyblock) f.close() def setUp(self): """Setup test.""" self.log = fancylogger.getLogger("EasyblocksInitTest", fname=False) fd, self.eb_file = tempfile.mkstemp(prefix='easyblocks_init_test_', suffix='.eb') os.close(fd) def tearDown(self): """Cleanup.""" try: os.remove(self.eb_file) except OSError, err: self.log.error("Failed to remove %s/%s: %s" % (self.eb_file, err))
import test.framework.parallelbuild as p import test.framework.repository as r import test.framework.robot as robot import test.framework.run as run import test.framework.scripts as sc import test.framework.systemtools as s import test.framework.toolchain as tc import test.framework.toolchainvariables as tcv import test.framework.toy_build as t import test.framework.tweak as tw import test.framework.variables as v # make sure temporary files can be created/used try: set_tmpdir(raise_error=True) except EasyBuildError, err: sys.stderr.write("No execution rights on temporary files, specify another location via $TMPDIR: %s\n" % err) sys.exit(1) # initialize logger for all the unit tests fd, log_fn = tempfile.mkstemp(prefix='easybuild-tests-', suffix='.log') os.close(fd) os.remove(log_fn) fancylogger.logToFile(log_fn) log = fancylogger.getLogger() # call suite() for each module and then run them all # note: make sure the options unit tests run first, to avoid running some of them with a readily initialized config tests = [gen, o, r, ef, ev, ebco, ep, e, mg, m, mt, f, run, a, robot, b, v, g, tcv, tc, t, c, s, l, f_c, sc, tw, p]
def setUp(self): """Set up testcase.""" super(EnhancedTestCase, self).setUp() # keep track of log handlers log = fancylogger.getLogger(fname=False) self.orig_log_handlers = log.handlers[:] self.orig_tmpdir = tempfile.gettempdir() # use a subdirectory for this test (which we can clean up easily after the test completes) self.test_prefix = set_tmpdir() self.log = fancylogger.getLogger(self.__class__.__name__, fname=False) fd, self.logfile = tempfile.mkstemp(suffix='.log', prefix='eb-test-') os.close(fd) self.cwd = os.getcwd() # keep track of original environment to restore self.orig_environ = copy.deepcopy(os.environ) # keep track of original environment/Python search path to restore self.orig_sys_path = sys.path[:] testdir = os.path.dirname(os.path.abspath(__file__)) self.test_sourcepath = os.path.join(testdir, 'sandbox', 'sources') os.environ['EASYBUILD_SOURCEPATH'] = self.test_sourcepath os.environ['EASYBUILD_PREFIX'] = self.test_prefix self.test_buildpath = tempfile.mkdtemp() os.environ['EASYBUILD_BUILDPATH'] = self.test_buildpath self.test_installpath = tempfile.mkdtemp() os.environ['EASYBUILD_INSTALLPATH'] = self.test_installpath # make sure that the tests only pick up easyconfigs provided with the tests os.environ['EASYBUILD_ROBOT_PATHS'] = os.path.join(testdir, 'easyconfigs') # make sure no deprecated behaviour is being triggered (unless intended by the test) # trip *all* log.deprecated statements by setting deprecation version ridiculously high self.orig_current_version = eb_build_log.CURRENT_VERSION os.environ['EASYBUILD_DEPRECATED'] = '10000000' init_config() # remove any entries in Python search path that seem to provide easyblocks for path in sys.path[:]: if os.path.exists(os.path.join(path, 'easybuild', 'easyblocks', '__init__.py')): sys.path.remove(path) # add test easyblocks to Python search path and (re)import and reload easybuild modules import easybuild sys.path.append(os.path.join(testdir, 'sandbox')) reload(easybuild) import easybuild.easyblocks reload(easybuild.easyblocks) import easybuild.easyblocks.generic reload(easybuild.easyblocks.generic) reload(easybuild.tools.module_naming_scheme) # required to run options unit tests stand-alone modtool = modules_tool() self.reset_modulepath([os.path.join(testdir, 'modules')]) # purge out any loaded modules with original $MODULEPATH before running each test modtool.purge()
class EasyConfigTest(TestCase): """Baseclass for easyconfig testcases.""" # initialize configuration (required for e.g. default modules_tool setting) eb_go = eboptions.parse_options() config.init(eb_go.options, eb_go.get_options_by_section('config')) build_options = { 'check_osdeps': False, 'external_modules_metadata': {}, 'force': True, 'optarch': 'test', 'robot_path': get_paths_for("easyconfigs")[0], 'suffix_modules_path': GENERAL_CLASS, 'valid_module_classes': config.module_classes(), 'valid_stops': [x[0] for x in EasyBlock.get_steps()], } config.init_build_options(build_options=build_options) config.set_tmpdir() del eb_go # put dummy 'craype-test' module in place, which is required for parsing easyconfigs using Cray* toolchains TMPDIR = tempfile.mkdtemp() os.environ['MODULEPATH'] = TMPDIR write_file(os.path.join(TMPDIR, 'craype-test'), '#%Module\n') log = fancylogger.getLogger("EasyConfigTest", fname=False) # make sure a logger is present for main main._log = log ordered_specs = None parsed_easyconfigs = [] def process_all_easyconfigs(self): """Process all easyconfigs and resolve inter-easyconfig dependencies.""" # all available easyconfig files easyconfigs_path = get_paths_for("easyconfigs")[0] specs = glob.glob('%s/*/*/*.eb' % easyconfigs_path) # parse all easyconfigs if they haven't been already if not self.parsed_easyconfigs: for spec in specs: self.parsed_easyconfigs.extend(process_easyconfig(spec)) # filter out external modules for ec in self.parsed_easyconfigs: for dep in ec['dependencies'][:]: if dep.get('external_module', False): ec['dependencies'].remove(dep) for dep in ec['unresolved_deps'][:]: if dep.get('external_module', False): ec['unresolved_deps'].remove(dep) self.ordered_specs = resolve_dependencies(self.parsed_easyconfigs, retain_all_deps=True) def test_dep_graph(self): """Unit test that builds a full dependency graph.""" # pygraph dependencies required for constructing dependency graph are not available prior to Python 2.6 if LooseVersion(sys.version) >= LooseVersion('2.6') and single_tests_ok: # temporary file for dep graph (hn, fn) = tempfile.mkstemp(suffix='.dot') os.close(hn) if self.ordered_specs is None: self.process_all_easyconfigs() dep_graph(fn, self.ordered_specs, silent=True) try: os.remove(fn) except OSError, err: log.error("Failed to remove %s: %s" % (fn, err)) else:
def main(testing_data=(None, None, None)): """ Main function: parse command line options, and act accordingly. @param testing_data: tuple with command line arguments, log file and boolean indicating whether or not to build """ # purposely session state very early, to avoid modules loaded by EasyBuild meddling in init_session_state = session_state() # steer behavior when testing main testing = testing_data[0] is not None args, logfile, do_build = testing_data # initialise options eb_go = eboptions.parse_options(args=args) options = eb_go.options orig_paths = eb_go.args # set umask (as early as possible) if options.umask is not None: new_umask = int(options.umask, 8) old_umask = os.umask(new_umask) # set temporary directory to use eb_tmpdir = set_tmpdir(options.tmpdir) # initialise logging for main global _log _log, logfile = init_logging(logfile, logtostdout=options.logtostdout, testing=testing) # disallow running EasyBuild as root if os.getuid() == 0: raise EasyBuildError( "You seem to be running EasyBuild with root privileges which is not wise, " "so let's end this here.") # log startup info eb_cmd_line = eb_go.generate_cmd_line() + eb_go.args log_start(eb_cmd_line, eb_tmpdir) if options.umask is not None: _log.info("umask set to '%s' (used to be '%s')" % (oct(new_umask), oct(old_umask))) # process software build specifications (if any), i.e. # software name/version, toolchain name/version, extra patches, ... (try_to_generate, build_specs) = process_software_build_specs(options) # determine robot path # --try-X, --dep-graph, --search use robot path for searching, so enable it with path of installed easyconfigs tweaked_ecs = try_to_generate and build_specs tweaked_ecs_path, pr_path = alt_easyconfig_paths(eb_tmpdir, tweaked_ecs=tweaked_ecs, from_pr=options.from_pr) auto_robot = try_to_generate or options.dep_graph or options.search or options.search_short robot_path = det_robot_path(options.robot_paths, tweaked_ecs_path, pr_path, auto_robot=auto_robot) _log.debug("Full robot path: %s" % robot_path) # configure & initialize build options config_options_dict = eb_go.get_options_by_section('config') build_options = { 'build_specs': build_specs, 'command_line': eb_cmd_line, 'pr_path': pr_path, 'robot_path': robot_path, 'silent': testing, 'try_to_generate': try_to_generate, 'valid_stops': [x[0] for x in EasyBlock.get_steps()], } # initialise the EasyBuild configuration & build options config.init(options, config_options_dict) config.init_build_options(build_options=build_options, cmdline_options=options) # update session state eb_config = eb_go.generate_cmd_line(add_default=True) modlist = session_module_list( testing=testing ) # build options must be initialized first before 'module list' works init_session_state.update({'easybuild_configuration': eb_config}) init_session_state.update({'module_list': modlist}) _log.debug("Initial session state: %s" % init_session_state) # search for easyconfigs, if a query is specified query = options.search or options.search_short if query: search_easyconfigs(query, short=not options.search) # determine easybuild-easyconfigs package install path easyconfigs_pkg_paths = get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR) if not easyconfigs_pkg_paths: _log.warning( "Failed to determine install path for easybuild-easyconfigs package." ) # determine paths to easyconfigs paths = det_easyconfig_paths(orig_paths) if paths: # transform paths into tuples, use 'False' to indicate the corresponding easyconfig files were not generated paths = [(p, False) for p in paths] else: if 'name' in build_specs: # try to obtain or generate an easyconfig file via build specifications if a software name is provided paths = find_easyconfigs_by_specs(build_specs, robot_path, try_to_generate, testing=testing) elif not any([ options.aggregate_regtest, options.search, options.search_short, options.regtest ]): print_error(( "Please provide one or multiple easyconfig files, or use software build " "options to make EasyBuild search for easyconfigs"), log=_log, opt_parser=eb_go.parser, exit_on_error=not testing) _log.debug("Paths: %s" % paths) # run regtest if options.regtest or options.aggregate_regtest: _log.info("Running regression test") # fallback: easybuild-easyconfigs install path regtest_ok = regtest([path[0] for path in paths] or easyconfigs_pkg_paths) if not regtest_ok: _log.info("Regression test failed (partially)!") sys.exit(31) # exit -> 3x1t -> 31 # read easyconfig files easyconfigs, generated_ecs = parse_easyconfigs(paths) # tweak obtained easyconfig files, if requested # don't try and tweak anything if easyconfigs were generated, since building a full dep graph will fail # if easyconfig files for the dependencies are not available if try_to_generate and build_specs and not generated_ecs: easyconfigs = tweak(easyconfigs, build_specs, targetdir=tweaked_ecs_path) # dry_run: print all easyconfigs and dependencies, and whether they are already built if options.dry_run or options.dry_run_short: txt = dry_run(easyconfigs, short=not options.dry_run, build_specs=build_specs) print_msg(txt, log=_log, silent=testing, prefix=False) # cleanup and exit after dry run, searching easyconfigs or submitting regression test if any([ options.dry_run, options.dry_run_short, options.regtest, options.search, options.search_short ]): cleanup(logfile, eb_tmpdir, testing) sys.exit(0) # skip modules that are already installed unless forced if not options.force: retained_ecs = skip_available(easyconfigs) if not testing: for skipped_ec in [ ec for ec in easyconfigs if ec not in retained_ecs ]: print_msg("%s is already installed (module found), skipping" % skipped_ec['full_mod_name']) easyconfigs = retained_ecs # determine an order that will allow all specs in the set to build if len(easyconfigs) > 0: if options.robot: print_msg("resolving dependencies ...", log=_log, silent=testing) ordered_ecs = resolve_dependencies(easyconfigs, build_specs=build_specs) else: ordered_ecs = easyconfigs else: print_msg("No easyconfigs left to be built.", log=_log, silent=testing) ordered_ecs = [] # create dependency graph and exit if options.dep_graph: _log.info("Creating dependency graph %s" % options.dep_graph) dep_graph(options.dep_graph, ordered_ecs) sys.exit(0) # submit build as job(s), clean up and exit if options.job: job_info_txt = submit_jobs(ordered_ecs, eb_go.generate_cmd_line(), testing=testing) if not testing: print_msg("Submitted parallel build jobs, exiting now: %s" % job_info_txt) cleanup(logfile, eb_tmpdir, testing) sys.exit(0) # build software, will exit when errors occurs (except when testing) exit_on_failure = not options.dump_test_report and not options.upload_test_report if not testing or (testing and do_build): ecs_with_res = build_and_install_software( ordered_ecs, init_session_state, exit_on_failure=exit_on_failure) else: ecs_with_res = [(ec, {}) for ec in ordered_ecs] correct_builds_cnt = len([ ec_res for (_, ec_res) in ecs_with_res if ec_res.get('success', False) ]) overall_success = correct_builds_cnt == len(ordered_ecs) success_msg = "Build succeeded for %s out of %s" % (correct_builds_cnt, len(ordered_ecs)) repo = init_repository(get_repository(), get_repositorypath()) repo.cleanup() # dump/upload overall test report test_report_msg = overall_test_report(ecs_with_res, len(paths), overall_success, success_msg, init_session_state) if test_report_msg is not None: print_msg(test_report_msg) print_msg(success_msg, log=_log, silent=testing) # cleanup and spec files for ec in easyconfigs: if 'original_spec' in ec and os.path.isfile(ec['spec']): os.remove(ec['spec']) # stop logging and cleanup tmp log file, unless one build failed (individual logs are located in eb_tmpdir path) stop_logging(logfile, logtostdout=options.logtostdout) if overall_success: cleanup(logfile, eb_tmpdir, testing)
def setUp(self): """Set up testcase.""" super(EnhancedTestCase, self).setUp() # keep track of log handlers log = fancylogger.getLogger(fname=False) self.orig_log_handlers = log.handlers[:] self.orig_tmpdir = tempfile.gettempdir() # use a subdirectory for this test (which we can clean up easily after the test completes) self.test_prefix = set_tmpdir() self.log = fancylogger.getLogger(self.__class__.__name__, fname=False) fd, self.logfile = tempfile.mkstemp(suffix='.log', prefix='eb-test-') os.close(fd) self.cwd = os.getcwd() # keep track of original environment to restore self.orig_environ = copy.deepcopy(os.environ) # keep track of original environment/Python search path to restore self.orig_sys_path = sys.path[:] testdir = os.path.dirname(os.path.abspath(__file__)) self.test_sourcepath = os.path.join(testdir, 'sandbox', 'sources') os.environ['EASYBUILD_SOURCEPATH'] = self.test_sourcepath os.environ['EASYBUILD_PREFIX'] = self.test_prefix self.test_buildpath = tempfile.mkdtemp() os.environ['EASYBUILD_BUILDPATH'] = self.test_buildpath self.test_installpath = tempfile.mkdtemp() os.environ['EASYBUILD_INSTALLPATH'] = self.test_installpath # make sure that the tests only pick up easyconfigs provided with the tests os.environ['EASYBUILD_ROBOT_PATHS'] = os.path.join( testdir, 'easyconfigs') # make sure no deprecated behaviour is being triggered (unless intended by the test) # trip *all* log.deprecated statements by setting deprecation version ridiculously high self.orig_current_version = eb_build_log.CURRENT_VERSION os.environ['EASYBUILD_DEPRECATED'] = '10000000' init_config() # remove any entries in Python search path that seem to provide easyblocks for path in sys.path[:]: if os.path.exists( os.path.join(path, 'easybuild', 'easyblocks', '__init__.py')): sys.path.remove(path) # add test easyblocks to Python search path and (re)import and reload easybuild modules import easybuild sys.path.append(os.path.join(testdir, 'sandbox')) reload(easybuild) import easybuild.easyblocks reload(easybuild.easyblocks) import easybuild.easyblocks.generic reload(easybuild.easyblocks.generic) reload(easybuild.tools.module_naming_scheme ) # required to run options unit tests stand-alone modtool = modules_tool() # purge out any loaded modules with original $MODULEPATH before running each test modtool.purge() self.reset_modulepath([os.path.join(testdir, 'modules')])
def main(testing_data=(None, None, None)): """ Main function: parse command line options, and act accordingly. @param testing_data: tuple with command line arguments, log file and boolean indicating whether or not to build """ # purposely session state very early, to avoid modules loaded by EasyBuild meddling in init_session_state = session_state() # steer behavior when testing main testing = testing_data[0] is not None args, logfile, do_build = testing_data # initialise options eb_go = eboptions.parse_options(args=args) options = eb_go.options orig_paths = eb_go.args # set umask (as early as possible) if options.umask is not None: new_umask = int(options.umask, 8) old_umask = os.umask(new_umask) # set temporary directory to use eb_tmpdir = set_tmpdir(options.tmpdir) # initialise logging for main global _log _log, logfile = init_logging(logfile, logtostdout=options.logtostdout, testing=testing) # disallow running EasyBuild as root if os.getuid() == 0: _log.error("You seem to be running EasyBuild with root privileges which is not wise, so let's end this here.") # log startup info eb_cmd_line = eb_go.generate_cmd_line() + eb_go.args log_start(eb_cmd_line, eb_tmpdir) if options.umask is not None: _log.info("umask set to '%s' (used to be '%s')" % (oct(new_umask), oct(old_umask))) # process software build specifications (if any), i.e. # software name/version, toolchain name/version, extra patches, ... (try_to_generate, build_specs) = process_software_build_specs(options) # determine robot path # --try-X, --dep-graph, --search use robot path for searching, so enable it with path of installed easyconfigs tweaked_ecs = try_to_generate and build_specs tweaked_ecs_path, pr_path = alt_easyconfig_paths(eb_tmpdir, tweaked_ecs=tweaked_ecs, from_pr=options.from_pr) auto_robot = try_to_generate or options.dep_graph or options.search or options.search_short robot_path = det_robot_path(options.robot_paths, tweaked_ecs_path, pr_path, auto_robot=auto_robot) _log.debug("Full robot path: %s" % robot_path) # configure & initialize build options config_options_dict = eb_go.get_options_by_section('config') build_options = { 'build_specs': build_specs, 'command_line': eb_cmd_line, 'pr_path': pr_path, 'robot_path': robot_path, 'silent': testing, 'try_to_generate': try_to_generate, 'valid_stops': [x[0] for x in EasyBlock.get_steps()], } # initialise the EasyBuild configuration & build options config.init(options, config_options_dict) config.init_build_options(build_options=build_options, cmdline_options=options) # update session state eb_config = eb_go.generate_cmd_line(add_default=True) modlist = session_module_list(testing=testing) # build options must be initialized first before 'module list' works init_session_state.update({'easybuild_configuration': eb_config}) init_session_state.update({'module_list': modlist}) _log.debug("Initial session state: %s" % init_session_state) # search for easyconfigs, if a query is specified query = options.search or options.search_short if query: search_easyconfigs(query, short=not options.search) # determine easybuild-easyconfigs package install path easyconfigs_pkg_paths = get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR) if not easyconfigs_pkg_paths: _log.warning("Failed to determine install path for easybuild-easyconfigs package.") # determine paths to easyconfigs paths = det_easyconfig_paths(orig_paths, options.from_pr, easyconfigs_pkg_paths) if not paths: if 'name' in build_specs: # try to obtain or generate an easyconfig file via build specifications if a software name is provided paths = find_easyconfigs_by_specs(build_specs, robot_path, try_to_generate, testing=testing) elif not any([options.aggregate_regtest, options.search, options.search_short, options.regtest]): print_error(("Please provide one or multiple easyconfig files, or use software build " "options to make EasyBuild search for easyconfigs"), log=_log, opt_parser=eb_go.parser, exit_on_error=not testing) _log.debug("Paths: %s" % paths) # run regtest if options.regtest or options.aggregate_regtest: _log.info("Running regression test") # fallback: easybuild-easyconfigs install path regtest_ok = regtest([path[0] for path in paths] or easyconfigs_pkg_paths) if not regtest_ok: _log.info("Regression test failed (partially)!") sys.exit(31) # exit -> 3x1t -> 31 # read easyconfig files easyconfigs, generated_ecs = parse_easyconfigs(paths) # tweak obtained easyconfig files, if requested # don't try and tweak anything if easyconfigs were generated, since building a full dep graph will fail # if easyconfig files for the dependencies are not available if try_to_generate and build_specs and not generated_ecs: easyconfigs = tweak(easyconfigs, build_specs, targetdir=tweaked_ecs_path) # dry_run: print all easyconfigs and dependencies, and whether they are already built if options.dry_run or options.dry_run_short: txt = dry_run(easyconfigs, short=not options.dry_run, build_specs=build_specs) print_msg(txt, log=_log, silent=testing, prefix=False) # cleanup and exit after dry run, searching easyconfigs or submitting regression test if any([options.dry_run, options.dry_run_short, options.regtest, options.search, options.search_short]): cleanup(logfile, eb_tmpdir, testing) sys.exit(0) # skip modules that are already installed unless forced if not options.force: retained_ecs = skip_available(easyconfigs) if not testing: for skipped_ec in [ec for ec in easyconfigs if ec not in retained_ecs]: print_msg("%s is already installed (module found), skipping" % skipped_ec['full_mod_name']) easyconfigs = retained_ecs # determine an order that will allow all specs in the set to build if len(easyconfigs) > 0: if options.robot: print_msg("resolving dependencies ...", log=_log, silent=testing) ordered_ecs = resolve_dependencies(easyconfigs, build_specs=build_specs) else: ordered_ecs = easyconfigs else: print_msg("No easyconfigs left to be built.", log=_log, silent=testing) ordered_ecs = [] # create dependency graph and exit if options.dep_graph: _log.info("Creating dependency graph %s" % options.dep_graph) dep_graph(options.dep_graph, ordered_ecs) sys.exit(0) # submit build as job(s), clean up and exit if options.job: job_info_txt = submit_jobs(ordered_ecs, eb_go.generate_cmd_line(), testing=testing) if not testing: print_msg("Submitted parallel build jobs, exiting now: %s" % job_info_txt) cleanup(logfile, eb_tmpdir, testing) sys.exit(0) # build software, will exit when errors occurs (except when testing) exit_on_failure = not options.dump_test_report and not options.upload_test_report if not testing or (testing and do_build): ecs_with_res = build_and_install_software(ordered_ecs, init_session_state, exit_on_failure=exit_on_failure) else: ecs_with_res = [(ec, {}) for ec in ordered_ecs] correct_builds_cnt = len([ec_res for (_, ec_res) in ecs_with_res if ec_res.get('success', False)]) overall_success = correct_builds_cnt == len(ordered_ecs) success_msg = "Build succeeded for %s out of %s" % (correct_builds_cnt, len(ordered_ecs)) repo = init_repository(get_repository(), get_repositorypath()) repo.cleanup() # dump/upload overall test report test_report_msg = overall_test_report(ecs_with_res, len(paths), overall_success, success_msg, init_session_state) if test_report_msg is not None: print_msg(test_report_msg) print_msg(success_msg, log=_log, silent=testing) # cleanup and spec files for ec in easyconfigs: if 'original_spec' in ec and os.path.isfile(ec['spec']): os.remove(ec['spec']) # stop logging and cleanup tmp log file, unless one build failed (individual logs are located in eb_tmpdir path) stop_logging(logfile, logtostdout=options.logtostdout) if overall_success: cleanup(logfile, eb_tmpdir, testing)
def main(testing_data=(None, None, None)): """ Main function: @arg options: a tuple: (options, paths, logger, logfile, hn) as defined in parse_options This function will: - read easyconfig - build software """ # disallow running EasyBuild as root if os.getuid() == 0: sys.stderr.write("ERROR: You seem to be running EasyBuild with root privileges.\n" "That's not wise, so let's end this here.\n" "Exiting.\n") sys.exit(1) # steer behavior when testing main testing = testing_data[0] is not None args, logfile, do_build = testing_data # initialise options eb_go = eboptions.parse_options(args=args) options = eb_go.options orig_paths = eb_go.args # set umask (as early as possible) if options.umask is not None: new_umask = int(options.umask, 8) old_umask = os.umask(new_umask) # set temporary directory to use eb_tmpdir = set_tmpdir(options.tmpdir) # initialise logging for main if options.logtostdout: fancylogger.logToScreen(enable=True, stdout=True) else: if logfile is None: # mkstemp returns (fd,filename), fd is from os.open, not regular open! fd, logfile = tempfile.mkstemp(suffix='.log', prefix='easybuild-') os.close(fd) fancylogger.logToFile(logfile) print_msg('temporary log file in case of crash %s' % (logfile), log=None, silent=testing) global _log _log = fancylogger.getLogger(fname=False) if options.umask is not None: _log.info("umask set to '%s' (used to be '%s')" % (oct(new_umask), oct(old_umask))) # hello world! _log.info(this_is_easybuild()) # how was EB called? eb_command_line = eb_go.generate_cmd_line() + eb_go.args _log.info("Command line: %s" % (" ".join(eb_command_line))) _log.info("Using %s as temporary directory" % eb_tmpdir) if not options.robot is None: if options.robot: _log.info("Using robot path(s): %s" % options.robot) else: _log.error("No robot paths specified, and unable to determine easybuild-easyconfigs install path.") # do not pass options.robot, it's not a list instance (and it shouldn't be modified) robot_path = None if options.robot: robot_path = list(options.robot) # determine easybuild-easyconfigs package install path easyconfigs_paths = get_paths_for("easyconfigs", robot_path=robot_path) # keep track of paths for install easyconfigs, so we can obtain find specified easyconfigs easyconfigs_pkg_full_paths = easyconfigs_paths[:] if not easyconfigs_paths: _log.warning("Failed to determine install path for easybuild-easyconfigs package.") # specified robot paths are preferred over installed easyconfig files if robot_path: robot_path.extend(easyconfigs_paths) easyconfigs_paths = robot_path[:] _log.info("Extended list of robot paths with paths for installed easyconfigs: %s" % robot_path) # initialise the easybuild configuration config.init(options, eb_go.get_options_by_section('config')) # building a dependency graph implies force, so that all dependencies are retained # and also skips validation of easyconfigs (e.g. checking os dependencies) retain_all_deps = False if options.dep_graph: _log.info("Enabling force to generate dependency graph.") options.force = True retain_all_deps = True config.init_build_options({ 'aggregate_regtest': options.aggregate_regtest, 'allow_modules_tool_mismatch': options.allow_modules_tool_mismatch, 'check_osdeps': not options.ignore_osdeps, 'command_line': eb_command_line, 'debug': options.debug, 'dry_run': options.dry_run, 'easyblock': options.easyblock, 'experimental': options.experimental, 'force': options.force, 'group': options.group, 'ignore_dirs': options.ignore_dirs, 'modules_footer': options.modules_footer, 'only_blocks': options.only_blocks, 'recursive_mod_unload': options.recursive_module_unload, 'regtest_online': options.regtest_online, 'regtest_output_dir': options.regtest_output_dir, 'retain_all_deps': retain_all_deps, 'robot_path': robot_path, 'sequential': options.sequential, 'silent': testing, 'set_gid_bit': options.set_gid_bit, 'skip': options.skip, 'skip_test_cases': options.skip_test_cases, 'sticky_bit': options.sticky_bit, 'stop': options.stop, 'umask': options.umask, 'valid_module_classes': module_classes(), 'valid_stops': [x[0] for x in EasyBlock.get_steps()], 'validate': not options.force, }) # search for easyconfigs if options.search or options.search_short: search_path = [os.getcwd()] if easyconfigs_paths: search_path = easyconfigs_paths query = options.search or options.search_short ignore_dirs = config.build_option('ignore_dirs') silent = config.build_option('silent') search_file(search_path, query, short=not options.search, ignore_dirs=ignore_dirs, silent=silent) # process software build specifications (if any), i.e. # software name/version, toolchain name/version, extra patches, ... (try_to_generate, build_specs) = process_software_build_specs(options) paths = [] if len(orig_paths) == 0: if 'name' in build_specs: paths = [obtain_path(build_specs, easyconfigs_paths, try_to_generate=try_to_generate, exit_on_error=not testing)] elif not any([options.aggregate_regtest, options.search, options.search_short, options.regtest]): print_error(("Please provide one or multiple easyconfig files, or use software build " "options to make EasyBuild search for easyconfigs"), log=_log, opt_parser=eb_go.parser, exit_on_error=not testing) else: # look for easyconfigs with relative paths in easybuild-easyconfigs package, # unless they were found at the given relative paths if easyconfigs_pkg_full_paths: # determine which easyconfigs files need to be found, if any ecs_to_find = [] for idx, orig_path in enumerate(orig_paths): if orig_path == os.path.basename(orig_path) and not os.path.exists(orig_path): ecs_to_find.append((idx, orig_path)) _log.debug("List of easyconfig files to find: %s" % ecs_to_find) # find missing easyconfigs by walking paths with installed easyconfig files for path in easyconfigs_pkg_full_paths: _log.debug("Looking for missing easyconfig files (%d left) in %s..." % (len(ecs_to_find), path)) for (subpath, dirnames, filenames) in os.walk(path, topdown=True): for idx, orig_path in ecs_to_find[:]: if orig_path in filenames: full_path = os.path.join(subpath, orig_path) _log.info("Found %s in %s: %s" % (orig_path, path, full_path)) orig_paths[idx] = full_path # if file was found, stop looking for it (first hit wins) ecs_to_find.remove((idx, orig_path)) # stop os.walk insanity as soon as we have all we need (os.walk loop) if len(ecs_to_find) == 0: break # ignore subdirs specified to be ignored by replacing items in dirnames list used by os.walk dirnames[:] = [d for d in dirnames if not d in options.ignore_dirs] # stop os.walk insanity as soon as we have all we need (paths loop) if len(ecs_to_find) == 0: break # indicate that specified paths do not contain generated easyconfig files paths = [(path, False) for path in orig_paths] _log.debug("Paths: %s" % paths) # run regtest if options.regtest or options.aggregate_regtest: _log.info("Running regression test") if paths: ec_paths = [path[0] for path in paths] else: # fallback: easybuild-easyconfigs install path ec_paths = easyconfigs_pkg_full_paths regtest_ok = regtest(ec_paths) if not regtest_ok: _log.info("Regression test failed (partially)!") sys.exit(31) # exit -> 3x1t -> 31 # read easyconfig files easyconfigs = [] for (path, generated) in paths: path = os.path.abspath(path) if not os.path.exists(path): print_error("Can't find path %s" % path) try: files = find_easyconfigs(path, ignore_dirs=options.ignore_dirs) for f in files: if not generated and try_to_generate and build_specs: ec_file = tweak(f, None, build_specs) else: ec_file = f ecs = process_easyconfig(ec_file, build_specs=build_specs) easyconfigs.extend(ecs) except IOError, err: _log.error("Processing easyconfigs in path %s failed: %s" % (path, err))
import test.framework.options as o import test.framework.parallelbuild as p import test.framework.repository as r import test.framework.robot as robot import test.framework.run as run import test.framework.scripts as sc import test.framework.systemtools as s import test.framework.toolchain as tc import test.framework.toolchainvariables as tcv import test.framework.toy_build as t import test.framework.tweak as tw import test.framework.variables as v # make sure temporary files can be created/used try: set_tmpdir(raise_error=True) except EasyBuildError, err: sys.stderr.write( "No execution rights on temporary files, specify another location via $TMPDIR: %s\n" % err) sys.exit(1) # initialize logger for all the unit tests fd, log_fn = tempfile.mkstemp(prefix='easybuild-tests-', suffix='.log') os.close(fd) os.remove(log_fn) fancylogger.logToFile(log_fn) log = fancylogger.getLogger() # call suite() for each module and then run them all # note: make sure the options unit tests run first, to avoid running some of them with a readily initialized config
def main(testing_data=(None, None, None)): """ Main function: @arg options: a tuple: (options, paths, logger, logfile, hn) as defined in parse_options This function will: - read easyconfig - build software """ # purposely session state very early, to avoid modules loaded by EasyBuild meddling in init_session_state = session_state() # disallow running EasyBuild as root if os.getuid() == 0: sys.stderr.write( "ERROR: You seem to be running EasyBuild with root privileges.\n" "That's not wise, so let's end this here.\n" "Exiting.\n") sys.exit(1) # steer behavior when testing main testing = testing_data[0] is not None args, logfile, do_build = testing_data # initialise options eb_go = eboptions.parse_options(args=args) options = eb_go.options orig_paths = eb_go.args eb_config = eb_go.generate_cmd_line(add_default=True) init_session_state.update({'easybuild_configuration': eb_config}) # set umask (as early as possible) if options.umask is not None: new_umask = int(options.umask, 8) old_umask = os.umask(new_umask) # set temporary directory to use eb_tmpdir = set_tmpdir(options.tmpdir) # initialise logging for main if options.logtostdout: fancylogger.logToScreen(enable=True, stdout=True) else: if logfile is None: # mkstemp returns (fd,filename), fd is from os.open, not regular open! fd, logfile = tempfile.mkstemp(suffix='.log', prefix='easybuild-') os.close(fd) fancylogger.logToFile(logfile) print_msg('temporary log file in case of crash %s' % (logfile), log=None, silent=testing) global _log _log = fancylogger.getLogger(fname=False) if options.umask is not None: _log.info("umask set to '%s' (used to be '%s')" % (oct(new_umask), oct(old_umask))) # hello world! _log.info(this_is_easybuild()) # how was EB called? eb_command_line = eb_go.generate_cmd_line() + eb_go.args _log.info("Command line: %s" % (" ".join(eb_command_line))) _log.info("Using %s as temporary directory" % eb_tmpdir) if not options.robot is None: if options.robot: _log.info("Using robot path(s): %s" % options.robot) else: _log.error( "No robot paths specified, and unable to determine easybuild-easyconfigs install path." ) # do not pass options.robot, it's not a list instance (and it shouldn't be modified) robot_path = None if options.robot: robot_path = list(options.robot) # determine easybuild-easyconfigs package install path easyconfigs_paths = get_paths_for("easyconfigs", robot_path=robot_path) # keep track of paths for install easyconfigs, so we can obtain find specified easyconfigs easyconfigs_pkg_full_paths = easyconfigs_paths[:] if not easyconfigs_paths: _log.warning( "Failed to determine install path for easybuild-easyconfigs package." ) # process software build specifications (if any), i.e. # software name/version, toolchain name/version, extra patches, ... (try_to_generate, build_specs) = process_software_build_specs(options) # specified robot paths are preferred over installed easyconfig files # --try-X and --dep-graph both require --robot, so enable it with path of installed easyconfigs if robot_path or try_to_generate or options.dep_graph: if robot_path is None: robot_path = [] robot_path.extend(easyconfigs_paths) easyconfigs_paths = robot_path[:] _log.info( "Extended list of robot paths with paths for installed easyconfigs: %s" % robot_path) # initialise the easybuild configuration config.init(options, eb_go.get_options_by_section('config')) # building a dependency graph implies force, so that all dependencies are retained # and also skips validation of easyconfigs (e.g. checking os dependencies) retain_all_deps = False if options.dep_graph: _log.info("Enabling force to generate dependency graph.") options.force = True retain_all_deps = True config.init_build_options({ 'aggregate_regtest': options.aggregate_regtest, 'allow_modules_tool_mismatch': options.allow_modules_tool_mismatch, 'check_osdeps': not options.ignore_osdeps, 'cleanup_builddir': options.cleanup_builddir, 'command_line': eb_command_line, 'debug': options.debug, 'dry_run': options.dry_run, 'easyblock': options.easyblock, 'experimental': options.experimental, 'force': options.force, 'github_user': options.github_user, 'group': options.group, 'ignore_dirs': options.ignore_dirs, 'modules_footer': options.modules_footer, 'only_blocks': options.only_blocks, 'recursive_mod_unload': options.recursive_module_unload, 'regtest_output_dir': options.regtest_output_dir, 'retain_all_deps': retain_all_deps, 'robot_path': robot_path, 'sequential': options.sequential, 'silent': testing, 'set_gid_bit': options.set_gid_bit, 'skip': options.skip, 'skip_test_cases': options.skip_test_cases, 'sticky_bit': options.sticky_bit, 'stop': options.stop, 'umask': options.umask, 'valid_module_classes': module_classes(), 'valid_stops': [x[0] for x in EasyBlock.get_steps()], 'validate': not options.force, }) # obtain list of loaded modules, build options must be initialized first modlist = session_module_list() init_session_state.update({'module_list': modlist}) _log.debug("Initial session state: %s" % init_session_state) # search for easyconfigs if options.search or options.search_short: search_path = [os.getcwd()] if easyconfigs_paths: search_path = easyconfigs_paths query = options.search or options.search_short ignore_dirs = config.build_option('ignore_dirs') silent = config.build_option('silent') search_file(search_path, query, short=not options.search, ignore_dirs=ignore_dirs, silent=silent) paths = [] if len(orig_paths) == 0: if options.from_pr: pr_path = os.path.join(eb_tmpdir, "files_pr%s" % options.from_pr) pr_files = fetch_easyconfigs_from_pr( options.from_pr, path=pr_path, github_user=options.github_user) paths = [(path, False) for path in pr_files if path.endswith('.eb')] elif 'name' in build_specs: paths = [ obtain_path(build_specs, easyconfigs_paths, try_to_generate=try_to_generate, exit_on_error=not testing) ] elif not any([ options.aggregate_regtest, options.search, options.search_short, options.regtest ]): print_error(( "Please provide one or multiple easyconfig files, or use software build " "options to make EasyBuild search for easyconfigs"), log=_log, opt_parser=eb_go.parser, exit_on_error=not testing) else: # look for easyconfigs with relative paths in easybuild-easyconfigs package, # unless they were found at the given relative paths if easyconfigs_pkg_full_paths: # determine which easyconfigs files need to be found, if any ecs_to_find = [] for idx, orig_path in enumerate(orig_paths): if orig_path == os.path.basename( orig_path) and not os.path.exists(orig_path): ecs_to_find.append((idx, orig_path)) _log.debug("List of easyconfig files to find: %s" % ecs_to_find) # find missing easyconfigs by walking paths with installed easyconfig files for path in easyconfigs_pkg_full_paths: _log.debug( "Looking for missing easyconfig files (%d left) in %s..." % (len(ecs_to_find), path)) for (subpath, dirnames, filenames) in os.walk(path, topdown=True): for idx, orig_path in ecs_to_find[:]: if orig_path in filenames: full_path = os.path.join(subpath, orig_path) _log.info("Found %s in %s: %s" % (orig_path, path, full_path)) orig_paths[idx] = full_path # if file was found, stop looking for it (first hit wins) ecs_to_find.remove((idx, orig_path)) # stop os.walk insanity as soon as we have all we need (os.walk loop) if len(ecs_to_find) == 0: break # ignore subdirs specified to be ignored by replacing items in dirnames list used by os.walk dirnames[:] = [ d for d in dirnames if not d in options.ignore_dirs ] # stop os.walk insanity as soon as we have all we need (paths loop) if len(ecs_to_find) == 0: break # indicate that specified paths do not contain generated easyconfig files paths = [(path, False) for path in orig_paths] _log.debug("Paths: %s" % paths) # run regtest if options.regtest or options.aggregate_regtest: _log.info("Running regression test") if paths: ec_paths = [path[0] for path in paths] else: # fallback: easybuild-easyconfigs install path ec_paths = easyconfigs_pkg_full_paths regtest_ok = regtest(ec_paths) if not regtest_ok: _log.info("Regression test failed (partially)!") sys.exit(31) # exit -> 3x1t -> 31 # read easyconfig files easyconfigs = [] for (path, generated) in paths: path = os.path.abspath(path) if not os.path.exists(path): print_error("Can't find path %s" % path) try: ec_files = find_easyconfigs(path, ignore_dirs=options.ignore_dirs) for ec_file in ec_files: # only pass build specs when not generating easyconfig files if try_to_generate: ecs = process_easyconfig(ec_file) else: ecs = process_easyconfig(ec_file, build_specs=build_specs) easyconfigs.extend(ecs) except IOError, err: _log.error("Processing easyconfigs in path %s failed: %s" % (path, err))