def getEasyBuildVersion(rootdirectory): # We make sure that the path given is totally expanded. absrootdirectory = os.path.abspath(os.path.expandvars(rootdirectory)) # Appending temporarily to PYTHONPATH the EasyBuild directories and vsc-base sys.path.insert( 0, os.path.join(os.path.join(absrootdirectory, '.installRef'), 'easybuild-framework')) sys.path.insert( 0, os.path.join(os.path.join(absrootdirectory, '.installRef'), 'easybuild-easyblocks')) sys.path.insert( 0, os.path.join(os.path.join(absrootdirectory, '.installRef'), 'easybuild-easyconfigs')) sys.path.insert( 0, os.path.join(os.path.join(absrootdirectory, '.installRef'), 'vsc-base')) # Importing the function that EasyBuild uses to determine its own version from easybuild.tools.version import this_is_easybuild # Getting the version of EasyBuild from the output message msg = this_is_easybuild() version = re.search("[0-9]*\.[0-9]*\.[0-9]*", msg).group(0) # Removing from the PYTHONPATH the EasyBuild directories (cleanup) sys.path.pop(0) sys.path.pop(0) sys.path.pop(0) sys.path.pop(0) return version
def log_start(eb_command_line, eb_tmpdir): """Log startup info.""" _log.info(this_is_easybuild()) # log used command line _log.info("Command line: %s" % (' '.join(eb_command_line))) _log.info("Using %s as temporary directory" % eb_tmpdir)
def log_start(log, eb_command_line, eb_tmpdir): """Log startup info.""" log.info(this_is_easybuild()) # log used command line log.info("Command line: %s", ' '.join(eb_command_line)) log.info("Using %s as temporary directory", eb_tmpdir)
class EasyBuildOptions(GeneralOption): """Easybuild generaloption class""" VERSION = this_is_easybuild() DEFAULT_LOGLEVEL = 'INFO' DEFAULT_CONFIGFILES = get_default_configfiles() ALLOPTSMANDATORY = False # allow more than one argument def basic_options(self): """basic runtime options""" all_stops = [x[0] for x in EasyBlock.get_steps()] strictness_options = [ filetools.IGNORE, filetools.WARN, filetools.ERROR ] try: default_robot_path = get_paths_for("easyconfigs", robot_path=None)[0] except: self.log.warning( "basic_options: unable to determine default easyconfig path") default_robot_path = False # False as opposed to None, since None is used for indicating that --robot was not used descr = ("Basic options", "Basic runtime options for EasyBuild.") opts = OrderedDict({ "only-blocks": ("Only build listed blocks", None, "extend", None, "b", { 'metavar': "BLOCKS" }), "force": (("Force to rebuild software even if it's already installed " "(i.e. if it can be found as module)"), None, "store_true", False, "f"), "job": ("Submit the build as a job", None, "store_true", False), "skip": ("Skip existing software (useful for installing additional packages)", None, "store_true", False, "k"), "robot": ("Path to search for easyconfigs for missing dependencies.", None, "store_or_None", default_robot_path, "r", { 'metavar': "PATH" }), "stop": ("Stop the installation after certain step", "choice", "store_or_None", "source", "s", all_stops), "strict": ("Set strictness level", "choice", "store", filetools.WARN, strictness_options), "logtostdout": ("Redirect main log to stdout", None, "store_true", False, "l"), "dry-run": ("Resolve dependencies and print build list, then stop", None, "store_true", False), }) self.log.debug("basic_options: descr %s opts %s" % (descr, opts)) self.add_group_parser(opts, descr) def software_options(self): # software build options descr = ("Software search and build options", ( "Specify software search and build options: EasyBuild will search for a " "matching easyconfig and build it. When called with the try prefix " "(i.e. --try-X ), EasyBuild will search for a matching easyconfig " "and if none are found, try to generate one based on a close matching one " "(NOTE: --try-X is best effort, it might produce wrong builds!)")) opts = OrderedDict({ 'software-name': ("Search and build software with name", None, 'store', None, { 'metavar': 'NAME' }), 'software-version': ("Search and build software with version", None, 'store', None, { 'metavar': 'VERSION' }), 'toolchain': ("Search and build with toolchain (name and version)", None, 'extend', None, { 'metavar': 'NAME,VERSION' }), 'toolchain-name': ("Search and build with toolchain name", None, 'store', None, { 'metavar': 'NAME' }), 'toolchain-version': ("Search and build with toolchain version", None, 'store', None, { 'metavar': 'VERSION' }), 'amend': (("Specify additional search and build parameters (can be used multiple times); " "for example: versionprefix=foo or patches=one.patch,two.patch)" ), None, 'append', None, { 'metavar': 'VAR=VALUE[,VALUE]' }), }) longopts = opts.keys() for longopt in longopts: hlp = opts[longopt][0] hlp = "Try to %s (USE WITH CARE!)" % (hlp[0].lower() + hlp[1:]) opts["try-%s" % longopt] = (hlp, ) + opts[longopt][1:] self.log.debug("software_options: descr %s opts %s" % (descr, opts)) self.add_group_parser(opts, descr) def override_options(self): # override options descr = ("Override options", "Override default EasyBuild behavior.") opts = { "easyblock": ( "easyblock to use for processing the spec file or dumping the options", None, "store", None, "e", { 'metavar': "CLASS" }, ), "pretend": ( ("Does the build/installation in " "a test directory located in $HOME/easybuildinstall "), None, "store_true", False, "p", ), "skip-test-cases": ( "Skip running test cases", None, "store_true", False, "t", ), } self.log.debug("override_options: descr %s opts %s" % (descr, opts)) self.add_group_parser(opts, descr) def config_options(self): # config options descr = ("Configuration options", "Configure EasyBuild behavior.") oldstyle_defaults = get_default_oldstyle_configfile_defaults() opts = { "config": ( "Path to EasyBuild config file", None, 'store', oldstyle_defaults['config'], "C", ), 'prefix': (('Change prefix for buildpath, installpath, sourcepath and repositorypath ' '(repositorypath prefix is only relevant in case of FileRepository repository)' '(used prefix for defaults %s)' % oldstyle_defaults['prefix']), None, 'store', None), 'buildpath': ('Temporary build path', None, 'store', oldstyle_defaults['buildpath']), 'installpath': ('Final install path', None, 'store', oldstyle_defaults['installpath']), 'subdir-modules': ('Subdir in installpath for modules', None, 'store', oldstyle_defaults['subdir_modules']), 'subdir-software': ('Subdir in installpath for software', None, 'store', oldstyle_defaults['subdir_software']), 'repository': ('Repository type, using repositorypath', 'choice', 'store', oldstyle_defaults['repository'], sorted(avail_repositories().keys())), 'repositorypath': (('Repository path, used by repository ' '(is passed as list of arguments to create the repository instance). ' 'For more info, use --avail-repositories.'), 'strlist', 'store', oldstyle_defaults['repositorypath'][ oldstyle_defaults['repository']]), "avail-repositories": ( "Show all repository types (incl. non-usable)", None, "store_true", False, ), 'logfile-format': ('Directory name and format of the log file ', 'strtuple', 'store', oldstyle_defaults['logfile_format'], { 'metavar': 'DIR,FORMAT' }), 'tmp-logdir': ('Log directory where temporary log files are stored', None, 'store', oldstyle_defaults['tmp_logdir']), 'sourcepath': ('Path(s) to where sources should be downloaded (string, colon-separated)', None, 'store', oldstyle_defaults['sourcepath']), 'moduleclasses': (('Extend supported module classes' ' (For more info on the default classes, use --show-default-moduleclasses)' ), None, 'extend', oldstyle_defaults['moduleclasses']), 'show-default-moduleclasses': ('Show default module classes with description', None, 'store_true', False), 'modules-tool': ('Modules tool to use', 'choice', 'store', oldstyle_defaults['modules_tool'], sorted(avail_modules_tools().keys())), "avail-modules-tools": ( "Show all supported module tools", None, "store_true", False, ), 'module-naming-scheme': ('Module naming scheme', 'choice', 'store', oldstyle_defaults['module_naming_scheme'], sorted(avail_module_naming_schemes().keys())), "avail-module-naming-schemes": ( "Show all supported module naming schemes", None, "store_true", False, ), # this one is sort of an exception, it's something jobscripts can set, # has no real meaning for regular eb usage "testoutput": ("Path to where a job should place the output (to be set within jobscript)", None, "store", None), } self.log.debug("config_options: descr %s opts %s" % (descr, opts)) self.add_group_parser(opts, descr) def informative_options(self): # informative options descr = ("Informative options", "Obtain information about EasyBuild.") opts = { "avail-easyconfig-params": ( ("Show all easyconfig parameters (include " "easyblock-specific ones by using -e)"), None, "store_true", False, "a", ), "avail-easyconfig-templates": (("Show all template names and template constants " "that can be used in easyconfigs."), None, "store_true", False), "avail-easyconfig-constants": (("Show all constants that can be used in easyconfigs."), None, "store_true", False), "avail-easyconfig-licenses": (("Show all license constants that can be used in easyconfigs."), None, "store_true", False), "list-easyblocks": ("Show list of available easyblocks", "choice", "store_or_None", "simple", ["simple", "detailed"]), "list-toolchains": ("Show list of known toolchains", None, "store_true", False), "search": ("Search for easyconfig files in the robot directory", None, "store", None, { 'metavar': "STR" }), "dep-graph": ( "Create dependency graph", None, "store", None, { 'metavar': "depgraph.<ext>" }, ), } self.log.debug("informative_options: descr %s opts %s" % (descr, opts)) self.add_group_parser(opts, descr) def regtest_options(self): # regression test options descr = ("Regression test options", "Run and control an EasyBuild regression test.") opts = { "regtest": ("Enable regression test mode", None, "store_true", False), "regtest-online": ( "Enable online regression test mode", None, "store_true", False, ), "sequential": ( "Specify this option if you want to prevent parallel build", None, "store_true", False, ), "regtest-output-dir": ( "Set output directory for test-run", None, "store", None, { 'metavar': "DIR" }, ), "aggregate-regtest": ( "Collect all the xmls inside the given directory and generate a single file", None, "store", None, { 'metavar': "DIR" }, ), } self.log.debug("regtest_options: descr %s opts %s" % (descr, opts)) self.add_group_parser(opts, descr) def easyconfig_options(self): # easyconfig options (to be passed to easyconfig instance) descr = ("Options for Easyconfigs", "Options to be passed to all Easyconfig.") opts = None self.log.debug("easyconfig_options: descr %s opts %s" % (descr, opts)) self.add_group_parser(opts, descr, prefix='easyconfig') def easyblock_options(self): # easyblock options (to be passed to easyblock instance) descr = ("Options for Easyblocks", "Options to be passed to all Easyblocks.") opts = None self.log.debug("easyblock_options: descr %s opts %s" % (descr, opts)) self.add_group_parser(opts, descr, prefix='easyblock') def unittest_options(self): # unittest options descr = ("Unittest options", "Options dedicated to unittesting (experts only).") opts = { "file": ("Log to this file in unittest mode", None, "store", None), } self.log.debug("unittest_options: descr %s opts %s" % (descr, opts)) self.add_group_parser(opts, descr, prefix='unittest') def validate(self): """Additional validation of options""" stop_msg = [] if self.options.toolchain and not len(self.options.toolchain) == 2: stop_msg.append('--toolchain requires NAME,VERSION (given %s)' % (','.join(self.options.toolchain))) if self.options.try_toolchain and not len( self.options.try_toolchain) == 2: stop_msg.append( '--try-toolchain requires NAME,VERSION (given %s)' % (','.join(self.options.try_toolchain))) if len(stop_msg) > 0: indent = " " * 2 stop_msg = ['%s%s' % (indent, x) for x in stop_msg] stop_msg.insert( 0, 'ERROR: Found %s problems validating the options:' % len(stop_msg)) print "\n".join(stop_msg) sys.exit(1) def postprocess(self): """Do some postprocessing, in particular print stuff""" if self.options.unittest_file: fancylogger.logToFile(self.options.unittest_file) if any([ self.options.avail_easyconfig_params, self.options.avail_easyconfig_templates, self.options.list_easyblocks, self.options.list_toolchains, self.options.avail_easyconfig_constants, self.options.avail_easyconfig_licenses, self.options.avail_repositories, self.options.show_default_moduleclasses, self.options.avail_modules_tools, self.options.avail_module_naming_schemes, ]): build_easyconfig_constants_dict( ) # runs the easyconfig constants sanity check self._postprocess_list_avail() self._postprocess_config() def _postprocess_config(self): """Postprocessing of configuration options""" if self.options.prefix is not None: changed_defaults = get_default_oldstyle_configfile_defaults( self.options.prefix) for dest in [ 'installpath', 'buildpath', 'sourcepath', 'repositorypath' ]: if not self.options._action_taken.get(dest, False): new_def = changed_defaults[dest] if dest == 'repositorypath': setattr(self.options, dest, new_def[changed_defaults['repository']]) else: setattr(self.options, dest, new_def) # LEGACY this line is here for oldstyle reasons self.log.deprecated( 'Fake action taken to distinguish from default', '2.0') self.options._action_taken[dest] = True if self.options.pretend: self.options.installpath = get_pretend_installpath() def _postprocess_list_avail(self): """Create all the additional info that can be requested (exit at the end)""" msg = '' # dump possible easyconfig params if self.options.avail_easyconfig_params: msg += self.avail_easyconfig_params() # dump easyconfig template options if self.options.avail_easyconfig_templates: msg += template_documentation() # dump easyconfig constant options if self.options.avail_easyconfig_constants: msg += constant_documentation() # dump easyconfig license options if self.options.avail_easyconfig_licenses: msg += license_documentation() # dump available easyblocks if self.options.list_easyblocks: msg += self.avail_easyblocks() # dump known toolchains if self.options.list_toolchains: msg += self.avail_toolchains() # dump known repository types if self.options.avail_repositories: msg += self.avail_repositories() # dump supported modules tools if self.options.avail_modules_tools: msg += self.avail_list('modules tools', avail_modules_tools()) # dump supported module naming schemes if self.options.avail_module_naming_schemes: msg += self.avail_list('module naming schemes', avail_module_naming_schemes()) # dump default moduleclasses with description if self.options.show_default_moduleclasses: msg += self.show_default_moduleclasses() if self.options.unittest_file: self.log.info(msg) else: print msg sys.exit(0) def avail_easyconfig_params(self): """ Print the available easyconfig parameters, for the given easyblock. """ app = get_class(self.options.easyblock) extra = app.extra_options() mapping = convert_to_help(extra, has_default=False) if len(extra) > 0: ebb_msg = " (* indicates specific for the %s EasyBlock)" % app.__name__ extra_names = [x[0] for x in extra] else: ebb_msg = '' extra_names = [] txt = ["Available easyconfig parameters%s" % ebb_msg] params = [(k, v) for (k, v) in mapping.items() if k.upper() not in ['HIDDEN']] for key, values in params: txt.append("%s" % key.upper()) txt.append('-' * len(key)) for name, value in values: tabs = "\t" * (3 - (len(name) + 1) / 8) if name in extra_names: starred = '(*)' else: starred = '' txt.append("%s%s:%s%s" % (name, starred, tabs, value)) txt.append('') return "\n".join(txt) def avail_classes_tree(self, classes, classNames, detailed, depth=0): """Print list of classes as a tree.""" txt = [] for className in classNames: classInfo = classes[className] if detailed: txt.append("%s|-- %s (%s)" % ("| " * depth, className, classInfo['module'])) else: txt.append("%s|-- %s" % ("| " * depth, className)) if 'children' in classInfo: txt.extend( self.avail_classes_tree(classes, classInfo['children'], detailed, depth + 1)) return txt def avail_easyblocks(self): """Get a class tree for easyblocks.""" detailed = self.options.list_easyblocks == "detailed" module_regexp = re.compile(r"^([^_].*)\.py$") # finish initialisation of the toolchain module (ie set the TC_CONSTANT constants) search_toolchain('') for package in [ "easybuild.easyblocks", "easybuild.easyblocks.generic" ]: __import__(package) # determine paths for this package paths = sys.modules[package].__path__ # import all modules in these paths for path in paths: if os.path.exists(path): for f in os.listdir(path): res = module_regexp.match(f) if res: __import__("%s.%s" % (package, res.group(1))) def add_class(classes, cls): """Add a new class, and all of its subclasses.""" children = cls.__subclasses__() classes.update({ cls.__name__: { 'module': cls.__module__, 'children': [x.__name__ for x in children] } }) for child in children: add_class(classes, child) roots = [EasyBlock, Extension] classes = {} for root in roots: add_class(classes, root) # Print the tree, start with the roots txt = [] for root in roots: root = root.__name__ if detailed: txt.append("%s (%s)" % (root, classes[root]['module'])) else: txt.append("%s" % root) if 'children' in classes[root]: txt.extend( self.avail_classes_tree(classes, classes[root]['children'], detailed)) txt.append("") return "\n".join(txt) def avail_toolchains(self): """Show list of known toolchains.""" _, all_tcs = search_toolchain('') all_tcs_names = [x.NAME for x in all_tcs] tclist = sorted(zip(all_tcs_names, all_tcs)) txt = ["List of known toolchains (toolchainname: module[,module...]):"] for (tcname, tcc) in tclist: tc = tcc( version='1.2.3' ) # version doesn't matter here, but something needs to be there tc_elems = set([ y for x in dir(tc) if x.endswith('_MODULE_NAME') for y in eval("tc.%s" % x) ]) txt.append("\t%s: %s" % (tcname, ', '.join(sorted(tc_elems)))) return '\n'.join(txt) def avail_repositories(self): """Show list of known repository types.""" repopath_defaults = get_default_oldstyle_configfile_defaults( )['repositorypath'] all_repos = avail_repositories(check_useable=False) usable_repos = avail_repositories(check_useable=True).keys() indent = ' ' * 2 txt = ['All avaialble repository types'] repos = sorted(all_repos.keys()) for repo in repos: if repo in usable_repos: missing = '' else: missing = ' (*Not usable*, something is missing (eg a specific module))' if repo in repopath_defaults: default = ' (Default arguments: %s)' % ( repopath_defaults[repo]) else: default = ' (No default arguments)' txt.append("%s%s%s%s" % (indent, repo, default, missing)) txt.append("%s%s" % (indent * 2, all_repos[repo].DESCRIPTION)) return "\n".join(txt) def avail_list(self, name, items): """Show list of available values passed by argument.""" return "List of supported %s:\n\t%s" % (name, '\n\t'.join(items)) def show_default_moduleclasses(self): """Show list of default moduleclasses and description.""" txt = ["Default available moduleclasses"] indent = " " * 2 maxlen = max([len(x[0]) for x in DEFAULT_MODULECLASSES]) + 1 # at least 1 space for name, descr in DEFAULT_MODULECLASSES: txt.append("%s%s:%s%s" % (indent, name, (" " * (maxlen - len(name))), descr)) return "\n".join(txt)
def main(testing_data=(None, None, None)): """ Main function: @arg options: a tuple: (options, paths, logger, logfile, hn) as defined in parse_options This function will: - read easyconfig - build software """ # disallow running EasyBuild as root if os.getuid() == 0: sys.stderr.write("ERROR: You seem to be running EasyBuild with root privileges.\n" "That's not wise, so let's end this here.\n" "Exiting.\n") sys.exit(1) # steer behavior when testing main testing = testing_data[0] is not None args, logfile, do_build = testing_data # initialise options eb_go = eboptions.parse_options(args=args) options = eb_go.options orig_paths = eb_go.args # set umask (as early as possible) if options.umask is not None: new_umask = int(options.umask, 8) old_umask = os.umask(new_umask) # set temporary directory to use eb_tmpdir = set_tmpdir(options.tmpdir) # initialise logging for main if options.logtostdout: fancylogger.logToScreen(enable=True, stdout=True) else: if logfile is None: # mkstemp returns (fd,filename), fd is from os.open, not regular open! fd, logfile = tempfile.mkstemp(suffix='.log', prefix='easybuild-') os.close(fd) fancylogger.logToFile(logfile) print_msg('temporary log file in case of crash %s' % (logfile), log=None, silent=testing) global _log _log = fancylogger.getLogger(fname=False) if options.umask is not None: _log.info("umask set to '%s' (used to be '%s')" % (oct(new_umask), oct(old_umask))) # hello world! _log.info(this_is_easybuild()) # how was EB called? eb_command_line = eb_go.generate_cmd_line() + eb_go.args _log.info("Command line: %s" % (" ".join(eb_command_line))) _log.info("Using %s as temporary directory" % eb_tmpdir) if not options.robot is None: if options.robot: _log.info("Using robot path(s): %s" % options.robot) else: _log.error("No robot paths specified, and unable to determine easybuild-easyconfigs install path.") # do not pass options.robot, it's not a list instance (and it shouldn't be modified) robot_path = None if options.robot: robot_path = list(options.robot) # determine easybuild-easyconfigs package install path easyconfigs_paths = get_paths_for("easyconfigs", robot_path=robot_path) # keep track of paths for install easyconfigs, so we can obtain find specified easyconfigs easyconfigs_pkg_full_paths = easyconfigs_paths[:] if not easyconfigs_paths: _log.warning("Failed to determine install path for easybuild-easyconfigs package.") # specified robot paths are preferred over installed easyconfig files if robot_path: robot_path.extend(easyconfigs_paths) easyconfigs_paths = robot_path[:] _log.info("Extended list of robot paths with paths for installed easyconfigs: %s" % robot_path) # initialise the easybuild configuration config.init(options, eb_go.get_options_by_section('config')) # building a dependency graph implies force, so that all dependencies are retained # and also skips validation of easyconfigs (e.g. checking os dependencies) retain_all_deps = False if options.dep_graph: _log.info("Enabling force to generate dependency graph.") options.force = True retain_all_deps = True config.init_build_options({ 'aggregate_regtest': options.aggregate_regtest, 'allow_modules_tool_mismatch': options.allow_modules_tool_mismatch, 'check_osdeps': not options.ignore_osdeps, 'command_line': eb_command_line, 'debug': options.debug, 'dry_run': options.dry_run, 'easyblock': options.easyblock, 'experimental': options.experimental, 'force': options.force, 'group': options.group, 'ignore_dirs': options.ignore_dirs, 'modules_footer': options.modules_footer, 'only_blocks': options.only_blocks, 'recursive_mod_unload': options.recursive_module_unload, 'regtest_online': options.regtest_online, 'regtest_output_dir': options.regtest_output_dir, 'retain_all_deps': retain_all_deps, 'robot_path': robot_path, 'sequential': options.sequential, 'silent': testing, 'set_gid_bit': options.set_gid_bit, 'skip': options.skip, 'skip_test_cases': options.skip_test_cases, 'sticky_bit': options.sticky_bit, 'stop': options.stop, 'umask': options.umask, 'valid_module_classes': module_classes(), 'valid_stops': [x[0] for x in EasyBlock.get_steps()], 'validate': not options.force, }) # search for easyconfigs if options.search or options.search_short: search_path = [os.getcwd()] if easyconfigs_paths: search_path = easyconfigs_paths query = options.search or options.search_short ignore_dirs = config.build_option('ignore_dirs') silent = config.build_option('silent') search_file(search_path, query, short=not options.search, ignore_dirs=ignore_dirs, silent=silent) # process software build specifications (if any), i.e. # software name/version, toolchain name/version, extra patches, ... (try_to_generate, build_specs) = process_software_build_specs(options) paths = [] if len(orig_paths) == 0: if 'name' in build_specs: paths = [obtain_path(build_specs, easyconfigs_paths, try_to_generate=try_to_generate, exit_on_error=not testing)] elif not any([options.aggregate_regtest, options.search, options.search_short, options.regtest]): print_error(("Please provide one or multiple easyconfig files, or use software build " "options to make EasyBuild search for easyconfigs"), log=_log, opt_parser=eb_go.parser, exit_on_error=not testing) else: # look for easyconfigs with relative paths in easybuild-easyconfigs package, # unless they were found at the given relative paths if easyconfigs_pkg_full_paths: # determine which easyconfigs files need to be found, if any ecs_to_find = [] for idx, orig_path in enumerate(orig_paths): if orig_path == os.path.basename(orig_path) and not os.path.exists(orig_path): ecs_to_find.append((idx, orig_path)) _log.debug("List of easyconfig files to find: %s" % ecs_to_find) # find missing easyconfigs by walking paths with installed easyconfig files for path in easyconfigs_pkg_full_paths: _log.debug("Looking for missing easyconfig files (%d left) in %s..." % (len(ecs_to_find), path)) for (subpath, dirnames, filenames) in os.walk(path, topdown=True): for idx, orig_path in ecs_to_find[:]: if orig_path in filenames: full_path = os.path.join(subpath, orig_path) _log.info("Found %s in %s: %s" % (orig_path, path, full_path)) orig_paths[idx] = full_path # if file was found, stop looking for it (first hit wins) ecs_to_find.remove((idx, orig_path)) # stop os.walk insanity as soon as we have all we need (os.walk loop) if len(ecs_to_find) == 0: break # ignore subdirs specified to be ignored by replacing items in dirnames list used by os.walk dirnames[:] = [d for d in dirnames if not d in options.ignore_dirs] # stop os.walk insanity as soon as we have all we need (paths loop) if len(ecs_to_find) == 0: break # indicate that specified paths do not contain generated easyconfig files paths = [(path, False) for path in orig_paths] _log.debug("Paths: %s" % paths) # run regtest if options.regtest or options.aggregate_regtest: _log.info("Running regression test") if paths: ec_paths = [path[0] for path in paths] else: # fallback: easybuild-easyconfigs install path ec_paths = easyconfigs_pkg_full_paths regtest_ok = regtest(ec_paths) if not regtest_ok: _log.info("Regression test failed (partially)!") sys.exit(31) # exit -> 3x1t -> 31 # read easyconfig files easyconfigs = [] for (path, generated) in paths: path = os.path.abspath(path) if not os.path.exists(path): print_error("Can't find path %s" % path) try: files = find_easyconfigs(path, ignore_dirs=options.ignore_dirs) for f in files: if not generated and try_to_generate and build_specs: ec_file = tweak(f, None, build_specs) else: ec_file = f ecs = process_easyconfig(ec_file, build_specs=build_specs) easyconfigs.extend(ecs) except IOError, err: _log.error("Processing easyconfigs in path %s failed: %s" % (path, err))
def main(testing_data=(None, None)): """ Main function: @arg options: a tuple: (options, paths, logger, logfile, hn) as defined in parse_options This function will: - read easyconfig - build software """ # disallow running EasyBuild as root if os.getuid() == 0: sys.stderr.write( "ERROR: You seem to be running EasyBuild with root privileges.\n" "That's not wise, so let's end this here.\n" "Exiting.\n" ) sys.exit(1) # steer behavior when testing main testing = testing_data[0] is not None args, logfile = testing_data # initialise options eb_go = eboptions.parse_options(args=args) options = eb_go.options orig_paths = eb_go.args # initialise logging for main if options.logtostdout: fancylogger.logToScreen(enable=True, stdout=True) else: if logfile is None: # mkstemp returns (fd,filename), fd is from os.open, not regular open! fd, logfile = tempfile.mkstemp(suffix=".log", prefix="easybuild-") os.close(fd) fancylogger.logToFile(logfile) print_msg("temporary log file in case of crash %s" % (logfile), log=None, silent=testing) global _log _log = fancylogger.getLogger(fname=False) # hello world! _log.info(this_is_easybuild()) # set strictness of filetools module if options.strict: filetools.strictness = options.strict if not options.robot is None: if options.robot: _log.info("Using robot path: %s" % options.robot) else: _log.error("No robot path specified, and unable to determine easybuild-easyconfigs install path.") # determine easybuild-easyconfigs package install path easyconfigs_paths = get_paths_for("easyconfigs", robot_path=options.robot) easyconfigs_pkg_full_path = None search_path = os.getcwd() if easyconfigs_paths: easyconfigs_pkg_full_path = easyconfigs_paths[0] if not options.robot: search_path = easyconfigs_pkg_full_path else: search_path = options.robot else: _log.info("Failed to determine install path for easybuild-easyconfigs package.") if options.robot: easyconfigs_paths = [options.robot] + easyconfigs_paths # initialise the easybuild configuration config.init(options, eb_go.get_options_by_section("config")) # search for easyconfigs if options.search: search_file(search_path, options.search, silent=testing) # process software build specifications (if any), i.e. # software name/version, toolchain name/version, extra patches, ... (try_to_generate, software_build_specs) = process_software_build_specs(options) paths = [] if len(orig_paths) == 0: if software_build_specs.has_key("name"): paths = [ obtain_path( software_build_specs, easyconfigs_paths, try_to_generate=try_to_generate, exit_on_error=not testing ) ] elif not any([options.aggregate_regtest, options.search, options.regtest]): print_error( ( "Please provide one or multiple easyconfig files, or use software build " "options to make EasyBuild search for easyconfigs" ), log=_log, opt_parser=eb_go.parser, exit_on_error=not testing, ) else: # look for easyconfigs with relative paths in easybuild-easyconfigs package, # unless they we found at the given relative paths if easyconfigs_pkg_full_path: # create a mapping from filename to path in easybuild-easyconfigs package install path easyconfigs_map = {} for (subpath, _, filenames) in os.walk(easyconfigs_pkg_full_path): for filename in filenames: easyconfigs_map.update({filename: os.path.join(subpath, filename)}) # try and find non-existing non-absolute eaysconfig paths in easybuild-easyconfigs package install path for idx, orig_path in enumerate(orig_paths): if not os.path.isabs(orig_path) and not os.path.exists(orig_path): if orig_path in easyconfigs_map: _log.info( "Found %s in %s: %s" % (orig_path, easyconfigs_pkg_full_path, easyconfigs_map[orig_path]) ) orig_paths[idx] = easyconfigs_map[orig_path] # indicate that specified paths do not contain generated easyconfig files paths = [(path, False) for path in orig_paths] _log.debug("Paths: %s" % paths) # run regtest if options.regtest or options.aggregate_regtest: _log.info("Running regression test") if paths: regtest_ok = regtest(options, [path[0] for path in paths]) else: # fallback: easybuild-easyconfigs install path regtest_ok = regtest(options, [easyconfigs_pkg_full_path]) if not regtest_ok: _log.info("Regression test failed (partially)!") sys.exit(31) # exit -> 3x1t -> 31 if any([options.search, options.regtest]): cleanup_logfile_and_exit(logfile, testing, True) # building a dependency graph implies force, so that all dependencies are retained # and also skips validation of easyconfigs (e.g. checking os dependencies) validate_easyconfigs = True retain_all_deps = False if options.dep_graph: _log.info("Enabling force to generate dependency graph.") options.force = True validate_easyconfigs = False retain_all_deps = True # read easyconfig files easyconfigs = [] for (path, generated) in paths: path = os.path.abspath(path) if not (os.path.exists(path)): print_error("Can't find path %s" % path) try: files = find_easyconfigs(path) for f in files: if not generated and try_to_generate and software_build_specs: ec_file = easyconfig.tools.tweak(f, None, software_build_specs) else: ec_file = f easyconfigs.extend(process_easyconfig(ec_file, options.only_blocks, validate=validate_easyconfigs)) except IOError, err: _log.error("Processing easyconfigs in path %s failed: %s" % (path, err))
class EasyBuildOptions(GeneralOption): """Easybuild generaloption class""" VERSION = this_is_easybuild() DEFAULT_LOGLEVEL = 'INFO' DEFAULT_CONFIGFILES = get_default_configfiles() ALLOPTSMANDATORY = False # allow more than one argument def basic_options(self): """basic runtime options""" all_stops = [x[0] for x in EasyBlock.get_steps()] strictness_options = [run.IGNORE, run.WARN, run.ERROR] try: default_robot_path = get_paths_for("easyconfigs", robot_path=None)[0] except: self.log.warning("basic_options: unable to determine default easyconfig path") default_robot_path = False # False as opposed to None, since None is used for indicating that --robot was not used descr = ("Basic options", "Basic runtime options for EasyBuild.") opts = OrderedDict({ 'dry-run': ("Print build overview incl. dependencies (full paths)", None, 'store_true', False), 'dry-run-short': ("Print build overview incl. dependencies (short paths)", None, 'store_true', False, 'D'), 'force': ("Force to rebuild software even if it's already installed (i.e. if it can be found as module)", None, 'store_true', False, 'f'), 'job': ("Submit the build as a job", None, 'store_true', False), 'logtostdout': ("Redirect main log to stdout", None, 'store_true', False, 'l'), 'only-blocks': ("Only build listed blocks", None, 'extend', None, 'b', {'metavar': 'BLOCKS'}), 'robot': ("Path(s) to search for easyconfigs for missing dependencies (colon-separated)" , None, 'store_or_None', default_robot_path, 'r', {'metavar': 'PATH'}), 'skip': ("Skip existing software (useful for installing additional packages)", None, 'store_true', False, 'k'), 'stop': ("Stop the installation after certain step", 'choice', 'store_or_None', 'source', 's', all_stops), 'strict': ("Set strictness level", 'choice', 'store', run.WARN, strictness_options), }) self.log.debug("basic_options: descr %s opts %s" % (descr, opts)) self.add_group_parser(opts, descr) def software_options(self): # software build options descr = ("Software search and build options", ("Specify software search and build options: EasyBuild will search for a " "matching easyconfig and build it. When called with the try prefix " "(i.e. --try-X ), EasyBuild will search for a matching easyconfig " "and if none are found, try to generate one based on a close matching one " "(NOTE: --try-X is best effort, it might produce wrong builds!)") ) opts = OrderedDict({ 'amend':(("Specify additional search and build parameters (can be used multiple times); " "for example: versionprefix=foo or patches=one.patch,two.patch)"), None, 'append', None, {'metavar': 'VAR=VALUE[,VALUE]'}), 'software-name': ("Search and build software with name", None, 'store', None, {'metavar': 'NAME'}), 'software-version': ("Search and build software with version", None, 'store', None, {'metavar': 'VERSION'}), 'toolchain': ("Search and build with toolchain (name and version)", None, 'extend', None, {'metavar': 'NAME,VERSION'}), 'toolchain-name': ("Search and build with toolchain name", None, 'store', None, {'metavar': 'NAME'}), 'toolchain-version': ("Search and build with toolchain version", None, 'store', None, {'metavar': 'VERSION'}), }) longopts = opts.keys() for longopt in longopts: hlp = opts[longopt][0] hlp = "Try to %s (USE WITH CARE!)" % (hlp[0].lower() + hlp[1:]) opts["try-%s" % longopt] = (hlp,) + opts[longopt][1:] # additional options that don't need a --try equivalent opts.update({ 'from-pr': ("Obtain easyconfigs from specified PR", int, 'store', None, {'metavar': 'PR#'}), }) self.log.debug("software_options: descr %s opts %s" % (descr, opts)) self.add_group_parser(opts, descr) def override_options(self): # override options descr = ("Override options", "Override default EasyBuild behavior.") opts = OrderedDict({ 'allow-modules-tool-mismatch': ("Allow mismatch of modules tool and definition of 'module' function", None, 'store_true', False), 'cleanup-builddir': ("Cleanup build dir after successful installation.", None, 'store_true', True), 'deprecated': ("Run pretending to be (future) version, to test removal of deprecated code.", None, 'store', None), 'easyblock': ("easyblock to use for processing the spec file or dumping the options", None, 'store', None, 'e', {'metavar': 'CLASS'}), 'experimental': ("Allow experimental code (with behaviour that can be changed or removed at any given time).", None, 'store_true', False), 'group': ("Group to be used for software installations (only verified, not set)", None, 'store', None), 'ignore-osdeps': ("Ignore any listed OS dependencies", None, 'store_true', False), 'oldstyleconfig': ("Look for and use the oldstyle configuration file.", None, 'store_true', True), 'pretend': (("Does the build/installation in a test directory located in $HOME/easybuildinstall"), None, 'store_true', False, 'p'), 'set-gid-bit': ("Set group ID bit on newly created directories", None, 'store_true', False), 'sticky-bit': ("Set sticky bit on newly created directories", None, 'store_true', False), 'skip-test-cases': ("Skip running test cases", None, 'store_true', False, 't'), 'umask': ("umask to use (e.g. '022'); non-user write permissions on install directories are removed", None, 'store', None), }) self.log.debug("override_options: descr %s opts %s" % (descr, opts)) self.add_group_parser(opts, descr) def config_options(self): # config options descr = ("Configuration options", "Configure EasyBuild behavior.") oldstyle_defaults = get_default_oldstyle_configfile_defaults() opts = OrderedDict({ 'avail-module-naming-schemes': ("Show all supported module naming schemes", None, 'store_true', False,), 'avail-modules-tools': ("Show all supported module tools", None, "store_true", False,), 'avail-repositories': ("Show all repository types (incl. non-usable)", None, "store_true", False,), 'buildpath': ("Temporary build path", None, 'store', oldstyle_defaults['buildpath']), 'ignore-dirs': ("Directory names to ignore when searching for files/dirs", 'strlist', 'store', ['.git', '.svn']), 'installpath': ("Install path for software and modules", None, 'store', oldstyle_defaults['installpath']), 'config': ("Path to EasyBuild config file", None, 'store', oldstyle_defaults['config'], 'C'), 'logfile-format': ("Directory name and format of the log file", 'strtuple', 'store', oldstyle_defaults['logfile_format'], {'metavar': 'DIR,FORMAT'}), 'module-naming-scheme': ("Module naming scheme", 'choice', 'store', oldstyle_defaults['module_naming_scheme'], sorted(avail_module_naming_schemes().keys())), 'moduleclasses': (("Extend supported module classes " "(For more info on the default classes, use --show-default-moduleclasses)"), None, 'extend', oldstyle_defaults['moduleclasses']), 'modules-footer': ("Path to file containing footer to be added to all generated module files", None, 'store_or_None', None, {'metavar': "PATH"}), 'modules-tool': ("Modules tool to use", 'choice', 'store', oldstyle_defaults['modules_tool'], sorted(avail_modules_tools().keys())), 'prefix': (("Change prefix for buildpath, installpath, sourcepath and repositorypath " "(repositorypath prefix is only relevant in case of FileRepository repository) " "(used prefix for defaults %s)" % oldstyle_defaults['prefix']), None, 'store', None), 'recursive-module-unload': ("Enable generating of modules that unload recursively.", None, 'store_true', False), 'repository': ("Repository type, using repositorypath", 'choice', 'store', oldstyle_defaults['repository'], sorted(avail_repositories().keys())), 'repositorypath': (("Repository path, used by repository " "(is passed as list of arguments to create the repository instance). " "For more info, use --avail-repositories."), 'strlist', 'store', oldstyle_defaults['repositorypath'][oldstyle_defaults['repository']]), 'show-default-moduleclasses': ("Show default module classes with description", None, 'store_true', False), 'sourcepath': ("Path(s) to where sources should be downloaded (string, colon-separated)", None, 'store', oldstyle_defaults['sourcepath']), 'subdir-modules': ("Installpath subdir for modules", None, 'store', oldstyle_defaults['subdir_modules']), 'subdir-software': ("Installpath subdir for software", None, 'store', oldstyle_defaults['subdir_software']), # this one is sort of an exception, it's something jobscripts can set, # has no real meaning for regular eb usage 'testoutput': ("Path to where a job should place the output (to be set within jobscript)", None, 'store', None), 'tmp-logdir': ("Log directory where temporary log files are stored", None, 'store', oldstyle_defaults['tmp_logdir']), 'tmpdir': ('Directory to use for temporary storage', None, 'store', None), }) self.log.debug("config_options: descr %s opts %s" % (descr, opts)) self.add_group_parser(opts, descr) def informative_options(self): # informative options descr = ("Informative options", "Obtain information about EasyBuild.") opts = OrderedDict({ 'avail-easyconfig-constants': ("Show all constants that can be used in easyconfigs", None, 'store_true', False), 'avail-easyconfig-licenses': ("Show all license constants that can be used in easyconfigs", None, 'store_true', False), 'avail-easyconfig-params': (("Show all easyconfig parameters (include " "easyblock-specific ones by using -e)"), None, "store_true", False, 'a'), 'avail-easyconfig-templates': (("Show all template names and template constants " "that can be used in easyconfigs"), None, 'store_true', False), 'dep-graph': ("Create dependency graph", None, "store", None, {'metavar': 'depgraph.<ext>'}), 'list-easyblocks': ("Show list of available easyblocks", 'choice', 'store_or_None', 'simple', ['simple', 'detailed']), 'list-toolchains': ("Show list of known toolchains", None, 'store_true', False), 'search': ("Search for easyconfig files in the robot directory, print full paths", None, 'store', None, {'metavar': 'STR'}), 'search-short': ("Search for easyconfig files in the robot directory, print short paths", None, 'store', None, 'S', {'metavar': 'STR'}), }) self.log.debug("informative_options: descr %s opts %s" % (descr, opts)) self.add_group_parser(opts, descr) def regtest_options(self): # regression test options descr = ("Regression test options", "Run and control an EasyBuild regression test.") opts = OrderedDict({ 'aggregate-regtest': ("Collect all the xmls inside the given directory and generate a single file", None, 'store', None, {'metavar': 'DIR'}), 'dump-test-report': ("Dump test report to specified path", None, 'store_or_None', 'test_report.md'), 'github-user': ("GitHub username", None, 'store', None), 'regtest': ("Enable regression test mode", None, 'store_true', False), 'regtest-output-dir': ("Set output directory for test-run", None, 'store', None, {'metavar': 'DIR'}), 'sequential': ("Specify this option if you want to prevent parallel build", None, 'store_true', False), 'upload-test-report': ("Upload full test report as a gist on GitHub", None, 'store_true', None), }) self.log.debug("regtest_options: descr %s opts %s" % (descr, opts)) self.add_group_parser(opts, descr) def easyconfig_options(self): # easyconfig options (to be passed to easyconfig instance) descr = ("Options for Easyconfigs", "Options to be passed to all Easyconfig.") opts = None self.log.debug("easyconfig_options: descr %s opts %s" % (descr, opts)) self.add_group_parser(opts, descr, prefix='easyconfig') def easyblock_options(self): # easyblock options (to be passed to easyblock instance) descr = ("Options for Easyblocks", "Options to be passed to all Easyblocks.") opts = None self.log.debug("easyblock_options: descr %s opts %s" % (descr, opts)) self.add_group_parser(opts, descr, prefix='easyblock') def unittest_options(self): # unittest options descr = ("Unittest options", "Options dedicated to unittesting (experts only).") opts = OrderedDict({ 'file': ("Log to this file in unittest mode", None, 'store', None), }) self.log.debug("unittest_options: descr %s opts %s" % (descr, opts)) self.add_group_parser(opts, descr, prefix='unittest') def validate(self): """Additional validation of options""" stop_msg = [] if self.options.toolchain and not len(self.options.toolchain) == 2: stop_msg.append('--toolchain requires NAME,VERSION (given %s)' % (','.join(self.options.toolchain))) if self.options.try_toolchain and not len(self.options.try_toolchain) == 2: stop_msg.append('--try-toolchain requires NAME,VERSION (given %s)' % (','.join(self.options.try_toolchain))) if self.options.umask: umask_regex = re.compile('^[0-7]{3}$') if not umask_regex.match(self.options.umask): stop_msg.append("--umask value should be 3 digits (0-7) (regex pattern '%s')" % umask_regex.pattern) if len(stop_msg) > 0: indent = " "*2 stop_msg = ['%s%s' % (indent, x) for x in stop_msg] stop_msg.insert(0, 'ERROR: Found %s problems validating the options:' % len(stop_msg)) print "\n".join(stop_msg) sys.exit(1) def postprocess(self): """Do some postprocessing, in particular print stuff""" build_log.EXPERIMENTAL = self.options.experimental config.SUPPORT_OLDSTYLE = self.options.oldstyleconfig # set strictness of run module if self.options.strict: run.strictness = self.options.strict # override current version of EasyBuild with version specified to --deprecated if self.options.deprecated: build_log.CURRENT_VERSION = LooseVersion(self.options.deprecated) # log to specified value of --unittest-file if self.options.unittest_file: fancylogger.logToFile(self.options.unittest_file) # prepare for --list/--avail if any([self.options.avail_easyconfig_params, self.options.avail_easyconfig_templates, self.options.list_easyblocks, self.options.list_toolchains, self.options.avail_easyconfig_constants, self.options.avail_easyconfig_licenses, self.options.avail_repositories, self.options.show_default_moduleclasses, self.options.avail_modules_tools, self.options.avail_module_naming_schemes, ]): build_easyconfig_constants_dict() # runs the easyconfig constants sanity check self._postprocess_list_avail() # fail early if required dependencies for functionality requiring using GitHub API are not available: if self.options.from_pr or self.options.upload_test_report: if not HAVE_GITHUB_API: self.log.error("Required support for using GitHub API is not available (see warnings).") # make sure a GitHub token is available when it's required if self.options.upload_test_report: if not HAVE_KEYRING: self.log.error("Python 'keyring' module required for obtaining GitHub token is not available.") if self.options.github_user is None: self.log.error("No GitHub user name provided, required for fetching GitHub token.") token = fetch_github_token(self.options.github_user) if token is None: self.log.error("Failed to obtain required GitHub token for user '%s'" % self.options.github_user) self._postprocess_config() def _postprocess_config(self): """Postprocessing of configuration options""" if self.options.prefix is not None: changed_defaults = get_default_oldstyle_configfile_defaults(self.options.prefix) for dest in ['installpath', 'buildpath', 'sourcepath', 'repositorypath']: if not self.options._action_taken.get(dest, False): new_def = changed_defaults[dest] if dest == 'repositorypath': setattr(self.options, dest, new_def[changed_defaults['repository']]) else: setattr(self.options, dest, new_def) # LEGACY this line is here for oldstyle reasons self.log.deprecated('Fake action taken to distinguish from default', '2.0') self.options._action_taken[dest] = True if self.options.pretend: self.options.installpath = get_pretend_installpath() # split supplied list of robot paths to obtain a list if self.options.robot: class RobotPath(ListOfStrings): SEPARATOR_LIST = os.pathsep # explicit definition of __str__ is required for unknown reason related to the way Wrapper is defined __str__ = ListOfStrings.__str__ self.options.robot = RobotPath(self.options.robot) def _postprocess_list_avail(self): """Create all the additional info that can be requested (exit at the end)""" msg = '' # dump possible easyconfig params if self.options.avail_easyconfig_params: msg += self.avail_easyconfig_params() # dump easyconfig template options if self.options.avail_easyconfig_templates: msg += template_documentation() # dump easyconfig constant options if self.options.avail_easyconfig_constants: msg += constant_documentation() # dump easyconfig license options if self.options.avail_easyconfig_licenses: msg += license_documentation() # dump available easyblocks if self.options.list_easyblocks: msg += self.avail_easyblocks() # dump known toolchains if self.options.list_toolchains: msg += self.avail_toolchains() # dump known repository types if self.options.avail_repositories: msg += self.avail_repositories() # dump supported modules tools if self.options.avail_modules_tools: msg += self.avail_list('modules tools', avail_modules_tools()) # dump supported module naming schemes if self.options.avail_module_naming_schemes: msg += self.avail_list('module naming schemes', avail_module_naming_schemes()) # dump default moduleclasses with description if self.options.show_default_moduleclasses: msg += self.show_default_moduleclasses() if self.options.unittest_file: self.log.info(msg) else: print msg sys.exit(0) def avail_easyconfig_params(self): """ Print the available easyconfig parameters, for the given easyblock. """ app = get_easyblock_class(self.options.easyblock) extra = app.extra_options() mapping = convert_to_help(extra, has_default=False) if len(extra) > 0: ebb_msg = " (* indicates specific for the %s EasyBlock)" % app.__name__ extra_names = [x[0] for x in extra] else: ebb_msg = '' extra_names = [] txt = ["Available easyconfig parameters%s" % ebb_msg] params = [(k, v) for (k, v) in mapping.items() if k.upper() not in ['HIDDEN']] for key, values in params: txt.append("%s" % key.upper()) txt.append('-' * len(key)) for name, value in values: tabs = "\t" * (3 - (len(name) + 1) / 8) if name in extra_names: starred = '(*)' else: starred = '' txt.append("%s%s:%s%s" % (name, starred, tabs, value)) txt.append('') return "\n".join(txt) def avail_classes_tree(self, classes, classNames, detailed, depth=0): """Print list of classes as a tree.""" txt = [] for className in classNames: classInfo = classes[className] if detailed: txt.append("%s|-- %s (%s)" % ("| " * depth, className, classInfo['module'])) else: txt.append("%s|-- %s" % ("| " * depth, className)) if 'children' in classInfo: txt.extend(self.avail_classes_tree(classes, classInfo['children'], detailed, depth + 1)) return txt def avail_easyblocks(self): """Get a class tree for easyblocks.""" detailed = self.options.list_easyblocks == "detailed" module_regexp = re.compile(r"^([^_].*)\.py$") # finish initialisation of the toolchain module (ie set the TC_CONSTANT constants) search_toolchain('') for package in ["easybuild.easyblocks", "easybuild.easyblocks.generic"]: __import__(package) # determine paths for this package paths = sys.modules[package].__path__ # import all modules in these paths for path in paths: if os.path.exists(path): for f in os.listdir(path): res = module_regexp.match(f) if res: __import__("%s.%s" % (package, res.group(1))) def add_class(classes, cls): """Add a new class, and all of its subclasses.""" children = cls.__subclasses__() classes.update({cls.__name__: { 'module': cls.__module__, 'children': [x.__name__ for x in children] } }) for child in children: add_class(classes, child) roots = [EasyBlock, Extension] classes = {} for root in roots: add_class(classes, root) # Print the tree, start with the roots txt = [] for root in roots: root = root.__name__ if detailed: txt.append("%s (%s)" % (root, classes[root]['module'])) else: txt.append("%s" % root) if 'children' in classes[root]: txt.extend(self.avail_classes_tree(classes, classes[root]['children'], detailed)) txt.append("") return "\n".join(txt) def avail_toolchains(self): """Show list of known toolchains.""" _, all_tcs = search_toolchain('') all_tcs_names = [x.NAME for x in all_tcs] tclist = sorted(zip(all_tcs_names, all_tcs)) txt = ["List of known toolchains (toolchainname: module[,module...]):"] for (tcname, tcc) in tclist: tc = tcc(version='1.2.3') # version doesn't matter here, but something needs to be there tc_elems = set([y for x in dir(tc) if x.endswith('_MODULE_NAME') for y in eval("tc.%s" % x)]) txt.append("\t%s: %s" % (tcname, ', '.join(sorted(tc_elems)))) return '\n'.join(txt) def avail_repositories(self): """Show list of known repository types.""" repopath_defaults = get_default_oldstyle_configfile_defaults()['repositorypath'] all_repos = avail_repositories(check_useable=False) usable_repos = avail_repositories(check_useable=True).keys() indent = ' ' * 2 txt = ['All avaliable repository types'] repos = sorted(all_repos.keys()) for repo in repos: if repo in usable_repos: missing = '' else: missing = ' (*not usable*, something is missing (e.g. a required Python module))' if repo in repopath_defaults: default = ' (default arguments: %s)' % ', '.join(repopath_defaults[repo]) else: default = ' (no default arguments)' txt.append("%s* %s%s%s" % (indent, repo, default, missing)) txt.append("%s%s" % (indent * 3, all_repos[repo].DESCRIPTION)) return "\n".join(txt) def avail_list(self, name, items): """Show list of available values passed by argument.""" return "List of supported %s:\n\t%s" % (name, '\n\t'.join(items)) def show_default_moduleclasses(self): """Show list of default moduleclasses and description.""" txt = ["Default available moduleclasses"] indent = " " * 2 maxlen = max([len(x[0]) for x in DEFAULT_MODULECLASSES]) + 1 # at least 1 space for name, descr in DEFAULT_MODULECLASSES: txt.append("%s%s:%s%s" % (indent, name, (" " * (maxlen - len(name))), descr)) return "\n".join(txt)
class EasyBuildOptions(GeneralOption): """Easybuild generaloption class""" VERSION = this_is_easybuild() DEFAULT_LOGLEVEL = 'INFO' DEFAULT_CONFIGFILES = DEFAULT_SYS_CFGFILES + [DEFAULT_USER_CFGFILE] ALLOPTSMANDATORY = False # allow more than one argument def __init__(self, *args, **kwargs): """Constructor.""" self.default_robot_paths = get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR, robot_path=None) or [] # set up constants to seed into config files parser, by section self.go_cfg_constants = { self.DEFAULTSECT: { 'DEFAULT_ROBOT_PATHS': (os.pathsep.join(self.default_robot_paths), "List of default robot paths ('%s'-separated)" % os.pathsep), } } # update or define go_configfiles_initenv in named arguments to pass to parent constructor go_cfg_initenv = kwargs.setdefault('go_configfiles_initenv', {}) for section, constants in self.go_cfg_constants.items(): constants = dict([(name, value) for (name, (value, _)) in constants.items()]) go_cfg_initenv.setdefault(section, {}).update(constants) super(EasyBuildOptions, self).__init__(*args, **kwargs) def basic_options(self): """basic runtime options""" all_stops = [x[0] for x in EasyBlock.get_steps()] strictness_options = [run.IGNORE, run.WARN, run.ERROR] descr = ("Basic options", "Basic runtime options for EasyBuild.") opts = OrderedDict({ 'dry-run': ("Print build overview incl. dependencies (full paths)", None, 'store_true', False), 'dry-run-short': ("Print build overview incl. dependencies (short paths)", None, 'store_true', False, 'D'), 'force': ("Force to rebuild software even if it's already installed (i.e. if it can be found as module)", None, 'store_true', False, 'f'), 'job': ("Submit the build as a job", None, 'store_true', False), 'logtostdout': ("Redirect main log to stdout", None, 'store_true', False, 'l'), 'only-blocks': ("Only build listed blocks", None, 'extend', None, 'b', { 'metavar': 'BLOCKS' }), 'robot': ("Enable dependency resolution, using easyconfigs in specified paths", 'pathlist', 'store_or_None', [], 'r', { 'metavar': 'PATH[%sPATH]' % os.pathsep }), 'robot-paths': ("Additional paths to consider by robot for easyconfigs (--robot paths get priority)", 'pathlist', 'store', self.default_robot_paths, { 'metavar': 'PATH[%sPATH]' % os.pathsep }), 'skip': ("Skip existing software (useful for installing additional packages)", None, 'store_true', False, 'k'), 'stop': ("Stop the installation after certain step", 'choice', 'store_or_None', 'source', 's', all_stops), 'strict': ("Set strictness level", 'choice', 'store', run.WARN, strictness_options), }) self.log.debug("basic_options: descr %s opts %s" % (descr, opts)) self.add_group_parser(opts, descr) def software_options(self): # software build options descr = ("Software search and build options", ( "Specify software search and build options: EasyBuild will search for a " "matching easyconfig and build it. When called with the try prefix " "(i.e. --try-X ), EasyBuild will search for a matching easyconfig " "and if none are found, try to generate one based on a close matching one " "(NOTE: --try-X is best effort, it might produce wrong builds!)")) opts = OrderedDict({ 'amend': (("Specify additional search and build parameters (can be used multiple times); " "for example: versionprefix=foo or patches=one.patch,two.patch)" ), None, 'append', None, { 'metavar': 'VAR=VALUE[,VALUE]' }), 'software': ("Search and build software with given name and version", None, 'extend', None, { 'metavar': 'NAME,VERSION' }), 'software-name': ("Search and build software with given name", None, 'store', None, { 'metavar': 'NAME' }), 'software-version': ("Search and build software with given version", None, 'store', None, { 'metavar': 'VERSION' }), 'toolchain': ("Search and build with given toolchain (name and version)", None, 'extend', None, { 'metavar': 'NAME,VERSION' }), 'toolchain-name': ("Search and build with given toolchain name", None, 'store', None, { 'metavar': 'NAME' }), 'toolchain-version': ("Search and build with given toolchain version", None, 'store', None, { 'metavar': 'VERSION' }), }) longopts = opts.keys() for longopt in longopts: hlp = opts[longopt][0] hlp = "Try to %s (USE WITH CARE!)" % (hlp[0].lower() + hlp[1:]) opts["try-%s" % longopt] = (hlp, ) + opts[longopt][1:] # additional options that don't need a --try equivalent opts.update({ 'from-pr': ("Obtain easyconfigs from specified PR", int, 'store', None, { 'metavar': 'PR#' }), }) self.log.debug("software_options: descr %s opts %s" % (descr, opts)) self.add_group_parser(opts, descr) def override_options(self): # override options descr = ("Override options", "Override default EasyBuild behavior.") opts = OrderedDict({ 'allow-modules-tool-mismatch': ("Allow mismatch of modules tool and definition of 'module' function", None, 'store_true', False), 'cleanup-builddir': ("Cleanup build dir after successful installation.", None, 'store_true', True), 'deprecated': ("Run pretending to be (future) version, to test removal of deprecated code.", None, 'store', None), 'download-timeout': ("Timeout for initiating downloads (in seconds)", None, 'store', None), 'easyblock': ("easyblock to use for processing the spec file or dumping the options", None, 'store', None, 'e', { 'metavar': 'CLASS' }), 'experimental': ("Allow experimental code (with behaviour that can be changed/removed at any given time).", None, 'store_true', False), 'group': ("Group to be used for software installations (only verified, not set)", None, 'store', None), 'hidden': ("Install 'hidden' module file(s) by prefixing their name with '.'", None, 'store_true', False), 'ignore-osdeps': ("Ignore any listed OS dependencies", None, 'store_true', False), 'filter-deps': ("Comma separated list of dependencies that you DON'T want to install with EasyBuild, " "because equivalent OS packages are installed. (e.g. --filter-deps=zlib,ncurses)", str, 'extend', None), 'oldstyleconfig': ("Look for and use the oldstyle configuration file.", None, 'store_true', True), 'optarch': ("Set architecture optimization, overriding native architecture optimizations", None, 'store', None), 'pretend': (("Does the build/installation in a test directory located in $HOME/easybuildinstall" ), None, 'store_true', False, 'p'), 'set-gid-bit': ("Set group ID bit on newly created directories", None, 'store_true', False), 'sticky-bit': ("Set sticky bit on newly created directories", None, 'store_true', False), 'skip-test-cases': ("Skip running test cases", None, 'store_true', False, 't'), 'umask': ("umask to use (e.g. '022'); non-user write permissions on install directories are removed", None, 'store', None), 'update-modules-tool-cache': ("Update modules tool cache file(s) after generating module file", None, 'store_true', False), }) self.log.debug("override_options: descr %s opts %s" % (descr, opts)) self.add_group_parser(opts, descr) def config_options(self): # config options descr = ("Configuration options", "Configure EasyBuild behavior.") opts = OrderedDict({ 'avail-module-naming-schemes': ( "Show all supported module naming schemes", None, 'store_true', False, ), 'avail-modules-tools': ( "Show all supported module tools", None, "store_true", False, ), 'avail-repositories': ( "Show all repository types (incl. non-usable)", None, "store_true", False, ), 'buildpath': ("Temporary build path", None, 'store', mk_full_default_path('buildpath')), 'ignore-dirs': ("Directory names to ignore when searching for files/dirs", 'strlist', 'store', ['.git', '.svn']), 'installpath': ("Install path for software and modules", None, 'store', mk_full_default_path('installpath')), # purposely take a copy for the default logfile format 'logfile-format': ("Directory name and format of the log file", 'strtuple', 'store', DEFAULT_LOGFILE_FORMAT[:], { 'metavar': 'DIR,FORMAT' }), 'module-naming-scheme': ("Module naming scheme", 'choice', 'store', DEFAULT_MNS, sorted(avail_module_naming_schemes().keys())), 'moduleclasses': (("Extend supported module classes " "(For more info on the default classes, use --show-default-moduleclasses)" ), None, 'extend', [x[0] for x in DEFAULT_MODULECLASSES]), 'modules-footer': ("Path to file containing footer to be added to all generated module files", None, 'store_or_None', None, { 'metavar': "PATH" }), 'modules-tool': ("Modules tool to use", 'choice', 'store', DEFAULT_MODULES_TOOL, sorted(avail_modules_tools().keys())), 'prefix': (("Change prefix for buildpath, installpath, sourcepath and repositorypath " "(used prefix for defaults %s)" % DEFAULT_PREFIX), None, 'store', None), 'recursive-module-unload': ("Enable generating of modules that unload recursively.", None, 'store_true', False), 'repository': ("Repository type, using repositorypath", 'choice', 'store', DEFAULT_REPOSITORY, sorted(avail_repositories().keys())), 'repositorypath': (("Repository path, used by repository " "(is passed as list of arguments to create the repository instance). " "For more info, use --avail-repositories."), 'strlist', 'store', [mk_full_default_path('repositorypath')]), 'show-default-moduleclasses': ("Show default module classes with description", None, 'store_true', False), 'sourcepath': ("Path(s) to where sources should be downloaded (string, colon-separated)", None, 'store', mk_full_default_path('sourcepath')), 'subdir-modules': ("Installpath subdir for modules", None, 'store', DEFAULT_PATH_SUBDIRS['subdir_modules']), 'subdir-software': ("Installpath subdir for software", None, 'store', DEFAULT_PATH_SUBDIRS['subdir_software']), 'suffix-modules-path': ("Suffix for module files install path", None, 'store', GENERAL_CLASS), # this one is sort of an exception, it's something jobscripts can set, # has no real meaning for regular eb usage 'testoutput': ("Path to where a job should place the output (to be set within jobscript)", None, 'store', None), 'tmp-logdir': ("Log directory where temporary log files are stored", None, 'store', None), 'tmpdir': ('Directory to use for temporary storage', None, 'store', None), }) self.log.debug("config_options: descr %s opts %s" % (descr, opts)) self.add_group_parser(opts, descr) def informative_options(self): # informative options descr = ("Informative options", "Obtain information about EasyBuild.") opts = OrderedDict({ 'avail-cfgfile-constants': ("Show all constants that can be used in configuration files", None, 'store_true', False), 'avail-easyconfig-constants': ("Show all constants that can be used in easyconfigs", None, 'store_true', False), 'avail-easyconfig-licenses': ("Show all license constants that can be used in easyconfigs", None, 'store_true', False), 'avail-easyconfig-params': (("Show all easyconfig parameters (include " "easyblock-specific ones by using -e)"), 'choice', 'store_or_None', FORMAT_TXT, [FORMAT_RST, FORMAT_TXT], 'a'), 'avail-easyconfig-templates': (("Show all template names and template constants " "that can be used in easyconfigs"), None, 'store_true', False), 'dep-graph': ("Create dependency graph", None, "store", None, { 'metavar': 'depgraph.<ext>' }), 'list-easyblocks': ("Show list of available easyblocks", 'choice', 'store_or_None', 'simple', ['simple', 'detailed']), 'list-toolchains': ("Show list of known toolchains", None, 'store_true', False), 'search': ("Search for easyconfig files in the robot directory, print full paths", None, 'store', None, { 'metavar': 'STR' }), 'search-short': ("Search for easyconfig files in the robot directory, print short paths", None, 'store', None, 'S', { 'metavar': 'STR' }), }) self.log.debug("informative_options: descr %s opts %s" % (descr, opts)) self.add_group_parser(opts, descr) def regtest_options(self): # regression test options descr = ("Regression test options", "Run and control an EasyBuild regression test.") opts = OrderedDict({ 'aggregate-regtest': ("Collect all the xmls inside the given directory and generate a single file", None, 'store', None, { 'metavar': 'DIR' }), 'dump-test-report': ("Dump test report to specified path", None, 'store_or_None', 'test_report.md'), 'github-user': ("GitHub username", None, 'store', None), 'regtest': ("Enable regression test mode", None, 'store_true', False), 'regtest-output-dir': ("Set output directory for test-run", None, 'store', None, { 'metavar': 'DIR' }), 'sequential': ("Specify this option if you want to prevent parallel build", None, 'store_true', False), 'upload-test-report': ("Upload full test report as a gist on GitHub", None, 'store_true', False), 'test-report-env-filter': ("Regex used to filter out variables in environment dump of test report", None, 'regex', None), }) self.log.debug("regtest_options: descr %s opts %s" % (descr, opts)) self.add_group_parser(opts, descr) def easyconfig_options(self): # easyconfig options (to be passed to easyconfig instance) descr = ("Options for Easyconfigs", "Options to be passed to all Easyconfig.") opts = None self.log.debug("easyconfig_options: descr %s opts %s" % (descr, opts)) self.add_group_parser(opts, descr, prefix='easyconfig') def easyblock_options(self): # easyblock options (to be passed to easyblock instance) descr = ("Options for Easyblocks", "Options to be passed to all Easyblocks.") opts = None self.log.debug("easyblock_options: descr %s opts %s" % (descr, opts)) self.add_group_parser(opts, descr, prefix='easyblock') def unittest_options(self): # unittest options descr = ("Unittest options", "Options dedicated to unittesting (experts only).") opts = OrderedDict({ 'file': ("Log to this file in unittest mode", None, 'store', None), }) self.log.debug("unittest_options: descr %s opts %s" % (descr, opts)) self.add_group_parser(opts, descr, prefix='unittest') def validate(self): """Additional validation of options""" error_cnt = 0 for opt in ['software', 'try-software', 'toolchain', 'try-toolchain']: val = getattr(self.options, opt.replace('-', '_')) if val and len(val) != 2: self.log.warning('--%s requires NAME,VERSION (given %s)' % (opt, ','.join(val))) error_cnt += 1 if self.options.umask: umask_regex = re.compile('^[0-7]{3}$') if not umask_regex.match(self.options.umask): self.log.warning( "--umask value should be 3 digits (0-7) (regex pattern '%s')" % umask_regex.pattern) error_cnt += 1 if error_cnt > 0: self.log.error( "Found %s problems validating the options, treating warnings above as fatal." % error_cnt) def postprocess(self): """Do some postprocessing, in particular print stuff""" build_log.EXPERIMENTAL = self.options.experimental config.SUPPORT_OLDSTYLE = self.options.oldstyleconfig # set strictness of run module if self.options.strict: run.strictness = self.options.strict # override current version of EasyBuild with version specified to --deprecated if self.options.deprecated: build_log.CURRENT_VERSION = LooseVersion(self.options.deprecated) # log to specified value of --unittest-file if self.options.unittest_file: fancylogger.logToFile(self.options.unittest_file) # prepare for --list/--avail if any([ self.options.avail_easyconfig_params, self.options.avail_easyconfig_templates, self.options.list_easyblocks, self.options.list_toolchains, self.options.avail_cfgfile_constants, self.options.avail_easyconfig_constants, self.options.avail_easyconfig_licenses, self.options.avail_repositories, self.options.show_default_moduleclasses, self.options.avail_modules_tools, self.options.avail_module_naming_schemes, ]): build_easyconfig_constants_dict( ) # runs the easyconfig constants sanity check self._postprocess_list_avail() # fail early if required dependencies for functionality requiring using GitHub API are not available: if self.options.from_pr or self.options.upload_test_report: if not HAVE_GITHUB_API: self.log.error( "Required support for using GitHub API is not available (see warnings)." ) # make sure a GitHub token is available when it's required if self.options.upload_test_report: if not HAVE_KEYRING: self.log.error( "Python 'keyring' module required for obtaining GitHub token is not available." ) if self.options.github_user is None: self.log.error( "No GitHub user name provided, required for fetching GitHub token." ) token = fetch_github_token(self.options.github_user) if token is None: self.log.error( "Failed to obtain required GitHub token for user '%s'" % self.options.github_user) self._postprocess_config() def _postprocess_config(self): """Postprocessing of configuration options""" if self.options.prefix is not None: # prefix applies to all paths, and repository has to be reinitialised to take new repositorypath in account # in the legacy-style configuration, repository is initialised in configuration file itself for dest in [ 'installpath', 'buildpath', 'sourcepath', 'repository', 'repositorypath' ]: if not self.options._action_taken.get(dest, False): if dest == 'repository': setattr(self.options, dest, DEFAULT_REPOSITORY) elif dest == 'repositorypath': setattr(self.options, dest, [ mk_full_default_path(dest, prefix=self.options.prefix) ]) else: setattr( self.options, dest, mk_full_default_path(dest, prefix=self.options.prefix)) # LEGACY this line is here for oldstyle config reasons self.options._action_taken[dest] = True if self.options.pretend: self.options.installpath = get_pretend_installpath() if self.options.robot is not None: # paths specified to --robot have preference over --robot-paths # keep both values in sync if robot is enabled, which implies enabling dependency resolver self.options.robot_paths = self.options.robot + self.options.robot_paths self.options.robot = self.options.robot_paths def _postprocess_list_avail(self): """Create all the additional info that can be requested (exit at the end)""" msg = '' # dump supported configuration file constants if self.options.avail_cfgfile_constants: msg += self.avail_cfgfile_constants() # dump possible easyconfig params if self.options.avail_easyconfig_params: msg += avail_easyconfig_params( self.options.easyblock, self.options.avail_easyconfig_params) # dump easyconfig template options if self.options.avail_easyconfig_templates: msg += template_documentation() # dump easyconfig constant options if self.options.avail_easyconfig_constants: msg += constant_documentation() # dump easyconfig license options if self.options.avail_easyconfig_licenses: msg += license_documentation() # dump available easyblocks if self.options.list_easyblocks: msg += self.avail_easyblocks() # dump known toolchains if self.options.list_toolchains: msg += self.avail_toolchains() # dump known repository types if self.options.avail_repositories: msg += self.avail_repositories() # dump supported modules tools if self.options.avail_modules_tools: msg += self.avail_list('modules tools', avail_modules_tools()) # dump supported module naming schemes if self.options.avail_module_naming_schemes: msg += self.avail_list('module naming schemes', avail_module_naming_schemes()) # dump default moduleclasses with description if self.options.show_default_moduleclasses: msg += self.show_default_moduleclasses() if self.options.unittest_file: self.log.info(msg) else: print msg sys.exit(0) def avail_cfgfile_constants(self): """ Return overview of constants supported in configuration files. """ lines = [ "Constants available (only) in configuration files:", "syntax: %(CONSTANT_NAME)s", ] for section in self.go_cfg_constants: lines.append('') if section != self.DEFAULTSECT: section_title = "only in '%s' section:" % section lines.append(section_title) for cst_name, (cst_value, cst_help) in sorted( self.go_cfg_constants[section].items()): lines.append("* %s: %s [value: %s]" % (cst_name, cst_help, cst_value)) return '\n'.join(lines) def avail_classes_tree(self, classes, classNames, detailed, depth=0): """Print list of classes as a tree.""" txt = [] for className in classNames: classInfo = classes[className] if detailed: txt.append("%s|-- %s (%s)" % ("| " * depth, className, classInfo['module'])) else: txt.append("%s|-- %s" % ("| " * depth, className)) if 'children' in classInfo: txt.extend( self.avail_classes_tree(classes, classInfo['children'], detailed, depth + 1)) return txt def avail_easyblocks(self): """Get a class tree for easyblocks.""" detailed = self.options.list_easyblocks == "detailed" module_regexp = re.compile(r"^([^_].*)\.py$") # finish initialisation of the toolchain module (ie set the TC_CONSTANT constants) search_toolchain('') for package in [ "easybuild.easyblocks", "easybuild.easyblocks.generic" ]: __import__(package) # determine paths for this package paths = sys.modules[package].__path__ # import all modules in these paths for path in paths: if os.path.exists(path): for f in os.listdir(path): res = module_regexp.match(f) if res: __import__("%s.%s" % (package, res.group(1))) def add_class(classes, cls): """Add a new class, and all of its subclasses.""" children = cls.__subclasses__() classes.update({ cls.__name__: { 'module': cls.__module__, 'children': [x.__name__ for x in children] } }) for child in children: add_class(classes, child) roots = [EasyBlock, Extension] classes = {} for root in roots: add_class(classes, root) # Print the tree, start with the roots txt = [] for root in roots: root = root.__name__ if detailed: txt.append("%s (%s)" % (root, classes[root]['module'])) else: txt.append("%s" % root) if 'children' in classes[root]: txt.extend( self.avail_classes_tree(classes, classes[root]['children'], detailed)) txt.append("") return "\n".join(txt) def avail_toolchains(self): """Show list of known toolchains.""" _, all_tcs = search_toolchain('') all_tcs_names = [x.NAME for x in all_tcs] tclist = sorted(zip(all_tcs_names, all_tcs)) txt = ["List of known toolchains (toolchainname: module[,module...]):"] for (tcname, tcc) in tclist: tc = tcc( version='1.2.3' ) # version doesn't matter here, but something needs to be there tc_elems = nub( sorted([e for es in tc.definition().values() for e in es])) txt.append("\t%s: %s" % (tcname, ', '.join(tc_elems))) return '\n'.join(txt) def avail_repositories(self): """Show list of known repository types.""" repopath_defaults = mk_full_default_path('repositorypath') all_repos = avail_repositories(check_useable=False) usable_repos = avail_repositories(check_useable=True).keys() indent = ' ' * 2 txt = ['All avaliable repository types'] repos = sorted(all_repos.keys()) for repo in repos: if repo in usable_repos: missing = '' else: missing = ' (*not usable*, something is missing (e.g. a required Python module))' if repo in repopath_defaults: default = ' (default arguments: %s)' % ', '.join( repopath_defaults[repo]) else: default = ' (no default arguments)' txt.append("%s* %s%s%s" % (indent, repo, default, missing)) txt.append("%s%s" % (indent * 3, all_repos[repo].DESCRIPTION)) return "\n".join(txt) def avail_list(self, name, items): """Show list of available values passed by argument.""" return "List of supported %s:\n\t%s" % (name, '\n\t'.join(items)) def show_default_moduleclasses(self): """Show list of default moduleclasses and description.""" txt = ["Default available moduleclasses"] indent = " " * 2 maxlen = max([len(x[0]) for x in DEFAULT_MODULECLASSES]) + 1 # at least 1 space for name, descr in DEFAULT_MODULECLASSES: txt.append("%s%s:%s%s" % (indent, name, (" " * (maxlen - len(name))), descr)) return "\n".join(txt)
def main(testing_data=(None, None, None)): """ Main function: @arg options: a tuple: (options, paths, logger, logfile, hn) as defined in parse_options This function will: - read easyconfig - build software """ # purposely session state very early, to avoid modules loaded by EasyBuild meddling in init_session_state = session_state() # disallow running EasyBuild as root if os.getuid() == 0: sys.stderr.write( "ERROR: You seem to be running EasyBuild with root privileges.\n" "That's not wise, so let's end this here.\n" "Exiting.\n") sys.exit(1) # steer behavior when testing main testing = testing_data[0] is not None args, logfile, do_build = testing_data # initialise options eb_go = eboptions.parse_options(args=args) options = eb_go.options orig_paths = eb_go.args eb_config = eb_go.generate_cmd_line(add_default=True) init_session_state.update({'easybuild_configuration': eb_config}) # set umask (as early as possible) if options.umask is not None: new_umask = int(options.umask, 8) old_umask = os.umask(new_umask) # set temporary directory to use eb_tmpdir = set_tmpdir(options.tmpdir) # initialise logging for main if options.logtostdout: fancylogger.logToScreen(enable=True, stdout=True) else: if logfile is None: # mkstemp returns (fd,filename), fd is from os.open, not regular open! fd, logfile = tempfile.mkstemp(suffix='.log', prefix='easybuild-') os.close(fd) fancylogger.logToFile(logfile) print_msg('temporary log file in case of crash %s' % (logfile), log=None, silent=testing) global _log _log = fancylogger.getLogger(fname=False) if options.umask is not None: _log.info("umask set to '%s' (used to be '%s')" % (oct(new_umask), oct(old_umask))) # hello world! _log.info(this_is_easybuild()) # how was EB called? eb_command_line = eb_go.generate_cmd_line() + eb_go.args _log.info("Command line: %s" % (" ".join(eb_command_line))) _log.info("Using %s as temporary directory" % eb_tmpdir) if not options.robot is None: if options.robot: _log.info("Using robot path(s): %s" % options.robot) else: _log.error( "No robot paths specified, and unable to determine easybuild-easyconfigs install path." ) # do not pass options.robot, it's not a list instance (and it shouldn't be modified) robot_path = None if options.robot: robot_path = list(options.robot) # determine easybuild-easyconfigs package install path easyconfigs_paths = get_paths_for("easyconfigs", robot_path=robot_path) # keep track of paths for install easyconfigs, so we can obtain find specified easyconfigs easyconfigs_pkg_full_paths = easyconfigs_paths[:] if not easyconfigs_paths: _log.warning( "Failed to determine install path for easybuild-easyconfigs package." ) # process software build specifications (if any), i.e. # software name/version, toolchain name/version, extra patches, ... (try_to_generate, build_specs) = process_software_build_specs(options) # specified robot paths are preferred over installed easyconfig files # --try-X and --dep-graph both require --robot, so enable it with path of installed easyconfigs if robot_path or try_to_generate or options.dep_graph: if robot_path is None: robot_path = [] robot_path.extend(easyconfigs_paths) easyconfigs_paths = robot_path[:] _log.info( "Extended list of robot paths with paths for installed easyconfigs: %s" % robot_path) # initialise the easybuild configuration config.init(options, eb_go.get_options_by_section('config')) # building a dependency graph implies force, so that all dependencies are retained # and also skips validation of easyconfigs (e.g. checking os dependencies) retain_all_deps = False if options.dep_graph: _log.info("Enabling force to generate dependency graph.") options.force = True retain_all_deps = True config.init_build_options({ 'aggregate_regtest': options.aggregate_regtest, 'allow_modules_tool_mismatch': options.allow_modules_tool_mismatch, 'check_osdeps': not options.ignore_osdeps, 'cleanup_builddir': options.cleanup_builddir, 'command_line': eb_command_line, 'debug': options.debug, 'dry_run': options.dry_run, 'easyblock': options.easyblock, 'experimental': options.experimental, 'force': options.force, 'github_user': options.github_user, 'group': options.group, 'ignore_dirs': options.ignore_dirs, 'modules_footer': options.modules_footer, 'only_blocks': options.only_blocks, 'recursive_mod_unload': options.recursive_module_unload, 'regtest_output_dir': options.regtest_output_dir, 'retain_all_deps': retain_all_deps, 'robot_path': robot_path, 'sequential': options.sequential, 'silent': testing, 'set_gid_bit': options.set_gid_bit, 'skip': options.skip, 'skip_test_cases': options.skip_test_cases, 'sticky_bit': options.sticky_bit, 'stop': options.stop, 'umask': options.umask, 'valid_module_classes': module_classes(), 'valid_stops': [x[0] for x in EasyBlock.get_steps()], 'validate': not options.force, }) # obtain list of loaded modules, build options must be initialized first modlist = session_module_list() init_session_state.update({'module_list': modlist}) _log.debug("Initial session state: %s" % init_session_state) # search for easyconfigs if options.search or options.search_short: search_path = [os.getcwd()] if easyconfigs_paths: search_path = easyconfigs_paths query = options.search or options.search_short ignore_dirs = config.build_option('ignore_dirs') silent = config.build_option('silent') search_file(search_path, query, short=not options.search, ignore_dirs=ignore_dirs, silent=silent) paths = [] if len(orig_paths) == 0: if options.from_pr: pr_path = os.path.join(eb_tmpdir, "files_pr%s" % options.from_pr) pr_files = fetch_easyconfigs_from_pr( options.from_pr, path=pr_path, github_user=options.github_user) paths = [(path, False) for path in pr_files if path.endswith('.eb')] elif 'name' in build_specs: paths = [ obtain_path(build_specs, easyconfigs_paths, try_to_generate=try_to_generate, exit_on_error=not testing) ] elif not any([ options.aggregate_regtest, options.search, options.search_short, options.regtest ]): print_error(( "Please provide one or multiple easyconfig files, or use software build " "options to make EasyBuild search for easyconfigs"), log=_log, opt_parser=eb_go.parser, exit_on_error=not testing) else: # look for easyconfigs with relative paths in easybuild-easyconfigs package, # unless they were found at the given relative paths if easyconfigs_pkg_full_paths: # determine which easyconfigs files need to be found, if any ecs_to_find = [] for idx, orig_path in enumerate(orig_paths): if orig_path == os.path.basename( orig_path) and not os.path.exists(orig_path): ecs_to_find.append((idx, orig_path)) _log.debug("List of easyconfig files to find: %s" % ecs_to_find) # find missing easyconfigs by walking paths with installed easyconfig files for path in easyconfigs_pkg_full_paths: _log.debug( "Looking for missing easyconfig files (%d left) in %s..." % (len(ecs_to_find), path)) for (subpath, dirnames, filenames) in os.walk(path, topdown=True): for idx, orig_path in ecs_to_find[:]: if orig_path in filenames: full_path = os.path.join(subpath, orig_path) _log.info("Found %s in %s: %s" % (orig_path, path, full_path)) orig_paths[idx] = full_path # if file was found, stop looking for it (first hit wins) ecs_to_find.remove((idx, orig_path)) # stop os.walk insanity as soon as we have all we need (os.walk loop) if len(ecs_to_find) == 0: break # ignore subdirs specified to be ignored by replacing items in dirnames list used by os.walk dirnames[:] = [ d for d in dirnames if not d in options.ignore_dirs ] # stop os.walk insanity as soon as we have all we need (paths loop) if len(ecs_to_find) == 0: break # indicate that specified paths do not contain generated easyconfig files paths = [(path, False) for path in orig_paths] _log.debug("Paths: %s" % paths) # run regtest if options.regtest or options.aggregate_regtest: _log.info("Running regression test") if paths: ec_paths = [path[0] for path in paths] else: # fallback: easybuild-easyconfigs install path ec_paths = easyconfigs_pkg_full_paths regtest_ok = regtest(ec_paths) if not regtest_ok: _log.info("Regression test failed (partially)!") sys.exit(31) # exit -> 3x1t -> 31 # read easyconfig files easyconfigs = [] for (path, generated) in paths: path = os.path.abspath(path) if not os.path.exists(path): print_error("Can't find path %s" % path) try: ec_files = find_easyconfigs(path, ignore_dirs=options.ignore_dirs) for ec_file in ec_files: # only pass build specs when not generating easyconfig files if try_to_generate: ecs = process_easyconfig(ec_file) else: ecs = process_easyconfig(ec_file, build_specs=build_specs) easyconfigs.extend(ecs) except IOError, err: _log.error("Processing easyconfigs in path %s failed: %s" % (path, err))
def main(testing_data=(None, None)): """ Main function: @arg options: a tuple: (options, paths, logger, logfile, hn) as defined in parse_options This function will: - read easyconfig - build software """ # disallow running EasyBuild as root if os.getuid() == 0: sys.stderr.write("ERROR: You seem to be running EasyBuild with root privileges.\n" \ "That's not wise, so let's end this here.\n" \ "Exiting.\n") sys.exit(1) # steer behavior when testing main testing = testing_data[0] is not None args, logfile = testing_data # initialise options eb_go = eboptions.parse_options(args=args) options = eb_go.options orig_paths = eb_go.args # initialise logging for main if options.logtostdout: fancylogger.logToScreen(enable=True, stdout=True) else: if logfile is None: # mkstemp returns (fd,filename), fd is from os.open, not regular open! fd, logfile = tempfile.mkstemp(suffix='.log', prefix='easybuild-') os.close(fd) fancylogger.logToFile(logfile) print_msg('temporary log file in case of crash %s' % (logfile), log=None, silent=testing) global _log _log = fancylogger.getLogger(fname=False) # hello world! _log.info(this_is_easybuild()) # set strictness of filetools module if options.strict: filetools.strictness = options.strict if not options.robot is None: if options.robot: _log.info("Using robot path: %s" % options.robot) else: _log.error("No robot path specified, and unable to determine easybuild-easyconfigs install path.") # determine easybuild-easyconfigs package install path easyconfigs_paths = get_paths_for("easyconfigs", robot_path=options.robot) easyconfigs_pkg_full_path = None search_path = os.getcwd() if easyconfigs_paths: easyconfigs_pkg_full_path = easyconfigs_paths[0] if not options.robot: search_path = easyconfigs_pkg_full_path else: search_path = options.robot else: _log.info("Failed to determine install path for easybuild-easyconfigs package.") if options.robot: easyconfigs_paths = [options.robot] + easyconfigs_paths # initialise the easybuild configuration config.init(options, eb_go.get_options_by_section('config')) # search for easyconfigs if options.search: search_file(search_path, options.search, silent=testing) # process software build specifications (if any), i.e. # software name/version, toolchain name/version, extra patches, ... (try_to_generate, software_build_specs) = process_software_build_specs(options) paths = [] if len(orig_paths) == 0: if software_build_specs.has_key('name'): paths = [obtain_path(software_build_specs, easyconfigs_paths, try_to_generate=try_to_generate, exit_on_error=not testing)] elif not any([options.aggregate_regtest, options.search, options.regtest]): print_error(("Please provide one or multiple easyconfig files, or use software build " "options to make EasyBuild search for easyconfigs"), log=_log, opt_parser=eb_go.parser, exit_on_error=not testing) else: # look for easyconfigs with relative paths in easybuild-easyconfigs package, # unless they we found at the given relative paths if easyconfigs_pkg_full_path: # create a mapping from filename to path in easybuild-easyconfigs package install path easyconfigs_map = {} for (subpath, _, filenames) in os.walk(easyconfigs_pkg_full_path): for filename in filenames: easyconfigs_map.update({filename: os.path.join(subpath, filename)}) # try and find non-existing non-absolute eaysconfig paths in easybuild-easyconfigs package install path for idx, orig_path in enumerate(orig_paths): if not os.path.isabs(orig_path) and not os.path.exists(orig_path): if orig_path in easyconfigs_map: _log.info("Found %s in %s: %s" % (orig_path, easyconfigs_pkg_full_path, easyconfigs_map[orig_path])) orig_paths[idx] = easyconfigs_map[orig_path] # indicate that specified paths do not contain generated easyconfig files paths = [(path, False) for path in orig_paths] _log.debug("Paths: %s" % paths) # run regtest if options.regtest or options.aggregate_regtest: _log.info("Running regression test") if paths: regtest_ok = regtest(options, [path[0] for path in paths]) else: # fallback: easybuild-easyconfigs install path regtest_ok = regtest(options, [easyconfigs_pkg_full_path]) if not regtest_ok: _log.info("Regression test failed (partially)!") sys.exit(31) # exit -> 3x1t -> 31 if any([options.search, options.regtest]): cleanup_logfile_and_exit(logfile, testing, True) # building a dependency graph implies force, so that all dependencies are retained # and also skips validation of easyconfigs (e.g. checking os dependencies) validate_easyconfigs = True retain_all_deps = False if options.dep_graph: _log.info("Enabling force to generate dependency graph.") options.force = True validate_easyconfigs = False retain_all_deps = True # read easyconfig files easyconfigs = [] for (path, generated) in paths: path = os.path.abspath(path) if not (os.path.exists(path)): print_error("Can't find path %s" % path) try: files = find_easyconfigs(path) for f in files: if not generated and try_to_generate and software_build_specs: ec_file = easyconfig.tools.tweak(f, None, software_build_specs) else: ec_file = f easyconfigs.extend(process_easyconfig(ec_file, options.only_blocks, validate=validate_easyconfigs)) except IOError, err: _log.error("Processing easyconfigs in path %s failed: %s" % (path, err))
class EasyBuildOptions(GeneralOption): """Easybuild generaloption class""" VERSION = this_is_easybuild() DEFAULT_LOGLEVEL = 'INFO' DEFAULT_CONFIGFILES = DEFAULT_SYS_CFGFILES[:] if os.path.exists(DEFAULT_USER_CFGFILE): DEFAULT_CONFIGFILES.append(DEFAULT_USER_CFGFILE) ALLOPTSMANDATORY = False # allow more than one argument CONFIGFILES_RAISE_MISSING = True # don't allow non-existing config files to be specified def __init__(self, *args, **kwargs): """Constructor.""" self.default_repositorypath = [mk_full_default_path('repositorypath')] self.default_robot_paths = get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR, robot_path=None) or [] # set up constants to seed into config files parser, by section self.go_cfg_constants = { self.DEFAULTSECT: { 'DEFAULT_REPOSITORYPATH': (self.default_repositorypath[0], "Default easyconfigs repository path"), 'DEFAULT_ROBOT_PATHS': (os.pathsep.join(self.default_robot_paths), "List of default robot paths ('%s'-separated)" % os.pathsep), } } # update or define go_configfiles_initenv in named arguments to pass to parent constructor go_cfg_initenv = kwargs.setdefault('go_configfiles_initenv', {}) for section, constants in self.go_cfg_constants.items(): constants = dict([(name, value) for (name, (value, _)) in constants.items()]) go_cfg_initenv.setdefault(section, {}).update(constants) super(EasyBuildOptions, self).__init__(*args, **kwargs) def basic_options(self): """basic runtime options""" all_stops = [x[0] for x in EasyBlock.get_steps()] strictness_options = [run.IGNORE, run.WARN, run.ERROR] descr = ("Basic options", "Basic runtime options for EasyBuild.") opts = OrderedDict({ 'dry-run': ("Print build overview incl. dependencies (full paths)", None, 'store_true', False), 'dry-run-short': ("Print build overview incl. dependencies (short paths)", None, 'store_true', False, 'D'), 'force': ("Force to rebuild software even if it's already installed (i.e. if it can be found as module)", None, 'store_true', False, 'f'), 'job': ("Submit the build as a job", None, 'store_true', False), 'logtostdout': ("Redirect main log to stdout", None, 'store_true', False, 'l'), 'only-blocks': ("Only build listed blocks", None, 'extend', None, 'b', { 'metavar': 'BLOCKS' }), 'robot': ("Enable dependency resolution, using easyconfigs in specified paths", 'pathlist', 'store_or_None', [], 'r', { 'metavar': 'PATH[%sPATH]' % os.pathsep }), 'robot-paths': ("Additional paths to consider by robot for easyconfigs (--robot paths get priority)", 'pathlist', 'add_flex', self.default_robot_paths, { 'metavar': 'PATH[%sPATH]' % os.pathsep }), 'skip': ("Skip existing software (useful for installing additional packages)", None, 'store_true', False, 'k'), 'stop': ("Stop the installation after certain step", 'choice', 'store_or_None', SOURCE_STEP, 's', all_stops), 'strict': ("Set strictness level", 'choice', 'store', DEFAULT_STRICT, strictness_options), }) self.log.debug("basic_options: descr %s opts %s" % (descr, opts)) self.add_group_parser(opts, descr) def software_options(self): # software build options descr = ("Software search and build options", ( "Specify software search and build options: EasyBuild will search for a " "matching easyconfig and build it. When called with the try prefix " "(i.e. --try-X ), EasyBuild will search for a matching easyconfig " "and if none are found, try to generate one based on a close matching one " "(NOTE: --try-X is best effort, it might produce wrong builds!)")) opts = OrderedDict({ 'amend': (("Specify additional search and build parameters (can be used multiple times); " "for example: versionprefix=foo or patches=one.patch,two.patch)" ), None, 'append', None, { 'metavar': 'VAR=VALUE[,VALUE]' }), 'software': ("Search and build software with given name and version", None, 'extend', None, { 'metavar': 'NAME,VERSION' }), 'software-name': ("Search and build software with given name", None, 'store', None, { 'metavar': 'NAME' }), 'software-version': ("Search and build software with given version", None, 'store', None, { 'metavar': 'VERSION' }), 'toolchain': ("Search and build with given toolchain (name and version)", None, 'extend', None, { 'metavar': 'NAME,VERSION' }), 'toolchain-name': ("Search and build with given toolchain name", None, 'store', None, { 'metavar': 'NAME' }), 'toolchain-version': ("Search and build with given toolchain version", None, 'store', None, { 'metavar': 'VERSION' }), }) longopts = opts.keys() for longopt in longopts: hlp = opts[longopt][0] hlp = "Try to %s (USE WITH CARE!)" % (hlp[0].lower() + hlp[1:]) opts["try-%s" % longopt] = (hlp, ) + opts[longopt][1:] # additional options that don't need a --try equivalent opts.update({ 'from-pr': ("Obtain easyconfigs from specified PR", int, 'store', None, { 'metavar': 'PR#' }), }) self.log.debug("software_options: descr %s opts %s" % (descr, opts)) self.add_group_parser(opts, descr) def override_options(self): # override options descr = ("Override options", "Override default EasyBuild behavior.") opts = OrderedDict({ 'allow-modules-tool-mismatch': ("Allow mismatch of modules tool and definition of 'module' function", None, 'store_true', False), 'cleanup-builddir': ("Cleanup build dir after successful installation.", None, 'store_true', True), 'deprecated': ("Run pretending to be (future) version, to test removal of deprecated code.", None, 'store', None), 'download-timeout': ("Timeout for initiating downloads (in seconds)", float, 'store', None), 'easyblock': ("easyblock to use for processing the spec file or dumping the options", None, 'store', None, 'e', { 'metavar': 'CLASS' }), 'experimental': ("Allow experimental code (with behaviour that can be changed/removed at any given time).", None, 'store_true', False), 'group': ("Group to be used for software installations (only verified, not set)", None, 'store', None), 'hidden': ("Install 'hidden' module file(s) by prefixing their name with '.'", None, 'store_true', False), 'ignore-osdeps': ("Ignore any listed OS dependencies", None, 'store_true', False), 'filter-deps': ("Comma separated list of dependencies that you DON'T want to install with EasyBuild, " "because equivalent OS packages are installed. (e.g. --filter-deps=zlib,ncurses)", 'strlist', 'extend', None), 'hide-deps': ("Comma separated list of dependencies that you want automatically hidden, " "(e.g. --hide-deps=zlib,ncurses)", 'strlist', 'extend', None), 'module-only': ("Only generate module file(s); skip all steps except for %s" % ', '.join(MODULE_ONLY_STEPS), None, 'store_true', False), 'optarch': ("Set architecture optimization, overriding native architecture optimizations", None, 'store', None), 'pretend': (("Does the build/installation in a test directory located in $HOME/easybuildinstall" ), None, 'store_true', False, 'p'), 'set-gid-bit': ("Set group ID bit on newly created directories", None, 'store_true', False), 'sticky-bit': ("Set sticky bit on newly created directories", None, 'store_true', False), 'skip-test-cases': ("Skip running test cases", None, 'store_true', False, 't'), 'umask': ("umask to use (e.g. '022'); non-user write permissions on install directories are removed", None, 'store', None), 'update-modules-tool-cache': ("Update modules tool cache file(s) after generating module file", None, 'store_true', False), }) self.log.debug("override_options: descr %s opts %s" % (descr, opts)) self.add_group_parser(opts, descr) def config_options(self): # config options descr = ("Configuration options", "Configure EasyBuild behavior.") opts = OrderedDict({ 'avail-module-naming-schemes': ( "Show all supported module naming schemes", None, 'store_true', False, ), 'avail-modules-tools': ( "Show all supported module tools", None, "store_true", False, ), 'avail-repositories': ( "Show all repository types (incl. non-usable)", None, "store_true", False, ), 'buildpath': ("Temporary build path", None, 'store', mk_full_default_path('buildpath')), 'external-modules-metadata': ("List of files specifying metadata for external modules (INI format)", 'strlist', 'store', []), 'ignore-dirs': ("Directory names to ignore when searching for files/dirs", 'strlist', 'store', ['.git', '.svn']), 'installpath': ("Install path for software and modules", None, 'store', mk_full_default_path('installpath')), 'installpath-modules': ("Install path for modules (if None, combine --installpath and --subdir-modules)", None, 'store', None), 'installpath-software': ("Install path for software (if None, combine --installpath and --subdir-software)", None, 'store', None), # purposely take a copy for the default logfile format 'logfile-format': ("Directory name and format of the log file", 'strtuple', 'store', DEFAULT_LOGFILE_FORMAT[:], { 'metavar': 'DIR,FORMAT' }), 'module-naming-scheme': ("Module naming scheme", 'choice', 'store', DEFAULT_MNS, sorted(avail_module_naming_schemes().keys())), 'module-syntax': ("Syntax to be used for module files", 'choice', 'store', DEFAULT_MODULE_SYNTAX, sorted(avail_module_generators().keys())), 'moduleclasses': (("Extend supported module classes " "(For more info on the default classes, use --show-default-moduleclasses)" ), None, 'extend', [x[0] for x in DEFAULT_MODULECLASSES]), 'modules-footer': ("Path to file containing footer to be added to all generated module files", None, 'store_or_None', None, { 'metavar': "PATH" }), 'modules-tool': ("Modules tool to use", 'choice', 'store', DEFAULT_MODULES_TOOL, sorted(avail_modules_tools().keys())), 'prefix': (("Change prefix for buildpath, installpath, sourcepath and repositorypath " "(used prefix for defaults %s)" % DEFAULT_PREFIX), None, 'store', None), 'recursive-module-unload': ("Enable generating of modules that unload recursively.", None, 'store_true', False), 'repository': ("Repository type, using repositorypath", 'choice', 'store', DEFAULT_REPOSITORY, sorted(avail_repositories().keys())), 'repositorypath': (("Repository path, used by repository " "(is passed as list of arguments to create the repository instance). " "For more info, use --avail-repositories."), 'strlist', 'store', self.default_repositorypath), 'sourcepath': ("Path(s) to where sources should be downloaded (string, colon-separated)", None, 'store', mk_full_default_path('sourcepath')), 'subdir-modules': ("Installpath subdir for modules", None, 'store', DEFAULT_PATH_SUBDIRS['subdir_modules']), 'subdir-software': ("Installpath subdir for software", None, 'store', DEFAULT_PATH_SUBDIRS['subdir_software']), 'suffix-modules-path': ("Suffix for module files install path", None, 'store', GENERAL_CLASS), # this one is sort of an exception, it's something jobscripts can set, # has no real meaning for regular eb usage 'testoutput': ("Path to where a job should place the output (to be set within jobscript)", None, 'store', None), 'tmp-logdir': ("Log directory where temporary log files are stored", None, 'store', None), 'tmpdir': ('Directory to use for temporary storage', None, 'store', None), }) self.log.debug("config_options: descr %s opts %s" % (descr, opts)) self.add_group_parser(opts, descr) def informative_options(self): # informative options descr = ("Informative options", "Obtain information about EasyBuild.") opts = OrderedDict({ 'avail-cfgfile-constants': ("Show all constants that can be used in configuration files", None, 'store_true', False), 'avail-easyconfig-constants': ("Show all constants that can be used in easyconfigs", None, 'store_true', False), 'avail-easyconfig-licenses': ("Show all license constants that can be used in easyconfigs", None, 'store_true', False), 'avail-easyconfig-params': (("Show all easyconfig parameters (include " "easyblock-specific ones by using -e)"), 'choice', 'store_or_None', FORMAT_TXT, [FORMAT_RST, FORMAT_TXT], 'a'), 'avail-easyconfig-templates': (("Show all template names and template constants " "that can be used in easyconfigs"), None, 'store_true', False), 'dep-graph': ("Create dependency graph", None, "store", None, { 'metavar': 'depgraph.<ext>' }), 'list-easyblocks': ("Show list of available easyblocks", 'choice', 'store_or_None', 'simple', ['simple', 'detailed']), 'list-toolchains': ("Show list of known toolchains", None, 'store_true', False), 'search': ("Search for easyconfig files in the robot directory, print full paths", None, 'store', None, { 'metavar': 'STR' }), 'search-short': ("Search for easyconfig files in the robot directory, print short paths", None, 'store', None, 'S', { 'metavar': 'STR' }), 'show-default-configfiles': ("Show list of default config files", None, 'store_true', False), 'show-default-moduleclasses': ("Show default module classes with description", None, 'store_true', False), }) self.log.debug("informative_options: descr %s opts %s" % (descr, opts)) self.add_group_parser(opts, descr) def regtest_options(self): # regression test options descr = ("Regression test options", "Run and control an EasyBuild regression test.") opts = OrderedDict({ 'aggregate-regtest': ("Collect all the xmls inside the given directory and generate a single file", None, 'store', None, { 'metavar': 'DIR' }), 'dump-test-report': ("Dump test report to specified path", None, 'store_or_None', 'test_report.md'), 'github-user': ("GitHub username", None, 'store', None), 'regtest': ("Enable regression test mode", None, 'store_true', False), 'regtest-output-dir': ("Set output directory for test-run", None, 'store', None, { 'metavar': 'DIR' }), 'sequential': ("Specify this option if you want to prevent parallel build", None, 'store_true', False), 'upload-test-report': ("Upload full test report as a gist on GitHub", None, 'store_true', False), 'test-report-env-filter': ("Regex used to filter out variables in environment dump of test report", None, 'regex', None), }) self.log.debug("regtest_options: descr %s opts %s" % (descr, opts)) self.add_group_parser(opts, descr) def easyconfig_options(self): # easyconfig options (to be passed to easyconfig instance) descr = ("Options for Easyconfigs", "Options to be passed to all Easyconfig.") opts = None self.log.debug("easyconfig_options: descr %s opts %s" % (descr, opts)) self.add_group_parser(opts, descr, prefix='easyconfig') def easyblock_options(self): # easyblock options (to be passed to easyblock instance) descr = ("Options for Easyblocks", "Options to be passed to all Easyblocks.") opts = None self.log.debug("easyblock_options: descr %s opts %s" % (descr, opts)) self.add_group_parser(opts, descr, prefix='easyblock') def unittest_options(self): # unittest options descr = ("Unittest options", "Options dedicated to unittesting (experts only).") opts = OrderedDict({ 'file': ("Log to this file in unittest mode", None, 'store', None), }) self.log.debug("unittest_options: descr %s opts %s" % (descr, opts)) self.add_group_parser(opts, descr, prefix='unittest') def validate(self): """Additional validation of options""" error_msgs = [] for opt in ['software', 'try-software', 'toolchain', 'try-toolchain']: val = getattr(self.options, opt.replace('-', '_')) if val and len(val) != 2: msg = "--%s requires NAME,VERSION (given %s)" % (opt, ','.join(val)) error_msgs.append(msg) if self.options.umask: umask_regex = re.compile('^[0-7]{3}$') if not umask_regex.match(self.options.umask): msg = "--umask value should be 3 digits (0-7) (regex pattern '%s')" % umask_regex.pattern error_msgs.append(msg) # subdir options must be relative for typ in ['modules', 'software']: subdir_opt = 'subdir_%s' % typ val = getattr(self.options, subdir_opt) if os.path.isabs(getattr(self.options, subdir_opt)): msg = "Configuration option '%s' must specify a *relative* path (use 'installpath-%s' instead?): '%s'" msg = msg % (subdir_opt, typ, val) error_msgs.append(msg) if error_msgs: raise EasyBuildError("Found problems validating the options: %s", '\n'.join(error_msgs)) def postprocess(self): """Do some postprocessing, in particular print stuff""" build_log.EXPERIMENTAL = self.options.experimental # set strictness of run module if self.options.strict: run.strictness = self.options.strict # override current version of EasyBuild with version specified to --deprecated if self.options.deprecated: build_log.CURRENT_VERSION = LooseVersion(self.options.deprecated) # log to specified value of --unittest-file if self.options.unittest_file: fancylogger.logToFile(self.options.unittest_file) # prepare for --list/--avail if any([ self.options.avail_easyconfig_params, self.options.avail_easyconfig_templates, self.options.list_easyblocks, self.options.list_toolchains, self.options.avail_cfgfile_constants, self.options.avail_easyconfig_constants, self.options.avail_easyconfig_licenses, self.options.avail_repositories, self.options.show_default_moduleclasses, self.options.avail_modules_tools, self.options.avail_module_naming_schemes, self.options.show_default_configfiles, ]): build_easyconfig_constants_dict( ) # runs the easyconfig constants sanity check self._postprocess_list_avail() # fail early if required dependencies for functionality requiring using GitHub API are not available: if self.options.from_pr or self.options.upload_test_report: if not HAVE_GITHUB_API: raise EasyBuildError( "Required support for using GitHub API is not available (see warnings)." ) if self.options.module_syntax == ModuleGeneratorLua.SYNTAX and self.options.modules_tool != Lmod.__name__: raise EasyBuildError( "Generating Lua module files requires Lmod as modules tool.") # make sure a GitHub token is available when it's required if self.options.upload_test_report: if not HAVE_KEYRING: raise EasyBuildError( "Python 'keyring' module required for obtaining GitHub token is not available." ) if self.options.github_user is None: raise EasyBuildError( "No GitHub user name provided, required for fetching GitHub token." ) token = fetch_github_token(self.options.github_user) if token is None: raise EasyBuildError( "Failed to obtain required GitHub token for user '%s'", self.options.github_user) self._postprocess_external_modules_metadata() self._postprocess_config() def _postprocess_external_modules_metadata(self): """Parse file(s) specifying metadata for external modules.""" # leave external_modules_metadata untouched if no files are provided if not self.options.external_modules_metadata: self.log.debug("No metadata provided for external modules.") return parsed_external_modules_metadata = ConfigObj() for path in self.options.external_modules_metadata: if os.path.exists(path): self.log.debug("Parsing %s with external modules metadata", path) try: parsed_external_modules_metadata.merge(ConfigObj(path)) except ConfigObjError, err: raise EasyBuildError( "Failed to parse %s with external modules metadata: %s", path, err) else: raise EasyBuildError( "Specified path for file with external modules metadata does not exist: %s", path) # make sure name/version values are always lists, make sure they're equal length for mod, entry in parsed_external_modules_metadata.items(): for key in ['name', 'version']: if isinstance(entry.get(key), basestring): entry[key] = [entry[key]] self.log.debug( "Transformed external module metadata value %s for %s into a single-value list: %s", key, mod, entry[key]) # if both names and versions are available, lists must be of same length names, versions = entry.get('name'), entry.get('version') if names is not None and versions is not None and len( names) != len(versions): raise EasyBuildError( "Different length for lists of names/versions in metadata for external module %s: " "names: %s; versions: %s", mod, names, versions) self.options.external_modules_metadata = parsed_external_modules_metadata self.log.debug("External modules metadata: %s", self.options.external_modules_metadata)