def software_options(self): # software build options descr = ("Software search and build options", ("Specify software search and build options: EasyBuild will search for a " "matching easyconfig and build it. When called with the try prefix " "(i.e. --try-X ), EasyBuild will search for a matching easyconfig " "and if none are found, try to generate one based on a close matching one " "(NOTE: --try-X is best effort, it might produce wrong builds!)") ) opts = OrderedDict({ 'software-name':("Search and build software with name", None, 'store', None, {'metavar':'NAME'}), 'software-version':("Search and build software with version", None, 'store', None, {'metavar':'VERSION'}), 'toolchain':("Search and build with toolchain (name and version)", None, 'extend', None, {'metavar':'NAME,VERSION'}), 'toolchain-name':("Search and build with toolchain name", None, 'store', None, {'metavar':'NAME'}), 'toolchain-version':("Search and build with toolchain version", None, 'store', None, {'metavar':'VERSION'}), 'amend':(("Specify additional search and build parameters (can be used multiple times); " "for example: versionprefix=foo or patches=one.patch,two.patch)"), None, 'append', None, {'metavar':'VAR=VALUE[,VALUE]'}), }) longopts = opts.keys() for longopt in longopts: hlp = opts[longopt][0] hlp = "Try to %s (USE WITH CARE!)" % (hlp[0].lower() + hlp[1:]) opts["try-%s" % longopt] = (hlp,) + opts[longopt][1:] self.log.debug("software_options: descr %s opts %s" % (descr, opts)) self.add_group_parser(opts, descr)
def software_options(self): # software build options descr = ("Software search and build options", ("Specify software search and build options: EasyBuild will search for a " "matching easyconfig and build it. When called with the try prefix " "(i.e. --try-X ), EasyBuild will search for a matching easyconfig " "and if none are found, try to generate one based on a close matching one " "(NOTE: --try-X is best effort, it might produce wrong builds!)") ) opts = OrderedDict({ 'amend':(("Specify additional search and build parameters (can be used multiple times); " "for example: versionprefix=foo or patches=one.patch,two.patch)"), None, 'append', None, {'metavar': 'VAR=VALUE[,VALUE]'}), 'software-name': ("Search and build software with name", None, 'store', None, {'metavar': 'NAME'}), 'software-version': ("Search and build software with version", None, 'store', None, {'metavar': 'VERSION'}), 'toolchain': ("Search and build with toolchain (name and version)", None, 'extend', None, {'metavar': 'NAME,VERSION'}), 'toolchain-name': ("Search and build with toolchain name", None, 'store', None, {'metavar': 'NAME'}), 'toolchain-version': ("Search and build with toolchain version", None, 'store', None, {'metavar': 'VERSION'}), }) longopts = opts.keys() for longopt in longopts: hlp = opts[longopt][0] hlp = "Try to %s (USE WITH CARE!)" % (hlp[0].lower() + hlp[1:]) opts["try-%s" % longopt] = (hlp,) + opts[longopt][1:] self.log.debug("software_options: descr %s opts %s" % (descr, opts)) self.add_group_parser(opts, descr)
def sanity_check_step(self): """Custom sanity check for EasyBuild.""" # list of dirs to check, by package # boolean indicates whether dir is expected to reside in Python lib/pythonX/site-packages dir subdirs_by_pkg = [ ('framework', [('easybuild/framework', True), ('easybuild/tools', True)]), ('easyblocks', [('easybuild/easyblocks', True)]), ('easyconfigs', [('easybuild/easyconfigs', False)]), ] # final list of directories to check, by setup tool # order matters, e.g. setuptools before distutils eb_dirs = OrderedDict() eb_dirs['setuptools'] = [] eb_dirs['distutils.core'] = flatten([x[1] for x in subdirs_by_pkg]) # determine setup tool (setuptools or distutils) setup_tool = None for tool in eb_dirs.keys(): self.log.debug("Trying %s.." % tool) try: exec "from %s import setup" % tool del setup setup_tool = tool break except ImportError: pass self.log.debug('setup_tool: %s' % setup_tool) # for a setuptools installation, we need to figure out the egg dirs since we don't know the individual package versions if setup_tool == 'setuptools': try: installed_dirs = os.listdir( os.path.join(self.installdir, self.pylibdir)) for (pkg, subdirs) in subdirs_by_pkg: sel_dirs = [ x for x in installed_dirs if x.startswith('easybuild_%s' % pkg) ] if not len(sel_dirs) == 1: self.log.error( "Failed to isolate installed egg dir for easybuild-%s" % pkg) for (subdir, _) in subdirs: # eggs always go in Python lib/pythonX/site-packages dir with setuptools eb_dirs['setuptools'].append( (os.path.join(sel_dirs[0], subdir), True)) except OSError, err: self.log.error( "Failed to determine sanity check dir paths: %s" % err)
def informative_options(self): # informative options descr = ("Informative options", "Obtain information about EasyBuild.") opts = OrderedDict({ 'avail-easyconfig-constants': ("Show all constants that can be used in easyconfigs", None, 'store_true', False), 'avail-easyconfig-licenses': ("Show all license constants that can be used in easyconfigs", None, 'store_true', False), 'avail-easyconfig-params': (("Show all easyconfig parameters (include " "easyblock-specific ones by using -e)"), None, "store_true", False, 'a'), 'avail-easyconfig-templates': (("Show all template names and template constants " "that can be used in easyconfigs"), None, 'store_true', False), 'dep-graph': ("Create dependency graph", None, "store", None, {'metavar': 'depgraph.<ext>'}), 'list-easyblocks': ("Show list of available easyblocks", 'choice', 'store_or_None', 'simple', ['simple', 'detailed']), 'list-toolchains': ("Show list of known toolchains", None, 'store_true', False), 'search': ("Search for easyconfig files in the robot directory, print full paths", None, 'store', None, {'metavar': 'STR'}), 'search-short': ("Search for easyconfig files in the robot directory, print short paths", None, 'store', None, 'S', {'metavar': 'STR'}), }) self.log.debug("informative_options: descr %s opts %s" % (descr, opts)) self.add_group_parser(opts, descr)
def basic_options(self): """basic runtime options""" all_stops = [x[0] for x in EasyBlock.get_steps()] strictness_options = [run.IGNORE, run.WARN, run.ERROR] try: default_robot_path = get_paths_for("easyconfigs", robot_path=None)[0] except: self.log.warning("basic_options: unable to determine default easyconfig path") default_robot_path = False # False as opposed to None, since None is used for indicating that --robot was not used descr = ("Basic options", "Basic runtime options for EasyBuild.") opts = OrderedDict({ 'dry-run': ("Print build overview incl. dependencies (full paths)", None, 'store_true', False), 'dry-run-short': ("Print build overview incl. dependencies (short paths)", None, 'store_true', False, 'D'), 'force': ("Force to rebuild software even if it's already installed (i.e. if it can be found as module)", None, 'store_true', False, 'f'), 'job': ("Submit the build as a job", None, 'store_true', False), 'logtostdout': ("Redirect main log to stdout", None, 'store_true', False, 'l'), 'only-blocks': ("Only build listed blocks", None, 'extend', None, 'b', {'metavar': 'BLOCKS'}), 'robot': ("Path(s) to search for easyconfigs for missing dependencies (colon-separated)" , None, 'store_or_None', default_robot_path, 'r', {'metavar': 'PATH'}), 'skip': ("Skip existing software (useful for installing additional packages)", None, 'store_true', False, 'k'), 'stop': ("Stop the installation after certain step", 'choice', 'store_or_None', 'source', 's', all_stops), 'strict': ("Set strictness level", 'choice', 'store', run.WARN, strictness_options), }) self.log.debug("basic_options: descr %s opts %s" % (descr, opts)) self.add_group_parser(opts, descr)
def basic_options(self): """basic runtime options""" all_stops = [x[0] for x in EasyBlock.get_steps()] strictness_options = [filetools.IGNORE, filetools.WARN, filetools.ERROR] try: default_robot_path = get_paths_for("easyconfigs", robot_path=None)[0] except: self.log.warning("basic_options: unable to determine default easyconfig path") default_robot_path = False # False as opposed to None, since None is used for indicating that --robot was not used descr = ("Basic options", "Basic runtime options for EasyBuild.") opts = OrderedDict({ "only-blocks":("Only build listed blocks", None, "extend", None, "b", {'metavar':"BLOCKS"}), "force":(("Force to rebuild software even if it's already installed " "(i.e. if it can be found as module)"), None, "store_true", False, "f"), "job":("Submit the build as a job", None, "store_true", False), "skip":("Skip existing software (useful for installing additional packages)", None, "store_true", False, "k"), "robot":("Path to search for easyconfigs for missing dependencies." , None, "store_or_None", default_robot_path, "r", {'metavar':"PATH"}), "stop":("Stop the installation after certain step", "choice", "store_or_None", "source", "s", all_stops), "strict":("Set strictness level", "choice", "store", filetools.WARN, strictness_options), "logtostdout":("Redirect main log to stdout", None, "store_true", False, "l"), }) self.log.debug("basic_options: descr %s opts %s" % (descr, opts)) self.add_group_parser(opts, descr)
def override_options(self): # override options descr = ("Override options", "Override default EasyBuild behavior.") opts = OrderedDict({ 'allow-modules-tool-mismatch': ("Allow mismatch of modules tool and definition of 'module' function", None, 'store_true', False), 'cleanup-builddir': ("Cleanup build dir after successful installation.", None, 'store_true', True), 'deprecated': ("Run pretending to be (future) version, to test removal of deprecated code.", None, 'store', None), 'easyblock': ("easyblock to use for processing the spec file or dumping the options", None, 'store', None, 'e', {'metavar': 'CLASS'}), 'experimental': ("Allow experimental code (with behaviour that can be changed or removed at any given time).", None, 'store_true', False), 'group': ("Group to be used for software installations (only verified, not set)", None, 'store', None), 'ignore-osdeps': ("Ignore any listed OS dependencies", None, 'store_true', False), 'oldstyleconfig': ("Look for and use the oldstyle configuration file.", None, 'store_true', True), 'pretend': (("Does the build/installation in a test directory located in $HOME/easybuildinstall"), None, 'store_true', False, 'p'), 'set-gid-bit': ("Set group ID bit on newly created directories", None, 'store_true', False), 'sticky-bit': ("Set sticky bit on newly created directories", None, 'store_true', False), 'skip-test-cases': ("Skip running test cases", None, 'store_true', False, 't'), 'umask': ("umask to use (e.g. '022'); non-user write permissions on install directories are removed", None, 'store', None), }) self.log.debug("override_options: descr %s opts %s" % (descr, opts)) self.add_group_parser(opts, descr)
def regtest_options(self): # regression test options descr = ("Regression test options", "Run and control an EasyBuild regression test.") opts = OrderedDict({ 'aggregate-regtest': ("Collect all the xmls inside the given directory and generate a single file", None, 'store', None, { 'metavar': 'DIR' }), 'dump-test-report': ("Dump test report to specified path", None, 'store_or_None', 'test_report.md'), 'github-user': ("GitHub username", None, 'store', None), 'regtest': ("Enable regression test mode", None, 'store_true', False), 'regtest-output-dir': ("Set output directory for test-run", None, 'store', None, { 'metavar': 'DIR' }), 'sequential': ("Specify this option if you want to prevent parallel build", None, 'store_true', False), 'upload-test-report': ("Upload full test report as a gist on GitHub", None, 'store_true', False), 'test-report-env-filter': ("Regex used to filter out variables in environment dump of test report", None, 'regex', None), }) self.log.debug("regtest_options: descr %s opts %s" % (descr, opts)) self.add_group_parser(opts, descr)
def sanity_check_step(self): """Custom sanity check for EasyBuild.""" # list of dirs to check, by package # boolean indicates whether dir is expected to reside in Python lib/pythonX/site-packages dir subdirs_by_pkg = [ ('framework', [('easybuild/framework', True), ('easybuild/tools', True)]), ('easyblocks', [('easybuild/easyblocks', True)]), ('easyconfigs', [('easybuild/easyconfigs', False)]), ] # final list of directories to check, by setup tool # order matters, e.g. setuptools before distutils eb_dirs = OrderedDict() eb_dirs['setuptools'] = [] eb_dirs['distutils.core'] = flatten([x[1] for x in subdirs_by_pkg]) # determine setup tool (setuptools or distutils) setup_tool = None for tool in eb_dirs.keys(): self.log.debug("Trying %s.." % tool) try: exec "from %s import setup" % tool del setup setup_tool = tool break except ImportError: pass self.log.debug('setup_tool: %s' % setup_tool) # for a setuptools installation, we need to figure out the egg dirs since we don't know the individual package versions if setup_tool == 'setuptools': try: installed_dirs = os.listdir(os.path.join(self.installdir, self.pylibdir)) for (pkg, subdirs) in subdirs_by_pkg: sel_dirs = [x for x in installed_dirs if x.startswith('easybuild_%s' % pkg)] if not len(sel_dirs) == 1: self.log.error("Failed to isolate installed egg dir for easybuild-%s" % pkg) for (subdir, _) in subdirs: # eggs always go in Python lib/pythonX/site-packages dir with setuptools eb_dirs['setuptools'].append((os.path.join(sel_dirs[0], subdir), True)) except OSError, err: self.log.error("Failed to determine sanity check dir paths: %s" % err)
def unittest_options(self): # unittest options descr = ("Unittest options", "Options dedicated to unittesting (experts only).") opts = OrderedDict({ 'file': ("Log to this file in unittest mode", None, 'store', None), }) self.log.debug("unittest_options: descr %s opts %s" % (descr, opts)) self.add_group_parser(opts, descr, prefix='unittest')
def get_build_stats(app, start_time, command_line): """ Return build statistics for this build """ time_now = time.time() build_time = round(time_now - start_time, 2) buildstats = OrderedDict([ ('easybuild-framework_version', str(FRAMEWORK_VERSION)), ('easybuild-easyblocks_version', str(EASYBLOCKS_VERSION)), ('timestamp', int(time_now)), ('build_time', build_time), ('install_size', app.det_installsize()), ('command_line', command_line), ('modules_tool', app.modules_tool.buildstats()), ]) for key, val in sorted(get_system_info().items()): buildstats.update({key: val}) return buildstats
def print_pr_summary(pr_data): """Print PR summary.""" keys = OrderedDict([ ('html_url', 'URL'), ('title', None), (('user', 'login'), "issued by"), (('head', 'ref'), "branch"), (('base', 'ref'), "target branch"), ('state', "status"), (('milestone', 'title'), None), ]) target = '%s/%s' % (pr_data['base']['repo']['owner']['login'], pr_data['base']['repo']['name']) print "\nSummary for %s PR #%s:\n" % (target, pr_data['number']) for key in keys: print_pr_info(pr_data, key, label=keys[key]) print ''
def basic_options(self): """basic runtime options""" all_stops = [x[0] for x in EasyBlock.get_steps()] strictness_options = [run.IGNORE, run.WARN, run.ERROR] descr = ("Basic options", "Basic runtime options for EasyBuild.") opts = OrderedDict({ 'dry-run': ("Print build overview incl. dependencies (full paths)", None, 'store_true', False), 'dry-run-short': ("Print build overview incl. dependencies (short paths)", None, 'store_true', False, 'D'), 'force': ("Force to rebuild software even if it's already installed (i.e. if it can be found as module)", None, 'store_true', False, 'f'), 'job': ("Submit the build as a job", None, 'store_true', False), 'logtostdout': ("Redirect main log to stdout", None, 'store_true', False, 'l'), 'only-blocks': ("Only build listed blocks", None, 'extend', None, 'b', { 'metavar': 'BLOCKS' }), 'robot': ("Enable dependency resolution, using easyconfigs in specified paths", 'pathlist', 'store_or_None', [], 'r', { 'metavar': 'PATH[%sPATH]' % os.pathsep }), 'robot-paths': ("Additional paths to consider by robot for easyconfigs (--robot paths get priority)", 'pathlist', 'store', self.default_robot_paths, { 'metavar': 'PATH[%sPATH]' % os.pathsep }), 'skip': ("Skip existing software (useful for installing additional packages)", None, 'store_true', False, 'k'), 'stop': ("Stop the installation after certain step", 'choice', 'store_or_None', 'source', 's', all_stops), 'strict': ("Set strictness level", 'choice', 'store', run.WARN, strictness_options), }) self.log.debug("basic_options: descr %s opts %s" % (descr, opts)) self.add_group_parser(opts, descr)
def avail_easyconfig_params(easyblock, output_format): """ Compose overview of available easyconfig parameters, in specified format. """ params = copy.deepcopy(DEFAULT_CONFIG) # include list of extra parameters (if any) extra_params = {} app = get_easyblock_class(easyblock, default_fallback=False) if app is not None: extra_params = app.extra_options() params.update(extra_params) # compose title title = "Available easyconfig parameters" if extra_params: title += " (* indicates specific to the %s easyblock)" % app.__name__ # group parameters by category grouped_params = OrderedDict() for category in sorted_categories(): # exclude hidden parameters if category[1].upper() in [HIDDEN]: continue grpname = category[1] grouped_params[grpname] = {} for name, (dflt, descr, cat) in params.items(): if cat == category: if name in extra_params: # mark easyblock-specific parameters name = '%s*' % name grouped_params[grpname].update({name: (descr, dflt)}) if not grouped_params[grpname]: del grouped_params[grpname] # compose output, according to specified format (txt, rst, ...) avail_easyconfig_params_functions = { FORMAT_RST: avail_easyconfig_params_rst, FORMAT_TXT: avail_easyconfig_params_txt, } return avail_easyconfig_params_functions[output_format](title, grouped_params)
def get_build_stats(app, starttime): """ Return build statistics for this build """ buildtime = round(time.time() - starttime, 2) buildstats = OrderedDict([ ('easybuild-framework_version', str(FRAMEWORK_VERSION)), ('easybuild-easyblocks_version', str(EASYBLOCKS_VERSION)), ('host', os.uname()[1]), ('platform' , platform.platform()), ('cpu_model', systemtools.get_cpu_model()), ('core_count', systemtools.get_core_count()), ('timestamp', int(time.time())), ('build_time', buildtime), ('install_size', app.det_installsize()), ]) return buildstats
def convert_to_help(opts, has_default=False): """ Converts the given list to a mapping of category -> [(name, help)] (OrderedDict) @param: has_default, if False, add the DEFAULT_CONFIG list """ mapping = OrderedDict() if isinstance(opts, dict): opts = opts.items() if not has_default: defs = [(k, [def_val, descr, ALL_CATEGORIES[cat]]) for k, (def_val, descr, cat) in DEFAULT_CONFIG.items()] opts = defs + opts # sort opts opts.sort() for cat in sorted_categories(): mapping[cat[1]] = [(opt[0], "%s (default: %s)" % (opt[1][1], opt[1][0])) for opt in opts if opt[1][2] == cat] return mapping
def software_options(self): # software build options descr = ( "Software search and build options", ( "Specify software search and build options: EasyBuild will search for a " "matching easyconfig and build it. When called with the try prefix " "(i.e. --try-X ), EasyBuild will search for a matching easyconfig " "and if none are found, try to generate one based on a close matching one " "(NOTE: --try-X is best effort, it might produce wrong builds!)" ), ) opts = OrderedDict( { "amend": ( ( "Specify additional search and build parameters (can be used multiple times); " "for example: versionprefix=foo or patches=one.patch,two.patch)" ), None, "append", None, {"metavar": "VAR=VALUE[,VALUE]"}, ), "software": ( "Search and build software with given name and version", None, "extend", None, {"metavar": "NAME,VERSION"}, ), "software-name": ( "Search and build software with given name", None, "store", None, {"metavar": "NAME"}, ), "software-version": ( "Search and build software with given version", None, "store", None, {"metavar": "VERSION"}, ), "toolchain": ( "Search and build with given toolchain (name and version)", None, "extend", None, {"metavar": "NAME,VERSION"}, ), "toolchain-name": ( "Search and build with given toolchain name", None, "store", None, {"metavar": "NAME"}, ), "toolchain-version": ( "Search and build with given toolchain version", None, "store", None, {"metavar": "VERSION"}, ), } ) longopts = opts.keys() for longopt in longopts: hlp = opts[longopt][0] hlp = "Try to %s (USE WITH CARE!)" % (hlp[0].lower() + hlp[1:]) opts["try-%s" % longopt] = (hlp,) + opts[longopt][1:] # additional options that don't need a --try equivalent opts.update({"from-pr": ("Obtain easyconfigs from specified PR", int, "store", None, {"metavar": "PR#"})}) self.log.debug("software_options: descr %s opts %s" % (descr, opts)) self.add_group_parser(opts, descr)
def sanity_check_step(self): """Custom sanity check for EasyBuild.""" # check whether easy-install.pth contains correct entries easy_install_pth = os.path.join(self.installdir, self.pylibdir, 'easy-install.pth') if os.path.exists(easy_install_pth): easy_install_pth_txt = read_file(easy_install_pth) for pkg in [p for p in self.easybuild_pkgs if p not in ['setuptools', 'vsc-install']]: if pkg == 'vsc-base': # don't include strict version check for vsc-base pkg_regex = re.compile(r"^\./%s" % pkg.replace('-', '_'), re.M) else: major_minor_version = '.'.join(self.version.split('.')[:2]) pkg_regex = re.compile(r"^\./%s-%s" % (pkg.replace('-', '_'), major_minor_version), re.M) if not pkg_regex.search(easy_install_pth_txt): raise EasyBuildError("Failed to find pattern '%s' in %s: %s", pkg_regex.pattern, easy_install_pth, easy_install_pth_txt) # list of dirs to check, by package # boolean indicates whether dir is expected to reside in Python lib/pythonX/site-packages dir subdirs_by_pkg = { 'easybuild-framework': [('easybuild/framework', True), ('easybuild/tools', True)], 'easybuild-easyblocks': [('easybuild/easyblocks', True)], 'easybuild-easyconfigs': [('easybuild/easyconfigs', False)], } if LooseVersion(self.version) >= LooseVersion('2.0'): subdirs_by_pkg.update({ 'vsc-base': [('vsc/utils', True)], }) # final list of directories to check, by setup tool # order matters, e.g. setuptools before distutils eb_dirs = OrderedDict() eb_dirs['setuptools'] = [] eb_dirs['distutils.core'] = flatten([x for x in subdirs_by_pkg.values()]) # determine setup tool (setuptools or distutils) setup_tool = None for tool in eb_dirs.keys(): self.log.debug("Trying %s.." % tool) try: exec "from %s import setup" % tool del setup setup_tool = tool break except ImportError: pass self.log.debug('setup_tool: %s' % setup_tool) # for a setuptools installation, we need to figure out the egg dirs, # since we don't know the individual package versions if setup_tool == 'setuptools': try: installed_dirs = os.listdir(os.path.join(self.installdir, self.pylibdir)) for (pkg, subdirs) in subdirs_by_pkg.items(): sel_dirs = [x for x in installed_dirs if x.startswith(pkg.replace('-', '_'))] if not len(sel_dirs) == 1: raise EasyBuildError("Failed to isolate installed egg dir for %s", pkg) for (subdir, _) in subdirs: # eggs always go in Python lib/pythonX/site-packages dir with setuptools eb_dirs['setuptools'].append((os.path.join(sel_dirs[0], subdir), True)) except OSError as err: raise EasyBuildError("Failed to determine sanity check dir paths: %s", err) # set of sanity check paths to check for EasyBuild custom_paths = { 'files': ['bin/eb'], 'dirs': [self.pylibdir] + [[x, os.path.join(self.pylibdir, x)][y] for (x, y) in eb_dirs[setup_tool]], } # make sure we don't trip over deprecated behavior in old EasyBuild versions eb_cmd = 'eb' if LooseVersion(self.version) <= LooseVersion('1.16.0'): eb_cmd = 'EASYBUILD_DEPRECATED=1.0 eb' # set of sanity check commands to run for EasyBuild custom_commands = [ # this may spit out a wrong version, but that should be safe to ignore # occurs when the EasyBuild being used is newer than the EasyBuild being installed (eb_cmd, '--version'), (eb_cmd, '-a'), (eb_cmd, '-e ConfigureMake -a'), ] # (temporary) cleanse copy of initial environment to avoid conflict with (potentially) loaded EasyBuild module self.real_initial_environ = copy.deepcopy(self.initial_environ) for env_var in ['_LMFILES_', 'LOADEDMODULES']: if env_var in self.initial_environ: self.initial_environ.pop(env_var) os.environ.pop(env_var) self.log.debug("Unset $%s in current env and copy of original env to make sanity check work" % env_var) super(EB_EasyBuildMeta, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands)
'', 'Please type the options to tune the gnu compilers, or ignore :': os.getenv('F90FLAGS'), } f90_seq = os.getenv('F90_SEQ') if not f90_seq: # F90_SEQ is only defined when usempi is enabled f90_seq = os.getenv('F90') stdqa = OrderedDict([ (r'Confirm library .* is .*', 'y'), # this one needs to be tried first! (r'.*fortran 90 compiler name .*\s*:\n\(suggestions\s*: .*\)', f90_seq), (r'.*fortran 90 compiler interfaced with .*\s*:\n\(suggestions\s*: .*\)', f90_seq), (r'Please type the ABSOLUTE name of .*library.*, or ignore\s*[:]*\s*[\n]*.*', ''), (r'Please .* to save this draft configuration file :\n.*', '%s.x' % self.conf_file), ]) no_qa = [ ".*ignored.", ] env.setvar('GMKTMP', self.builddir) env.setvar('GMKFILE', self.conf_file) run_cmd_qa("gmkfilemaker", qa, std_qa=stdqa, no_qa=no_qa)
def sanity_check_step(self): """Custom sanity check for EasyBuild.""" # check whether easy-install.pth contains correct entries easy_install_pth = os.path.join(self.installdir, self.pylibdir, 'easy-install.pth') if os.path.exists(easy_install_pth): easy_install_pth_txt = read_file(easy_install_pth) for pkg in self.easybuild_pkgs: if pkg == 'vsc-base': # don't include strict version check for vsc-base pkg_regex = re.compile(r"^\./%s" % pkg.replace('-', '_'), re.M) else: major_minor_version = '.'.join(self.version.split('.')[:2]) pkg_regex = re.compile(r"^\./%s-%s" % (pkg.replace('-', '_'), major_minor_version), re.M) if not pkg_regex.search(easy_install_pth_txt): raise EasyBuildError("Failed to find pattern '%s' in %s: %s", pkg_regex.pattern, easy_install_pth, easy_install_pth_txt) # list of dirs to check, by package # boolean indicates whether dir is expected to reside in Python lib/pythonX/site-packages dir subdirs_by_pkg = { 'easybuild-framework': [('easybuild/framework', True), ('easybuild/tools', True)], 'easybuild-easyblocks': [('easybuild/easyblocks', True)], 'easybuild-easyconfigs': [('easybuild/easyconfigs', False)], } if LooseVersion(self.version) >= LooseVersion('2.0'): subdirs_by_pkg.update({ 'vsc-base': [('vsc/utils', True)], }) # final list of directories to check, by setup tool # order matters, e.g. setuptools before distutils eb_dirs = OrderedDict() eb_dirs['setuptools'] = [] eb_dirs['distutils.core'] = flatten([x for x in subdirs_by_pkg.values()]) # determine setup tool (setuptools or distutils) setup_tool = None for tool in eb_dirs.keys(): self.log.debug("Trying %s.." % tool) try: exec "from %s import setup" % tool del setup setup_tool = tool break except ImportError: pass self.log.debug('setup_tool: %s' % setup_tool) # for a setuptools installation, we need to figure out the egg dirs, # since we don't know the individual package versions if setup_tool == 'setuptools': try: installed_dirs = os.listdir(os.path.join(self.installdir, self.pylibdir)) for (pkg, subdirs) in subdirs_by_pkg.items(): sel_dirs = [x for x in installed_dirs if x.startswith(pkg.replace('-', '_'))] if not len(sel_dirs) == 1: raise EasyBuildError("Failed to isolate installed egg dir for %s", pkg) for (subdir, _) in subdirs: # eggs always go in Python lib/pythonX/site-packages dir with setuptools eb_dirs['setuptools'].append((os.path.join(sel_dirs[0], subdir), True)) except OSError, err: raise EasyBuildError("Failed to determine sanity check dir paths: %s", err)
def override_options(self): # override options descr = ("Override options", "Override default EasyBuild behavior.") opts = OrderedDict({ 'allow-modules-tool-mismatch': ("Allow mismatch of modules tool and definition of 'module' function", None, 'store_true', False), 'cleanup-builddir': ("Cleanup build dir after successful installation.", None, 'store_true', True), 'deprecated': ("Run pretending to be (future) version, to test removal of deprecated code.", None, 'store', None), 'download-timeout': ("Timeout for initiating downloads (in seconds)", None, 'store', None), 'easyblock': ("easyblock to use for processing the spec file or dumping the options", None, 'store', None, 'e', { 'metavar': 'CLASS' }), 'experimental': ("Allow experimental code (with behaviour that can be changed/removed at any given time).", None, 'store_true', False), 'group': ("Group to be used for software installations (only verified, not set)", None, 'store', None), 'hidden': ("Install 'hidden' module file(s) by prefixing their name with '.'", None, 'store_true', False), 'ignore-osdeps': ("Ignore any listed OS dependencies", None, 'store_true', False), 'filter-deps': ("Comma separated list of dependencies that you DON'T want to install with EasyBuild, " "because equivalent OS packages are installed. (e.g. --filter-deps=zlib,ncurses)", str, 'extend', None), 'oldstyleconfig': ("Look for and use the oldstyle configuration file.", None, 'store_true', True), 'optarch': ("Set architecture optimization, overriding native architecture optimizations", None, 'store', None), 'pretend': (("Does the build/installation in a test directory located in $HOME/easybuildinstall" ), None, 'store_true', False, 'p'), 'set-gid-bit': ("Set group ID bit on newly created directories", None, 'store_true', False), 'sticky-bit': ("Set sticky bit on newly created directories", None, 'store_true', False), 'skip-test-cases': ("Skip running test cases", None, 'store_true', False, 't'), 'umask': ("umask to use (e.g. '022'); non-user write permissions on install directories are removed", None, 'store', None), 'update-modules-tool-cache': ("Update modules tool cache file(s) after generating module file", None, 'store_true', False), }) self.log.debug("override_options: descr %s opts %s" % (descr, opts)) self.add_group_parser(opts, descr)
def config_options(self): # config options descr = ("Configuration options", "Configure EasyBuild behavior.") opts = OrderedDict({ 'avail-module-naming-schemes': ( "Show all supported module naming schemes", None, 'store_true', False, ), 'avail-modules-tools': ( "Show all supported module tools", None, "store_true", False, ), 'avail-repositories': ( "Show all repository types (incl. non-usable)", None, "store_true", False, ), 'buildpath': ("Temporary build path", None, 'store', mk_full_default_path('buildpath')), 'ignore-dirs': ("Directory names to ignore when searching for files/dirs", 'strlist', 'store', ['.git', '.svn']), 'installpath': ("Install path for software and modules", None, 'store', mk_full_default_path('installpath')), # purposely take a copy for the default logfile format 'logfile-format': ("Directory name and format of the log file", 'strtuple', 'store', DEFAULT_LOGFILE_FORMAT[:], { 'metavar': 'DIR,FORMAT' }), 'module-naming-scheme': ("Module naming scheme", 'choice', 'store', DEFAULT_MNS, sorted(avail_module_naming_schemes().keys())), 'moduleclasses': (("Extend supported module classes " "(For more info on the default classes, use --show-default-moduleclasses)" ), None, 'extend', [x[0] for x in DEFAULT_MODULECLASSES]), 'modules-footer': ("Path to file containing footer to be added to all generated module files", None, 'store_or_None', None, { 'metavar': "PATH" }), 'modules-tool': ("Modules tool to use", 'choice', 'store', DEFAULT_MODULES_TOOL, sorted(avail_modules_tools().keys())), 'prefix': (("Change prefix for buildpath, installpath, sourcepath and repositorypath " "(used prefix for defaults %s)" % DEFAULT_PREFIX), None, 'store', None), 'recursive-module-unload': ("Enable generating of modules that unload recursively.", None, 'store_true', False), 'repository': ("Repository type, using repositorypath", 'choice', 'store', DEFAULT_REPOSITORY, sorted(avail_repositories().keys())), 'repositorypath': (("Repository path, used by repository " "(is passed as list of arguments to create the repository instance). " "For more info, use --avail-repositories."), 'strlist', 'store', [mk_full_default_path('repositorypath')]), 'show-default-moduleclasses': ("Show default module classes with description", None, 'store_true', False), 'sourcepath': ("Path(s) to where sources should be downloaded (string, colon-separated)", None, 'store', mk_full_default_path('sourcepath')), 'subdir-modules': ("Installpath subdir for modules", None, 'store', DEFAULT_PATH_SUBDIRS['subdir_modules']), 'subdir-software': ("Installpath subdir for software", None, 'store', DEFAULT_PATH_SUBDIRS['subdir_software']), 'suffix-modules-path': ("Suffix for module files install path", None, 'store', GENERAL_CLASS), # this one is sort of an exception, it's something jobscripts can set, # has no real meaning for regular eb usage 'testoutput': ("Path to where a job should place the output (to be set within jobscript)", None, 'store', None), 'tmp-logdir': ("Log directory where temporary log files are stored", None, 'store', None), 'tmpdir': ('Directory to use for temporary storage', None, 'store', None), }) self.log.debug("config_options: descr %s opts %s" % (descr, opts)) self.add_group_parser(opts, descr)
def configure_step(self): """Custom configuration procedure for ALADIN.""" # unset $LIBRARY_PATH set by modules of dependencies, because it may screw up linking if 'LIBRARY_PATH' in os.environ: self.log.debug("Unsetting $LIBRARY_PATH (was: %s)" % os.environ['LIBRARY_PATH']) self.orig_library_path = os.environ.pop('LIBRARY_PATH') # build auxiliary libraries auxlibs_dir = None my_gnu = None if self.toolchain.comp_family() == toolchain.GCC: my_gnu = 'y' # gfortran for var in ['CFLAGS', 'CXXFLAGS', 'F90FLAGS', 'FFLAGS']: flags = os.getenv(var) env.setvar(var, "%s -fdefault-real-8 -fdefault-double-8" % flags) self.log.info("Updated %s to '%s'" % (var, os.getenv(var))) elif self.toolchain.comp_family() == toolchain.INTELCOMP: my_gnu = 'i' # icc/ifort else: raise EasyBuildError( "Don't know how to set 'my_gnu' variable in auxlibs build script." ) self.log.info("my_gnu set to '%s'" % my_gnu) tmp_installroot = tempfile.mkdtemp(prefix='aladin_auxlibs_') try: cwd = os.getcwd() os.chdir(self.builddir) builddirs = os.listdir(self.builddir) auxlibs_dir = [ x for x in builddirs if x.startswith('auxlibs_installer') ][0] os.chdir(auxlibs_dir) auto_driver = 'driver_automatic' for line in fileinput.input(auto_driver, inplace=1, backup='.orig.eb'): line = re.sub(r"^(my_gnu\s*=\s*).*$", r"\1%s" % my_gnu, line) line = re.sub(r"^(my_r32\s*=\s*).*$", r"\1n", line) # always 64-bit real precision line = re.sub(r"^(my_readonly\s*=\s*).*$", r"\1y", line) # make libs read-only after build line = re.sub(r"^(my_installroot\s*=\s*).*$", r"\1%s" % tmp_installroot, line) sys.stdout.write(line) run_cmd("./%s" % auto_driver) os.chdir(cwd) except OSError as err: raise EasyBuildError("Failed to build ALADIN: %s", err) # build gmkpack, update PATH and set GMKROOT # we build gmkpack here because a config file is generated in the gmkpack isntall path try: gmkpack_dir = [x for x in builddirs if x.startswith('gmkpack')][0] os.chdir(os.path.join(self.builddir, gmkpack_dir)) qa = { 'Do you want to run the configuration file maker assistant now (y) or later [n] ?': 'n', } run_cmd_qa("./build_gmkpack", qa) os.chdir(cwd) paths = os.getenv('PATH').split(':') paths.append(os.path.join(self.builddir, gmkpack_dir, 'util')) env.setvar('PATH', ':'.join(paths)) env.setvar('GMKROOT', os.path.join(self.builddir, gmkpack_dir)) except OSError as err: raise EasyBuildError("Failed to build gmkpack: %s", err) # generate gmkpack configuration file self.conf_file = 'ALADIN_%s' % self.version self.conf_filepath = os.path.join(self.builddir, 'gmkpack_support', 'arch', '%s.x' % self.conf_file) try: if os.path.exists(self.conf_filepath): os.remove(self.conf_filepath) self.log.info("Removed existing gmpack config file %s" % self.conf_filepath) archdir = os.path.dirname(self.conf_filepath) if not os.path.exists(archdir): mkdir(archdir, parents=True) except OSError as err: raise EasyBuildError("Failed to remove existing file %s: %s", self.conf_filepath, err) mpich = 'n' known_mpi_libs = [ toolchain.MPICH, toolchain.MPICH2, toolchain.INTELMPI ] if self.toolchain.options.get( 'usempi', None) and self.toolchain.mpi_family() in known_mpi_libs: mpich = 'y' qpref = 'Please type the ABSOLUTE name of ' qsuff = ', or ignore (environment variables allowed) :' qsuff2 = ', or ignore : (environment variables allowed) :' comp_fam = self.toolchain.comp_family() if comp_fam == toolchain.GCC: gribdir = 'GNU' elif comp_fam == toolchain.INTELCOMP: gribdir = 'INTEL' else: raise EasyBuildError( "Don't know which grib lib dir to use for compiler %s", comp_fam) aux_lib_gribex = os.path.join(tmp_installroot, gribdir, 'lib', 'libgribex.a') aux_lib_ibm = os.path.join(tmp_installroot, gribdir, 'lib', 'libibmdummy.a') grib_api_lib = os.path.join(get_software_root('grib_api'), 'lib', 'libgrib_api.a') grib_api_f90_lib = os.path.join(get_software_root('grib_api'), 'lib', 'libgrib_api_f90.a') grib_api_inc = os.path.join(get_software_root('grib_api'), 'include') jasperlib = os.path.join(get_software_root('JasPer'), 'lib', 'libjasper.a') mpilib = os.path.join(os.getenv('MPI_LIB_DIR'), os.getenv('MPI_LIB_SHARED')) # netCDF netcdf = get_software_root('netCDF') netcdf_fortran = get_software_root('netCDF-Fortran') if netcdf: netcdfinc = os.path.join(netcdf, 'include') if netcdf_fortran: netcdflib = os.path.join(netcdf_fortran, get_software_libdir('netCDF-Fortran'), 'libnetcdff.a') else: netcdflib = os.path.join(netcdf, get_software_libdir('netCDF'), 'libnetcdff.a') if not os.path.exists(netcdflib): raise EasyBuildError("%s does not exist", netcdflib) else: raise EasyBuildError("netCDF(-Fortran) not available") ldpaths = [ldflag[2:] for ldflag in os.getenv('LDFLAGS').split(' ') ] # LDFLAGS have form '-L/path/to' lapacklibs = [] for lib in os.getenv('LAPACK_STATIC_LIBS').split(','): libpaths = [os.path.join(ldpath, lib) for ldpath in ldpaths] lapacklibs.append([ libpath for libpath in libpaths if os.path.exists(libpath) ][0]) lapacklib = ' '.join(lapacklibs) blaslibs = [] for lib in os.getenv('BLAS_STATIC_LIBS').split(','): libpaths = [os.path.join(ldpath, lib) for ldpath in ldpaths] blaslibs.append([ libpath for libpath in libpaths if os.path.exists(libpath) ][0]) blaslib = ' '.join(blaslibs) qa = { 'Do you want to run the configuration file maker assistant now (y) or later [n] ?': 'y', 'Do you want to setup your configuration file for MPICH (y/n) [n] ?': mpich, 'Please type the directory name where to find a dummy file mpif.h or ignore :': os.getenv('MPI_INC_DIR'), '%sthe library gribex or emos%s' % (qpref, qsuff2): aux_lib_gribex, '%sthe library ibm%s' % (qpref, qsuff): aux_lib_ibm, '%sthe library grib_api%s' % (qpref, qsuff): grib_api_lib, '%sthe library grib_api_f90%s' % (qpref, qsuff): grib_api_f90_lib, '%sthe JPEG auxilary library if enabled by Grib_api%s' % (qpref, qsuff2): jasperlib, '%sthe library netcdf%s' % (qpref, qsuff): netcdflib, '%sthe library lapack%s' % (qpref, qsuff): lapacklib, '%sthe library blas%s' % (qpref, qsuff): blaslib, '%sthe library mpi%s' % (qpref, qsuff): mpilib, '%sa MPI dummy library for serial executions, or ignore :' % qpref: '', 'Please type the directory name where to find grib_api headers, or ignore :': grib_api_inc, 'Please type the directory name where to find fortint.h or ignore :': '', 'Please type the directory name where to find netcdf headers, or ignore :': netcdfinc, 'Do you want to define CANARI (y/n) [y] ?': 'y', 'Please type the name of the script file used to generate a preprocessed blacklist file, or ignore :': '', 'Please type the name of the script file used to recover local libraries (gget), or ignore :': '', 'Please type the options to tune the gnu compilers, or ignore :': os.getenv('F90FLAGS'), } f90_seq = os.getenv('F90_SEQ') if not f90_seq: # F90_SEQ is only defined when usempi is enabled f90_seq = os.getenv('F90') stdqa = OrderedDict([ (r'Confirm library .* is .*', 'y'), # this one needs to be tried first! (r'.*fortran 90 compiler name .*\s*:\n\(suggestions\s*: .*\)', f90_seq), (r'.*fortran 90 compiler interfaced with .*\s*:\n\(suggestions\s*: .*\)', f90_seq), (r'Please type the ABSOLUTE name of .*library.*, or ignore\s*[:]*\s*[\n]*.*', ''), (r'Please .* to save this draft configuration file :\n.*', '%s.x' % self.conf_file), ]) no_qa = [ ".*ignored.", ] env.setvar('GMKTMP', self.builddir) env.setvar('GMKFILE', self.conf_file) run_cmd_qa("gmkfilemaker", qa, std_qa=stdqa, no_qa=no_qa) # set environment variables for installation dirs env.setvar('ROOTPACK', os.path.join(self.installdir, 'rootpack')) env.setvar('ROOTBIN', os.path.join(self.installdir, 'rootpack')) env.setvar('HOMEPACK', os.path.join(self.installdir, 'pack')) env.setvar('HOMEBIN', os.path.join(self.installdir, 'pack')) # patch config file to include right Fortran compiler flags regex_subs = [(r"^(FRTFLAGS\s*=.*)$", r"\1 %s" % os.getenv('FFLAGS'))] apply_regex_substitutions(self.conf_filepath, regex_subs)
def config_options(self): # config options descr = ("Configuration options", "Configure EasyBuild behavior.") oldstyle_defaults = get_default_oldstyle_configfile_defaults() opts = OrderedDict({ 'avail-module-naming-schemes': ("Show all supported module naming schemes", None, 'store_true', False,), 'avail-modules-tools': ("Show all supported module tools", None, "store_true", False,), 'avail-repositories': ("Show all repository types (incl. non-usable)", None, "store_true", False,), 'buildpath': ("Temporary build path", None, 'store', oldstyle_defaults['buildpath']), 'ignore-dirs': ("Directory names to ignore when searching for files/dirs", 'strlist', 'store', ['.git', '.svn']), 'installpath': ("Install path for software and modules", None, 'store', oldstyle_defaults['installpath']), 'config': ("Path to EasyBuild config file", None, 'store', oldstyle_defaults['config'], 'C'), 'logfile-format': ("Directory name and format of the log file", 'strtuple', 'store', oldstyle_defaults['logfile_format'], {'metavar': 'DIR,FORMAT'}), 'module-naming-scheme': ("Module naming scheme", 'choice', 'store', oldstyle_defaults['module_naming_scheme'], sorted(avail_module_naming_schemes().keys())), 'moduleclasses': (("Extend supported module classes " "(For more info on the default classes, use --show-default-moduleclasses)"), None, 'extend', oldstyle_defaults['moduleclasses']), 'modules-footer': ("Path to file containing footer to be added to all generated module files", None, 'store_or_None', None, {'metavar': "PATH"}), 'modules-tool': ("Modules tool to use", 'choice', 'store', oldstyle_defaults['modules_tool'], sorted(avail_modules_tools().keys())), 'prefix': (("Change prefix for buildpath, installpath, sourcepath and repositorypath " "(repositorypath prefix is only relevant in case of FileRepository repository) " "(used prefix for defaults %s)" % oldstyle_defaults['prefix']), None, 'store', None), 'recursive-module-unload': ("Enable generating of modules that unload recursively.", None, 'store_true', False), 'repository': ("Repository type, using repositorypath", 'choice', 'store', oldstyle_defaults['repository'], sorted(avail_repositories().keys())), 'repositorypath': (("Repository path, used by repository " "(is passed as list of arguments to create the repository instance). " "For more info, use --avail-repositories."), 'strlist', 'store', oldstyle_defaults['repositorypath'][oldstyle_defaults['repository']]), 'show-default-moduleclasses': ("Show default module classes with description", None, 'store_true', False), 'sourcepath': ("Path(s) to where sources should be downloaded (string, colon-separated)", None, 'store', oldstyle_defaults['sourcepath']), 'subdir-modules': ("Installpath subdir for modules", None, 'store', oldstyle_defaults['subdir_modules']), 'subdir-software': ("Installpath subdir for software", None, 'store', oldstyle_defaults['subdir_software']), # this one is sort of an exception, it's something jobscripts can set, # has no real meaning for regular eb usage 'testoutput': ("Path to where a job should place the output (to be set within jobscript)", None, 'store', None), 'tmp-logdir': ("Log directory where temporary log files are stored", None, 'store', oldstyle_defaults['tmp_logdir']), 'tmpdir': ('Directory to use for temporary storage', None, 'store', None), }) self.log.debug("config_options: descr %s opts %s" % (descr, opts)) self.add_group_parser(opts, descr)