def stage2(tmpdir, versions, install_path): """STAGE 2: install EasyBuild to temporary dir with EasyBuild from stage 1.""" info( "\n\n+++ STAGE 2: installing EasyBuild in %s with EasyBuild from stage 1...\n\n" % install_path) # make sure we still have distribute in PYTHONPATH, so we have control over which 'setup' is used pythonpaths = [ x for x in os.environ.get('PYTHONPATH', '').split(os.pathsep) if len(x) > 0 ] os.environ['PYTHONPATH'] = os.pathsep.join( [find_egg_dir_for(tmpdir, 'distribute')] + pythonpaths) # create easyconfig file ebfile = os.path.join(tmpdir, 'EasyBuild-%s.eb' % versions['version']) f = open(ebfile, "w") f.write(EB_EC_FILE % versions) f.close() # unset $MODULEPATH, we don't care about already installed modules os.environ['MODULEPATH'] = '' # set command line arguments for eb eb_args = ['eb', ebfile, '--allow-modules-tool-mismatch'] if print_debug: eb_args.extend(['--debug', '--logtostdout']) # make sure we don't leave any stuff behind in default path $HOME/.local/easybuild # and set build and install path explicitely if LooseVersion(versions['version']) < LooseVersion("1.3.0"): os.environ['EASYBUILD_PREFIX'] = tmpdir os.environ['EASYBUILD_BUILDPATH'] = tmpdir if install_path is not None: os.environ['EASYBUILD_INSTALLPATH'] = install_path else: # only for v1.3 and up eb_args.append('--prefix=%s' % tmpdir) eb_args.append('--buildpath=%s' % tmpdir) if install_path is not None: eb_args.append('--installpath=%s' % install_path) # make sure parent modules path already exists (Lmod trips over a non-existing entry in $MODULEPATH) if install_path is not None: modules_path = os.path.join(install_path, 'modules', 'all') if not os.path.exists(modules_path): os.makedirs(modules_path) debug("Created path %s" % modules_path) debug("Running EasyBuild with arguments '%s'" % ' '.join(eb_args)) sys.argv = eb_args # install EasyBuild with EasyBuild from easybuild.main import main as easybuild_main easybuild_main()
def stage2(tmpdir, versions, install_path): """STAGE 2: install EasyBuild to temporary dir with EasyBuild from stage 1.""" info("\n\n+++ STAGE 2: installing EasyBuild in %s with EasyBuild from stage 1...\n\n" % install_path) # make sure we still have distribute in PYTHONPATH, so we have control over which 'setup' is used pythonpaths = [x for x in os.environ.get('PYTHONPATH', '').split(os.pathsep) if len(x) > 0] os.environ['PYTHONPATH'] = os.pathsep.join([find_egg_dir_for(tmpdir, 'distribute')] + pythonpaths) # create easyconfig file ebfile = os.path.join(tmpdir, 'EasyBuild-%s.eb' % versions['version']) f = open(ebfile, "w") f.write(EB_EC_FILE % versions) f.close() # unset $MODULEPATH, we don't care about already installed modules os.environ['MODULEPATH'] = '' # set command line arguments for eb eb_args = ['eb', ebfile, '--allow-modules-tool-mismatch'] if print_debug: eb_args.extend(['--debug', '--logtostdout']) # make sure we don't leave any stuff behind in default path $HOME/.local/easybuild # and set build and install path explicitely if LooseVersion(versions['version']) < LooseVersion("1.3.0"): os.environ['EASYBUILD_PREFIX'] = tmpdir os.environ['EASYBUILD_BUILDPATH'] = tmpdir if install_path is not None: os.environ['EASYBUILD_INSTALLPATH'] = install_path else: # only for v1.3 and up eb_args.append('--prefix=%s' % tmpdir) eb_args.append('--buildpath=%s' % tmpdir) if install_path is not None: eb_args.append('--installpath=%s' % install_path) # make sure parent modules path already exists (Lmod trips over a non-existing entry in $MODULEPATH) if install_path is not None: modules_path = os.path.join(install_path, 'modules', 'all') if not os.path.exists(modules_path): os.makedirs(modules_path) debug("Created path %s" % modules_path) debug("Running EasyBuild with arguments '%s'" % ' '.join(eb_args)) sys.argv = eb_args # install EasyBuild with EasyBuild from easybuild.main import main as easybuild_main easybuild_main()
def stage2(tmpdir, templates, install_path, distribute_egg_dir, sourcepath): """STAGE 2: install EasyBuild to temporary dir with EasyBuild from stage 1.""" info( "\n\n+++ STAGE 2: installing EasyBuild in %s with EasyBuild from stage 1...\n\n" % install_path) if distribute_egg_dir is None: preinstallopts = '' else: # inject path to distribute installed in stage 1 into $PYTHONPATH via preinstallopts # other approaches are not reliable, since EasyBuildMeta easyblock unsets $PYTHONPATH; preinstallopts = "export PYTHONPATH=%s:$PYTHONPATH && " % distribute_egg_dir # also add location to easy_install provided through stage0 to $PATH curr_path = os.environ.get('PATH', '').split(os.pathsep) os.environ['PATH'] = os.pathsep.join([os.path.join(tmpdir, 'bin')] + curr_path) debug("$PATH: %s" % os.environ['PATH']) # ensure that (latest) setuptools is installed as well alongside EasyBuild, # since it is a required runtime dependency for recent vsc-base and EasyBuild versions # this is necessary since we provide our own distribute installation during the bootstrap (cfr. stage0) preinstallopts += "easy_install -U --prefix %(installdir)s setuptools && " # vsc-install is a runtime dependency for the EasyBuild unit test suite, # and is easily picked up from stage1 rather than being actually installed, so force it preinstallopts += "easy_install -U --prefix %(installdir)s vsc-install && " templates.update({ 'preinstallopts': preinstallopts, }) # create easyconfig file ebfile = os.path.join(tmpdir, 'EasyBuild-%s.eb' % templates['version']) handle = open(ebfile, 'w') templates.update({ 'source_urls': '\n'.join([ "'%s/%s/%s'," % (PYPI_SOURCE_URL, pkg[0], pkg) for pkg in EASYBUILD_PACKAGES ]), 'sources': "%(vsc-base)s%(easybuild-framework)s%(easybuild-easyblocks)s%(easybuild-easyconfigs)s" % templates, 'pythonpath': distribute_egg_dir, }) handle.write(EASYBUILD_EASYCONFIG_TEMPLATE % templates) handle.close() # unset $MODULEPATH, we don't care about already installed modules os.environ['MODULEPATH'] = '' # set command line arguments for eb eb_args = ['eb', ebfile, '--allow-modules-tool-mismatch'] if print_debug: eb_args.extend(['--debug', '--logtostdout']) # make sure we don't leave any stuff behind in default path $HOME/.local/easybuild # and set build and install path explicitely if LooseVersion(templates['version']) < LooseVersion('1.3.0'): os.environ['EASYBUILD_PREFIX'] = tmpdir os.environ['EASYBUILD_BUILDPATH'] = tmpdir if install_path is not None: os.environ['EASYBUILD_INSTALLPATH'] = install_path else: # only for v1.3 and up eb_args.append('--prefix=%s' % tmpdir) eb_args.append('--buildpath=%s' % tmpdir) if install_path is not None: eb_args.append('--installpath=%s' % install_path) if sourcepath is not None: eb_args.append('--sourcepath=%s' % sourcepath) # make sure parent modules path already exists (Lmod trips over a non-existing entry in $MODULEPATH) if install_path is not None: modules_path = os.path.join(install_path, 'modules', 'all') if not os.path.exists(modules_path): os.makedirs(modules_path) debug("Created path %s" % modules_path) debug("Running EasyBuild with arguments '%s'" % ' '.join(eb_args)) sys.argv = eb_args # install EasyBuild with EasyBuild from easybuild.main import main as easybuild_main easybuild_main()
def stage2(tmpdir, templates, install_path, distribute_egg_dir, sourcepath): """STAGE 2: install EasyBuild to temporary dir with EasyBuild from stage 1.""" print('\n') info("+++ STAGE 2: installing EasyBuild in %s with EasyBuild from stage 1...\n" % install_path) preinstallopts = '' if distribute_egg_dir is not None: # inject path to distribute installed in stage 1 into $PYTHONPATH via preinstallopts # other approaches are not reliable, since EasyBuildMeta easyblock unsets $PYTHONPATH; # this is required for the easy_install from stage 1 to work preinstallopts += "export PYTHONPATH=%s:$PYTHONPATH && " % distribute_egg_dir # ensure that (latest) setuptools is installed as well alongside EasyBuild, # since it is a required runtime dependency for recent vsc-base and EasyBuild versions # this is necessary since we provide our own distribute installation during the bootstrap (cfr. stage0) preinstallopts += "%s $(which easy_install) -U --prefix %%(installdir)s setuptools && " % sys.executable # vsc-install is a runtime dependency for the EasyBuild unit test suite, # and is easily picked up from stage1 rather than being actually installed, so force it vsc_install = 'vsc-install' if sourcepath: vsc_install_tarball_paths = glob.glob(os.path.join(sourcepath, 'vsc-install*.tar.gz')) if len(vsc_install_tarball_paths) == 1: vsc_install = vsc_install_tarball_paths[0] preinstallopts += "%s $(which easy_install) -U --prefix %%(installdir)s %s && " % (sys.executable, vsc_install) templates.update({ 'preinstallopts': preinstallopts, }) # create easyconfig file ebfile = os.path.join(tmpdir, 'EasyBuild-%s.eb' % templates['version']) handle = open(ebfile, 'w') templates.update({ 'source_urls': '\n'.join(["'%s/%s/%s'," % (PYPI_SOURCE_URL, pkg[0], pkg) for pkg in EASYBUILD_PACKAGES]), 'sources': "%(vsc-install)s%(vsc-base)s%(easybuild-framework)s%(easybuild-easyblocks)s%(easybuild-easyconfigs)s" % templates, 'pythonpath': distribute_egg_dir, }) handle.write(EASYBUILD_EASYCONFIG_TEMPLATE % templates) handle.close() # set command line arguments for eb eb_args = ['eb', ebfile, '--allow-modules-tool-mismatch'] if print_debug: eb_args.extend(['--debug', '--logtostdout']) if forced_install: info("Performing FORCED installation, as requested...") eb_args.append('--force') # make sure we don't leave any stuff behind in default path $HOME/.local/easybuild # and set build and install path explicitely if LooseVersion(templates['version']) < LooseVersion('1.3.0'): os.environ['EASYBUILD_PREFIX'] = tmpdir os.environ['EASYBUILD_BUILDPATH'] = tmpdir if install_path is not None: os.environ['EASYBUILD_INSTALLPATH'] = install_path else: # only for v1.3 and up eb_args.append('--prefix=%s' % tmpdir) eb_args.append('--buildpath=%s' % tmpdir) if install_path is not None: eb_args.append('--installpath=%s' % install_path) if sourcepath is not None: eb_args.append('--sourcepath=%s' % sourcepath) # make sure parent modules path already exists (Lmod trips over a non-existing entry in $MODULEPATH) if install_path is not None: modules_path = os.path.join(install_path, 'modules', 'all') if not os.path.exists(modules_path): os.makedirs(modules_path) debug("Created path %s" % modules_path) debug("Running EasyBuild with arguments '%s'" % ' '.join(eb_args)) sys.argv = eb_args # install EasyBuild with EasyBuild from easybuild.main import main as easybuild_main easybuild_main()
def stage2(tmpdir, templates, install_path, distribute_egg_dir, sourcepath): """STAGE 2: install EasyBuild to temporary dir with EasyBuild from stage 1.""" print('\n') info("+++ STAGE 2: installing EasyBuild in %s with EasyBuild from stage 1...\n" % install_path) preinstallopts = '' if distribute_egg_dir is not None: # inject path to distribute installed in stage 0 into $PYTHONPATH via preinstallopts # other approaches are not reliable, since EasyBuildMeta easyblock unsets $PYTHONPATH; # this is required for the easy_install from stage 0 to work preinstallopts += "export PYTHONPATH=%s:$PYTHONPATH && " % distribute_egg_dir # ensure that (latest) setuptools is installed as well alongside EasyBuild, # since it is a required runtime dependency for recent vsc-base and EasyBuild versions # this is necessary since we provide our own distribute installation during the bootstrap (cfr. stage0) preinstallopts += "%s -m easy_install -U --prefix %%(installdir)s setuptools && " % sys.executable # vsc-install is a runtime dependency for the EasyBuild unit test suite, # and is easily picked up from stage1 rather than being actually installed, so force it vsc_install = "'%s<0.11.4'" % VSC_INSTALL if sourcepath: vsc_install_tarball_paths = glob.glob(os.path.join(sourcepath, 'vsc-install*.tar.gz')) if len(vsc_install_tarball_paths) == 1: vsc_install = vsc_install_tarball_paths[0] preinstallopts += "%s -m easy_install -U --prefix %%(installdir)s %s && " % (sys.executable, vsc_install) templates.update({ 'preinstallopts': preinstallopts, }) # determine PyPI URLs for individual packages pkg_urls = [] for pkg in EASYBUILD_PACKAGES: # format of pkg entries in templates: "'<pkg_filename>'," pkg_filename = templates[pkg][1:-2] # the lines below implement a simplified version of the 'pypi_source_urls' and 'derive_alt_pypi_url' functions, # which we can't leverage here, partially because of transitional changes in PyPI (#md5= -> #sha256=) # determine download URL via PyPI's 'simple' API pkg_simple = None try: pkg_simple = urllib2.urlopen('https://pypi.python.org/simple/%s' % pkg, timeout=10).read() except (urllib2.URLError, urllib2.HTTPError) as err: # failing to figure out the package download URl may be OK when source tarballs are provided if sourcepath: info("Ignoring failed attempt to determine '%s' download URL since source tarballs are provided" % pkg) else: raise err if pkg_simple: pkg_url_part_regex = re.compile('/(packages/[^#]+)/%s#' % pkg_filename) res = pkg_url_part_regex.search(pkg_simple) if res: pkg_url_part = res.group(1) else: error_msg = "Failed to determine PyPI package URL for %s using pattern '%s': %s\n" error(error_msg % (pkg, pkg_url_part_regex.pattern, pkg_simple)) pkg_url = 'https://pypi.python.org/' + pkg_url_part pkg_urls.append(pkg_url) templates.update({ 'source_urls': '\n'.join(["'%s'," % pkg_url for pkg_url in pkg_urls]), 'sources': "%(vsc-install)s%(vsc-base)s%(easybuild-framework)s%(easybuild-easyblocks)s%(easybuild-easyconfigs)s" % templates, 'pythonpath': distribute_egg_dir, }) # create easyconfig file ebfile = os.path.join(tmpdir, 'EasyBuild-%s.eb' % templates['version']) handle = open(ebfile, 'w') ebfile_txt = EASYBUILD_EASYCONFIG_TEMPLATE % templates handle.write(ebfile_txt) handle.close() debug("Contents of generated easyconfig file:\n%s" % ebfile_txt) # set command line arguments for eb eb_args = ['eb', ebfile, '--allow-modules-tool-mismatch'] if print_debug: eb_args.extend(['--debug', '--logtostdout']) if forced_install: info("Performing FORCED installation, as requested...") eb_args.append('--force') # make sure we don't leave any stuff behind in default path $HOME/.local/easybuild # and set build and install path explicitely if LooseVersion(templates['version']) < LooseVersion('1.3.0'): os.environ['EASYBUILD_PREFIX'] = tmpdir os.environ['EASYBUILD_BUILDPATH'] = tmpdir if install_path is not None: os.environ['EASYBUILD_INSTALLPATH'] = install_path else: # only for v1.3 and up eb_args.append('--prefix=%s' % tmpdir) eb_args.append('--buildpath=%s' % tmpdir) if install_path is not None: eb_args.append('--installpath=%s' % install_path) if sourcepath is not None: eb_args.append('--sourcepath=%s' % sourcepath) # make sure EasyBuild can find EasyBuild-*.eb easyconfig file when it needs to; # (for example when HierarchicalMNS is used as module naming scheme, # see https://github.com/easybuilders/easybuild-framework/issues/2393) eb_args.append('--robot-paths=%s:' % tmpdir) # make sure parent modules path already exists (Lmod trips over a non-existing entry in $MODULEPATH) if install_path is not None: modules_path = det_modules_path(install_path) if not os.path.exists(modules_path): os.makedirs(modules_path) debug("Created path %s" % modules_path) debug("Running EasyBuild with arguments '%s'" % ' '.join(eb_args)) sys.argv = eb_args # location to 'eb' command (from stage 1) may be expected to be included in $PATH # it usually is there after stage1, unless 'prep' is called again with another location # (only when stage 0 is not skipped) # cfr. https://github.com/easybuilders/easybuild-framework/issues/2279 curr_path = [x for x in os.environ.get('PATH', '').split(os.pathsep) if len(x) > 0] os.environ['PATH'] = os.pathsep.join([os.path.join(tmpdir, STAGE1_SUBDIR, 'bin')] + curr_path) debug("$PATH: %s" % os.environ['PATH']) # install EasyBuild with EasyBuild from easybuild.main import main as easybuild_main easybuild_main() if print_debug: os.environ['EASYBUILD_DEBUG'] = '1' # make sure the EasyBuild module was actually installed # EasyBuild configuration options that are picked up from configuration files/environment may break the bootstrap, # for example by having $EASYBUILD_VERSION defined or via a configuration file specifies a value for 'stop'... from easybuild.tools.config import build_option, install_path, get_module_syntax from easybuild.framework.easyconfig.easyconfig import ActiveMNS eb_spec = { 'name': 'EasyBuild', 'hidden': False, 'toolchain': {'name': 'dummy', 'version': 'dummy'}, 'version': templates['version'], 'versionprefix': '', 'versionsuffix': '', 'moduleclass': 'tools', } mod_path = os.path.join(install_path('mod'), build_option('suffix_modules_path')) debug("EasyBuild module should have been installed to %s" % mod_path) eb_mod_name = ActiveMNS().det_full_module_name(eb_spec) debug("EasyBuild module name: %s" % eb_mod_name) eb_mod_path = os.path.join(mod_path, eb_mod_name) if get_module_syntax() == 'Lua': eb_mod_path += '.lua' if os.path.exists(eb_mod_path): info("EasyBuild module installed: %s" % eb_mod_path) else: error("EasyBuild module not found at %s, define $EASYBUILD_BOOTSTRAP_DEBUG to debug" % eb_mod_path)
def stage2(tmpdir, templates, install_path, distribute_egg_dir, sourcepath): """STAGE 2: install EasyBuild to temporary dir with EasyBuild from stage 1.""" info("\n\n+++ STAGE 2: installing EasyBuild in %s with EasyBuild from stage 1...\n\n" % install_path) # inject path to distribute installed in stage 1 into $PYTHONPATH via preinstallopts # other approaches are not reliable, since EasyBuildMeta easyblock unsets $PYTHONPATH if distribute_egg_dir is None: preinstallopts = '' else: preinstallopts = 'PYTHONPATH=%s:$PYTHONPATH' % distribute_egg_dir templates.update({ 'preinstallopts': preinstallopts, }) # create easyconfig file ebfile = os.path.join(tmpdir, 'EasyBuild-%s.eb' % templates['version']) f = open(ebfile, "w") templates.update({ 'source_urls': '\n'.join(["'%s/%s/%s'," % (PYPI_SOURCE_URL, pkg[0], pkg) for pkg in EASYBUILD_PACKAGES]), 'sources': "%(vsc-base)s%(easybuild-framework)s%(easybuild-easyblocks)s%(easybuild-easyconfigs)s" % templates, 'pythonpath': distribute_egg_dir, }) f.write(EASYBUILD_EASYCONFIG_TEMPLATE % templates) f.close() # unset $MODULEPATH, we don't care about already installed modules os.environ['MODULEPATH'] = '' # set command line arguments for eb eb_args = ['eb', ebfile, '--allow-modules-tool-mismatch'] if print_debug: eb_args.extend(['--debug', '--logtostdout']) # make sure we don't leave any stuff behind in default path $HOME/.local/easybuild # and set build and install path explicitely if LooseVersion(templates['version']) < LooseVersion('1.3.0'): os.environ['EASYBUILD_PREFIX'] = tmpdir os.environ['EASYBUILD_BUILDPATH'] = tmpdir if install_path is not None: os.environ['EASYBUILD_INSTALLPATH'] = install_path else: # only for v1.3 and up eb_args.append('--prefix=%s' % tmpdir) eb_args.append('--buildpath=%s' % tmpdir) if install_path is not None: eb_args.append('--installpath=%s' % install_path) if sourcepath is not None: eb_args.append('--sourcepath=%s' % sourcepath) # make sure parent modules path already exists (Lmod trips over a non-existing entry in $MODULEPATH) if install_path is not None: modules_path = os.path.join(install_path, 'modules', 'all') if not os.path.exists(modules_path): os.makedirs(modules_path) debug("Created path %s" % modules_path) debug("Running EasyBuild with arguments '%s'" % ' '.join(eb_args)) sys.argv = eb_args # install EasyBuild with EasyBuild from easybuild.main import main as easybuild_main easybuild_main()
def stage2(tmpdir, templates, install_path, distribute_egg_dir, sourcepath): """STAGE 2: install EasyBuild to temporary dir with EasyBuild from stage 1.""" print('\n') info("+++ STAGE 2: installing EasyBuild in %s with EasyBuild from stage 1...\n" % install_path) preinstallopts = '' if distribute_egg_dir is not None: # ensure that (latest) setuptools is installed as well alongside EasyBuild, # since it is a required runtime dependency for recent vsc-base and EasyBuild versions # this is necessary since we provide our own distribute installation during the bootstrap (cfr. stage0) preinstallopts += "easy_install -U --prefix %(installdir)s setuptools && " # vsc-install is a runtime dependency for the EasyBuild unit test suite, # and is easily picked up from stage1 rather than being actually installed, so force it vsc_install_tarball_paths = glob.glob(os.path.join(sourcepath, 'vsc-install*.tar.gz')) if len(vsc_install_tarball_paths) == 1: vsc_install = vsc_install_tarball_paths[0] else: vsc_install = 'vsc-install' preinstallopts += "easy_install -U --prefix %%(installdir)s %s && " % vsc_install templates.update({ 'preinstallopts': preinstallopts, }) # create easyconfig file ebfile = os.path.join(tmpdir, 'EasyBuild-%s.eb' % templates['version']) handle = open(ebfile, 'w') templates.update({ 'source_urls': '\n'.join(["'%s/%s/%s'," % (PYPI_SOURCE_URL, pkg[0], pkg) for pkg in EASYBUILD_PACKAGES]), 'sources': "%(vsc-install)s%(vsc-base)s%(easybuild-framework)s%(easybuild-easyblocks)s%(easybuild-easyconfigs)s" % templates, 'pythonpath': distribute_egg_dir, }) handle.write(EASYBUILD_EASYCONFIG_TEMPLATE % templates) handle.close() # set command line arguments for eb eb_args = ['eb', ebfile, '--allow-modules-tool-mismatch'] if print_debug: eb_args.extend(['--debug', '--logtostdout']) if forced_install: info("Performing FORCED installation, as requested...") eb_args.append('--force') # make sure we don't leave any stuff behind in default path $HOME/.local/easybuild # and set build and install path explicitely if LooseVersion(templates['version']) < LooseVersion('1.3.0'): os.environ['EASYBUILD_PREFIX'] = tmpdir os.environ['EASYBUILD_BUILDPATH'] = tmpdir if install_path is not None: os.environ['EASYBUILD_INSTALLPATH'] = install_path else: # only for v1.3 and up eb_args.append('--prefix=%s' % tmpdir) eb_args.append('--buildpath=%s' % tmpdir) if install_path is not None: eb_args.append('--installpath=%s' % install_path) if sourcepath is not None: eb_args.append('--sourcepath=%s' % sourcepath) # make sure parent modules path already exists (Lmod trips over a non-existing entry in $MODULEPATH) if install_path is not None: modules_path = os.path.join(install_path, 'modules', 'all') if not os.path.exists(modules_path): os.makedirs(modules_path) debug("Created path %s" % modules_path) debug("Running EasyBuild with arguments '%s'" % ' '.join(eb_args)) sys.argv = eb_args # install EasyBuild with EasyBuild from easybuild.main import main as easybuild_main easybuild_main()
def stage2(tmpdir, templates, install_path, distribute_egg_dir, sourcepath): """STAGE 2: install EasyBuild to temporary dir with EasyBuild from stage 1.""" print('\n') info("+++ STAGE 2: installing EasyBuild in %s with EasyBuild from stage 1...\n" % install_path) preinstallopts = '' if distribute_egg_dir is not None: # inject path to distribute installed in stage 0 into $PYTHONPATH via preinstallopts # other approaches are not reliable, since EasyBuildMeta easyblock unsets $PYTHONPATH; # this is required for the easy_install from stage 0 to work preinstallopts += "export PYTHONPATH=%s:$PYTHONPATH && " % distribute_egg_dir # ensure that (latest) setuptools is installed as well alongside EasyBuild, # since it is a required runtime dependency for recent vsc-base and EasyBuild versions # this is necessary since we provide our own distribute installation during the bootstrap (cfr. stage0) preinstallopts += "%s -m easy_install -U --prefix %%(installdir)s setuptools && " % sys.executable # vsc-install is a runtime dependency for the EasyBuild unit test suite, # and is easily picked up from stage1 rather than being actually installed, so force it vsc_install = 'vsc-install' if sourcepath: vsc_install_tarball_paths = glob.glob(os.path.join(sourcepath, 'vsc-install*.tar.gz')) if len(vsc_install_tarball_paths) == 1: vsc_install = vsc_install_tarball_paths[0] preinstallopts += "%s -m easy_install -U --prefix %%(installdir)s %s && " % (sys.executable, vsc_install) templates.update({ 'preinstallopts': preinstallopts, }) # determine PyPI URLs for individual packages pkg_urls = [] for pkg in EASYBUILD_PACKAGES: # format of pkg entries in templates: "'<pkg_filename>'," pkg_filename = templates[pkg][1:-2] # the lines below implement a simplified version of the 'pypi_source_urls' and 'derive_alt_pypi_url' functions, # which we can't leverage here, partially because of transitional changes in PyPI (#md5= -> #sha256=) # determine download URL via PyPI's 'simple' API pkg_simple = None try: pkg_simple = urllib2.urlopen('https://pypi.python.org/simple/%s' % pkg, timeout=10).read() except (urllib2.URLError, urllib2.HTTPError) as err: # failing to figure out the package download URl may be OK when source tarballs are provided if sourcepath: info("Ignoring failed attempt to determine '%s' download URL since source tarballs are provided" % pkg) else: raise err if pkg_simple: pkg_url_part_regex = re.compile('/(packages/[^#]+)/%s#' % pkg_filename) res = pkg_url_part_regex.search(pkg_simple) if res: pkg_url_part = res.group(1) else: error("Failed to determine PyPI package URL for %s: %s\n" % (pkg, pkg_simple)) pkg_url = 'https://pypi.python.org/' + pkg_url_part pkg_urls.append(pkg_url) templates.update({ 'source_urls': '\n'.join(["'%s'," % pkg_url for pkg_url in pkg_urls]), 'sources': "%(vsc-install)s%(vsc-base)s%(easybuild-framework)s%(easybuild-easyblocks)s%(easybuild-easyconfigs)s" % templates, 'pythonpath': distribute_egg_dir, }) # create easyconfig file ebfile = os.path.join(tmpdir, 'EasyBuild-%s.eb' % templates['version']) handle = open(ebfile, 'w') ebfile_txt = EASYBUILD_EASYCONFIG_TEMPLATE % templates handle.write(ebfile_txt) handle.close() debug("Contents of generated easyconfig file:\n%s" % ebfile_txt) # set command line arguments for eb eb_args = ['eb', ebfile, '--allow-modules-tool-mismatch'] if print_debug: eb_args.extend(['--debug', '--logtostdout']) if forced_install: info("Performing FORCED installation, as requested...") eb_args.append('--force') # make sure we don't leave any stuff behind in default path $HOME/.local/easybuild # and set build and install path explicitely if LooseVersion(templates['version']) < LooseVersion('1.3.0'): os.environ['EASYBUILD_PREFIX'] = tmpdir os.environ['EASYBUILD_BUILDPATH'] = tmpdir if install_path is not None: os.environ['EASYBUILD_INSTALLPATH'] = install_path else: # only for v1.3 and up eb_args.append('--prefix=%s' % tmpdir) eb_args.append('--buildpath=%s' % tmpdir) if install_path is not None: eb_args.append('--installpath=%s' % install_path) if sourcepath is not None: eb_args.append('--sourcepath=%s' % sourcepath) # make sure parent modules path already exists (Lmod trips over a non-existing entry in $MODULEPATH) if install_path is not None: modules_path = det_modules_path(install_path) if not os.path.exists(modules_path): os.makedirs(modules_path) debug("Created path %s" % modules_path) debug("Running EasyBuild with arguments '%s'" % ' '.join(eb_args)) sys.argv = eb_args # location to 'eb' command (from stage 1) may be expected to be included in $PATH # it usually is there after stage1, unless 'prep' is called again with another location # (only when stage 0 is not skipped) # cfr. https://github.com/easybuilders/easybuild-framework/issues/2279 curr_path = [x for x in os.environ.get('PATH', '').split(os.pathsep) if len(x) > 0] os.environ['PATH'] = os.pathsep.join([os.path.join(tmpdir, STAGE1_SUBDIR, 'bin')] + curr_path) debug("$PATH: %s" % os.environ['PATH']) # install EasyBuild with EasyBuild from easybuild.main import main as easybuild_main easybuild_main() if print_debug: os.environ['EASYBUILD_DEBUG'] = '1' # make sure the EasyBuild module was actually installed # EasyBuild configuration options that are picked up from configuration files/environment may break the bootstrap, # for example by having $EASYBUILD_VERSION defined or via a configuration file specifies a value for 'stop'... from easybuild.tools.config import build_option, install_path, get_module_syntax from easybuild.framework.easyconfig.easyconfig import ActiveMNS eb_spec = { 'name': 'EasyBuild', 'hidden': False, 'toolchain': {'name': 'dummy', 'version': 'dummy'}, 'version': templates['version'], 'versionprefix': '', 'versionsuffix': '', 'moduleclass': 'tools', } mod_path = os.path.join(install_path('mod'), build_option('suffix_modules_path')) debug("EasyBuild module should have been installed to %s" % mod_path) eb_mod_name = ActiveMNS().det_full_module_name(eb_spec) debug("EasyBuild module name: %s" % eb_mod_name) eb_mod_path = os.path.join(mod_path, eb_mod_name) if get_module_syntax() == 'Lua': eb_mod_path += '.lua' if os.path.exists(eb_mod_path): info("EasyBuild module installed: %s" % eb_mod_path) else: error("EasyBuild module not found at %s, define $EASYBUILD_BOOTSTRAP_DEBUG to debug" % eb_mod_path)
def stage2(tmpdir, templates, install_path, distribute_egg_dir, sourcepath): """STAGE 2: install EasyBuild to temporary dir with EasyBuild from stage 1.""" print('\n') info( "+++ STAGE 2: installing EasyBuild in %s with EasyBuild from stage 1...\n" % install_path) preinstallopts = '' if distribute_egg_dir is not None: # inject path to distribute installed in stage 1 into $PYTHONPATH via preinstallopts # other approaches are not reliable, since EasyBuildMeta easyblock unsets $PYTHONPATH; # this is required for the easy_install from stage 1 to work preinstallopts += "export PYTHONPATH=%s:$PYTHONPATH && " % distribute_egg_dir # ensure that (latest) setuptools is installed as well alongside EasyBuild, # since it is a required runtime dependency for recent vsc-base and EasyBuild versions # this is necessary since we provide our own distribute installation during the bootstrap (cfr. stage0) preinstallopts += "%s $(which easy_install) -U --prefix %%(installdir)s setuptools && " % sys.executable # vsc-install is a runtime dependency for the EasyBuild unit test suite, # and is easily picked up from stage1 rather than being actually installed, so force it vsc_install = 'vsc-install' if sourcepath: vsc_install_tarball_paths = glob.glob( os.path.join(sourcepath, 'vsc-install*.tar.gz')) if len(vsc_install_tarball_paths) == 1: vsc_install = vsc_install_tarball_paths[0] preinstallopts += "%s $(which easy_install) -U --prefix %%(installdir)s %s && " % ( sys.executable, vsc_install) templates.update({ 'preinstallopts': preinstallopts, }) # create easyconfig file ebfile = os.path.join(tmpdir, 'EasyBuild-%s.eb' % templates['version']) handle = open(ebfile, 'w') templates.update({ 'source_urls': '\n'.join([ "'%s/%s/%s'," % (PYPI_SOURCE_URL, pkg[0], pkg) for pkg in EASYBUILD_PACKAGES ]), 'sources': "%(vsc-install)s%(vsc-base)s%(easybuild-framework)s%(easybuild-easyblocks)s%(easybuild-easyconfigs)s" % templates, 'pythonpath': distribute_egg_dir, }) handle.write(EASYBUILD_EASYCONFIG_TEMPLATE % templates) handle.close() # set command line arguments for eb eb_args = ['eb', ebfile, '--allow-modules-tool-mismatch'] if print_debug: eb_args.extend(['--debug', '--logtostdout']) if forced_install: info("Performing FORCED installation, as requested...") eb_args.append('--force') # make sure we don't leave any stuff behind in default path $HOME/.local/easybuild # and set build and install path explicitely if LooseVersion(templates['version']) < LooseVersion('1.3.0'): os.environ['EASYBUILD_PREFIX'] = tmpdir os.environ['EASYBUILD_BUILDPATH'] = tmpdir if install_path is not None: os.environ['EASYBUILD_INSTALLPATH'] = install_path else: # only for v1.3 and up eb_args.append('--prefix=%s' % tmpdir) eb_args.append('--buildpath=%s' % tmpdir) if install_path is not None: eb_args.append('--installpath=%s' % install_path) if sourcepath is not None: eb_args.append('--sourcepath=%s' % sourcepath) # make sure parent modules path already exists (Lmod trips over a non-existing entry in $MODULEPATH) if install_path is not None: modules_path = det_modules_path(install_path) if not os.path.exists(modules_path): os.makedirs(modules_path) debug("Created path %s" % modules_path) debug("Running EasyBuild with arguments '%s'" % ' '.join(eb_args)) sys.argv = eb_args # location to 'eb' command (from stage 1) may be expected to be included in $PATH # it usually is there after stage1, unless 'prep' is called again with another location # (only when stage 0 is not skipped) # cfr. https://github.com/easybuilders/easybuild-framework/issues/2279 curr_path = [ x for x in os.environ.get('PATH', '').split(os.pathsep) if len(x) > 0 ] os.environ['PATH'] = os.pathsep.join( [os.path.join(tmpdir, STAGE1_SUBDIR, 'bin')] + curr_path) debug("$PATH: %s" % os.environ['PATH']) # install EasyBuild with EasyBuild from easybuild.main import main as easybuild_main easybuild_main() # make sure the EasyBuild module was actually installed # EasyBuild configuration options that are picked up from configuration files/environment may break the bootstrap, # for example by having $EASYBUILD_VERSION defined or via a configuration file specifies a value for 'stop'... from easybuild.tools.config import build_option, install_path, get_module_syntax from easybuild.framework.easyconfig.easyconfig import ActiveMNS eb_spec = { 'name': 'EasyBuild', 'hidden': False, 'toolchain': { 'name': 'dummy', 'version': 'dummy' }, 'version': templates['version'], 'versionprefix': '', 'versionsuffix': '', 'moduleclass': 'tools', } mod_path = os.path.join(install_path('mod'), build_option('suffix_modules_path')) debug("EasyBuild module should have been installed to %s" % mod_path) eb_mod_name = ActiveMNS().det_full_module_name(eb_spec) debug("EasyBuild module name: %s" % eb_mod_name) eb_mod_path = os.path.join(mod_path, eb_mod_name) if get_module_syntax() == 'Lua': eb_mod_path += '.lua' if os.path.exists(eb_mod_path): info("EasyBuild module installed: %s" % eb_mod_path) else: error( "EasyBuild module not found at %s, define $EASYBUILD_BOOTSTRAP_DEBUG to debug" % eb_mod_path)
def stage2(tmpdir, templates, install_path, distribute_egg_dir, sourcepath): """STAGE 2: install EasyBuild to temporary dir with EasyBuild from stage 1.""" print('\n') info("+++ STAGE 2: installing EasyBuild in %s with EasyBuild from stage 1...\n" % install_path) preinstallopts = '' if distribute_egg_dir is not None: # inject path to distribute installed in stage 1 into $PYTHONPATH via preinstallopts # other approaches are not reliable, since EasyBuildMeta easyblock unsets $PYTHONPATH; # this is required for the easy_install from stage 1 to work preinstallopts += "export PYTHONPATH=%s:$PYTHONPATH && " % distribute_egg_dir # ensure that (latest) setuptools is installed as well alongside EasyBuild, # since it is a required runtime dependency for recent vsc-base and EasyBuild versions # this is necessary since we provide our own distribute installation during the bootstrap (cfr. stage0) preinstallopts += "%s $(which easy_install) -U --prefix %%(installdir)s setuptools && " % sys.executable # vsc-install is a runtime dependency for the EasyBuild unit test suite, # and is easily picked up from stage1 rather than being actually installed, so force it vsc_install = 'vsc-install' if sourcepath: vsc_install_tarball_paths = glob.glob(os.path.join(sourcepath, 'vsc-install*.tar.gz')) if len(vsc_install_tarball_paths) == 1: vsc_install = vsc_install_tarball_paths[0] preinstallopts += "%s $(which easy_install) -U --prefix %%(installdir)s %s && " % (sys.executable, vsc_install) templates.update({ 'preinstallopts': preinstallopts, }) # create easyconfig file ebfile = os.path.join(tmpdir, 'EasyBuild-%s.eb' % templates['version']) handle = open(ebfile, 'w') templates.update({ 'source_urls': '\n'.join(["'%s/%s/%s'," % (PYPI_SOURCE_URL, pkg[0], pkg) for pkg in EASYBUILD_PACKAGES]), 'sources': "%(vsc-install)s%(vsc-base)s%(easybuild-framework)s%(easybuild-easyblocks)s%(easybuild-easyconfigs)s" % templates, 'pythonpath': distribute_egg_dir, }) handle.write(EASYBUILD_EASYCONFIG_TEMPLATE % templates) handle.close() # set command line arguments for eb eb_args = ['eb', ebfile, '--allow-modules-tool-mismatch'] if print_debug: eb_args.extend(['--debug', '--logtostdout']) if forced_install: info("Performing FORCED installation, as requested...") eb_args.append('--force') # make sure we don't leave any stuff behind in default path $HOME/.local/easybuild # and set build and install path explicitely if LooseVersion(templates['version']) < LooseVersion('1.3.0'): os.environ['EASYBUILD_PREFIX'] = tmpdir os.environ['EASYBUILD_BUILDPATH'] = tmpdir if install_path is not None: os.environ['EASYBUILD_INSTALLPATH'] = install_path else: # only for v1.3 and up eb_args.append('--prefix=%s' % tmpdir) eb_args.append('--buildpath=%s' % tmpdir) if install_path is not None: eb_args.append('--installpath=%s' % install_path) if sourcepath is not None: eb_args.append('--sourcepath=%s' % sourcepath) # make sure parent modules path already exists (Lmod trips over a non-existing entry in $MODULEPATH) if install_path is not None: modules_path = det_modules_path(install_path) if not os.path.exists(modules_path): os.makedirs(modules_path) debug("Created path %s" % modules_path) debug("Running EasyBuild with arguments '%s'" % ' '.join(eb_args)) sys.argv = eb_args # location to 'eb' command (from stage 1) may be expected to be included in $PATH # it usually is there after stage1, unless 'prep' is called again with another location # (only when stage 0 is not skipped) # cfr. https://github.com/easybuilders/easybuild-framework/issues/2279 curr_path = [x for x in os.environ.get('PATH', '').split(os.pathsep) if len(x) > 0] os.environ['PATH'] = os.pathsep.join([os.path.join(tmpdir, STAGE1_SUBDIR, 'bin')] + curr_path) debug("$PATH: %s" % os.environ['PATH']) # install EasyBuild with EasyBuild from easybuild.main import main as easybuild_main easybuild_main() # make sure the EasyBuild module was actually installed # EasyBuild configuration options that are picked up from configuration files/environment may break the bootstrap, # for example by having $EASYBUILD_VERSION defined or via a configuration file specifies a value for 'stop'... from easybuild.tools.config import build_option, install_path, get_module_syntax from easybuild.framework.easyconfig.easyconfig import ActiveMNS eb_spec = { 'name': 'EasyBuild', 'hidden': False, 'toolchain': {'name': 'dummy', 'version': 'dummy'}, 'version': templates['version'], 'versionprefix': '', 'versionsuffix': '', 'moduleclass': 'tools', } mod_path = os.path.join(install_path('mod'), build_option('suffix_modules_path')) debug("EasyBuild module should have been installed to %s" % mod_path) eb_mod_name = ActiveMNS().det_full_module_name(eb_spec) debug("EasyBuild module name: %s" % eb_mod_name) eb_mod_path = os.path.join(mod_path, eb_mod_name) if get_module_syntax() == 'Lua': eb_mod_path += '.lua' if os.path.exists(eb_mod_path): info("EasyBuild module installed: %s" % eb_mod_path) else: error("EasyBuild module not found at %s, define $EASYBUILD_BOOTSTRAP_DEBUG to debug" % eb_mod_path)
def stage2(tmpdir, templates, install_path, distribute_egg_dir, sourcepath): """STAGE 2: install EasyBuild to temporary dir with EasyBuild from stage 1.""" info( "\n\n+++ STAGE 2: installing EasyBuild in %s with EasyBuild from stage 1...\n\n" % install_path) # inject path to distribute installed in stage 1 into $PYTHONPATH via preinstallopts # other approaches are not reliable, since EasyBuildMeta easyblock unsets $PYTHONPATH if distribute_egg_dir is None: preinstallopts = '' else: preinstallopts = 'PYTHONPATH=%s:$PYTHONPATH' % distribute_egg_dir templates.update({ 'preinstallopts': preinstallopts, }) # create easyconfig file ebfile = os.path.join(tmpdir, 'EasyBuild-%s.eb' % templates['version']) f = open(ebfile, "w") templates.update({ 'source_urls': '\n'.join([ "'%s/%s/%s'," % (PYPI_SOURCE_URL, pkg[0], pkg) for pkg in EASYBUILD_PACKAGES ]), 'sources': "%(vsc-base)s%(easybuild-framework)s%(easybuild-easyblocks)s%(easybuild-easyconfigs)s" % templates, 'pythonpath': distribute_egg_dir, }) f.write(EASYBUILD_EASYCONFIG_TEMPLATE % templates) f.close() # unset $MODULEPATH, we don't care about already installed modules os.environ['MODULEPATH'] = '' # set command line arguments for eb eb_args = ['eb', ebfile, '--allow-modules-tool-mismatch'] if print_debug: eb_args.extend(['--debug', '--logtostdout']) # make sure we don't leave any stuff behind in default path $HOME/.local/easybuild # and set build and install path explicitely if LooseVersion(templates['version']) < LooseVersion('1.3.0'): os.environ['EASYBUILD_PREFIX'] = tmpdir os.environ['EASYBUILD_BUILDPATH'] = tmpdir if install_path is not None: os.environ['EASYBUILD_INSTALLPATH'] = install_path else: # only for v1.3 and up eb_args.append('--prefix=%s' % tmpdir) eb_args.append('--buildpath=%s' % tmpdir) if install_path is not None: eb_args.append('--installpath=%s' % install_path) if sourcepath is not None: eb_args.append('--sourcepath=%s' % sourcepath) # make sure parent modules path already exists (Lmod trips over a non-existing entry in $MODULEPATH) if install_path is not None: modules_path = os.path.join(install_path, 'modules', 'all') if not os.path.exists(modules_path): os.makedirs(modules_path) debug("Created path %s" % modules_path) debug("Running EasyBuild with arguments '%s'" % ' '.join(eb_args)) sys.argv = eb_args # install EasyBuild with EasyBuild from easybuild.main import main as easybuild_main easybuild_main()