def stage2(tmpdir, versions, install_path):
"""STAGE 2: install EasyBuild to temporary dir with EasyBuild from stage 1."""
info(
"\n\n+++ STAGE 2: installing EasyBuild in %s with EasyBuild from stage 1...\n\n"
% install_path)
# make sure we still have distribute in PYTHONPATH, so we have control over which 'setup' is used
pythonpaths = [
x for x in os.environ.get('PYTHONPATH', '').split(os.pathsep)
if len(x) > 0
]
os.environ['PYTHONPATH'] = os.pathsep.join(
[find_egg_dir_for(tmpdir, 'distribute')] + pythonpaths)
# create easyconfig file
ebfile = os.path.join(tmpdir, 'EasyBuild-%s.eb' % versions['version'])
f = open(ebfile, "w")
f.write(EB_EC_FILE % versions)
f.close()
# unset $MODULEPATH, we don't care about already installed modules
os.environ['MODULEPATH'] = ''
# set command line arguments for eb
eb_args = ['eb', ebfile, '--allow-modules-tool-mismatch']
if print_debug:
eb_args.extend(['--debug', '--logtostdout'])
# make sure we don't leave any stuff behind in default path $HOME/.local/easybuild
# and set build and install path explicitely
if LooseVersion(versions['version']) < LooseVersion("1.3.0"):
os.environ['EASYBUILD_PREFIX'] = tmpdir
os.environ['EASYBUILD_BUILDPATH'] = tmpdir
if install_path is not None:
os.environ['EASYBUILD_INSTALLPATH'] = install_path
else:
# only for v1.3 and up
eb_args.append('--prefix=%s' % tmpdir)
eb_args.append('--buildpath=%s' % tmpdir)
if install_path is not None:
eb_args.append('--installpath=%s' % install_path)
# make sure parent modules path already exists (Lmod trips over a non-existing entry in $MODULEPATH)
if install_path is not None:
modules_path = os.path.join(install_path, 'modules', 'all')
if not os.path.exists(modules_path):
os.makedirs(modules_path)
debug("Created path %s" % modules_path)
debug("Running EasyBuild with arguments '%s'" % ' '.join(eb_args))
sys.argv = eb_args
# install EasyBuild with EasyBuild
from easybuild.main import main as easybuild_main
easybuild_main()
def stage2(tmpdir, versions, install_path):
"""STAGE 2: install EasyBuild to temporary dir with EasyBuild from stage 1."""
info("\n\n+++ STAGE 2: installing EasyBuild in %s with EasyBuild from stage 1...\n\n" % install_path)
# make sure we still have distribute in PYTHONPATH, so we have control over which 'setup' is used
pythonpaths = [x for x in os.environ.get('PYTHONPATH', '').split(os.pathsep) if len(x) > 0]
os.environ['PYTHONPATH'] = os.pathsep.join([find_egg_dir_for(tmpdir, 'distribute')] + pythonpaths)
# create easyconfig file
ebfile = os.path.join(tmpdir, 'EasyBuild-%s.eb' % versions['version'])
f = open(ebfile, "w")
f.write(EB_EC_FILE % versions)
f.close()
# unset $MODULEPATH, we don't care about already installed modules
os.environ['MODULEPATH'] = ''
# set command line arguments for eb
eb_args = ['eb', ebfile, '--allow-modules-tool-mismatch']
if print_debug:
eb_args.extend(['--debug', '--logtostdout'])
# make sure we don't leave any stuff behind in default path $HOME/.local/easybuild
# and set build and install path explicitely
if LooseVersion(versions['version']) < LooseVersion("1.3.0"):
os.environ['EASYBUILD_PREFIX'] = tmpdir
os.environ['EASYBUILD_BUILDPATH'] = tmpdir
if install_path is not None:
os.environ['EASYBUILD_INSTALLPATH'] = install_path
else:
# only for v1.3 and up
eb_args.append('--prefix=%s' % tmpdir)
eb_args.append('--buildpath=%s' % tmpdir)
if install_path is not None:
eb_args.append('--installpath=%s' % install_path)
# make sure parent modules path already exists (Lmod trips over a non-existing entry in $MODULEPATH)
if install_path is not None:
modules_path = os.path.join(install_path, 'modules', 'all')
if not os.path.exists(modules_path):
os.makedirs(modules_path)
debug("Created path %s" % modules_path)
debug("Running EasyBuild with arguments '%s'" % ' '.join(eb_args))
sys.argv = eb_args
# install EasyBuild with EasyBuild
from easybuild.main import main as easybuild_main
easybuild_main()
def stage2(tmpdir, templates, install_path, distribute_egg_dir, sourcepath):
"""STAGE 2: install EasyBuild to temporary dir with EasyBuild from stage 1."""
info(
"\n\n+++ STAGE 2: installing EasyBuild in %s with EasyBuild from stage 1...\n\n"
% install_path)
if distribute_egg_dir is None:
preinstallopts = ''
else:
# inject path to distribute installed in stage 1 into $PYTHONPATH via preinstallopts
# other approaches are not reliable, since EasyBuildMeta easyblock unsets $PYTHONPATH;
preinstallopts = "export PYTHONPATH=%s:$PYTHONPATH && " % distribute_egg_dir
# also add location to easy_install provided through stage0 to $PATH
curr_path = os.environ.get('PATH', '').split(os.pathsep)
os.environ['PATH'] = os.pathsep.join([os.path.join(tmpdir, 'bin')] +
curr_path)
debug("$PATH: %s" % os.environ['PATH'])
# ensure that (latest) setuptools is installed as well alongside EasyBuild,
# since it is a required runtime dependency for recent vsc-base and EasyBuild versions
# this is necessary since we provide our own distribute installation during the bootstrap (cfr. stage0)
preinstallopts += "easy_install -U --prefix %(installdir)s setuptools && "
# vsc-install is a runtime dependency for the EasyBuild unit test suite,
# and is easily picked up from stage1 rather than being actually installed, so force it
preinstallopts += "easy_install -U --prefix %(installdir)s vsc-install && "
templates.update({
'preinstallopts': preinstallopts,
})
# create easyconfig file
ebfile = os.path.join(tmpdir, 'EasyBuild-%s.eb' % templates['version'])
handle = open(ebfile, 'w')
templates.update({
'source_urls':
'\n'.join([
"'%s/%s/%s'," % (PYPI_SOURCE_URL, pkg[0], pkg)
for pkg in EASYBUILD_PACKAGES
]),
'sources':
"%(vsc-base)s%(easybuild-framework)s%(easybuild-easyblocks)s%(easybuild-easyconfigs)s"
% templates,
'pythonpath':
distribute_egg_dir,
})
handle.write(EASYBUILD_EASYCONFIG_TEMPLATE % templates)
handle.close()
# unset $MODULEPATH, we don't care about already installed modules
os.environ['MODULEPATH'] = ''
# set command line arguments for eb
eb_args = ['eb', ebfile, '--allow-modules-tool-mismatch']
if print_debug:
eb_args.extend(['--debug', '--logtostdout'])
# make sure we don't leave any stuff behind in default path $HOME/.local/easybuild
# and set build and install path explicitely
if LooseVersion(templates['version']) < LooseVersion('1.3.0'):
os.environ['EASYBUILD_PREFIX'] = tmpdir
os.environ['EASYBUILD_BUILDPATH'] = tmpdir
if install_path is not None:
os.environ['EASYBUILD_INSTALLPATH'] = install_path
else:
# only for v1.3 and up
eb_args.append('--prefix=%s' % tmpdir)
eb_args.append('--buildpath=%s' % tmpdir)
if install_path is not None:
eb_args.append('--installpath=%s' % install_path)
if sourcepath is not None:
eb_args.append('--sourcepath=%s' % sourcepath)
# make sure parent modules path already exists (Lmod trips over a non-existing entry in $MODULEPATH)
if install_path is not None:
modules_path = os.path.join(install_path, 'modules', 'all')
if not os.path.exists(modules_path):
os.makedirs(modules_path)
debug("Created path %s" % modules_path)
debug("Running EasyBuild with arguments '%s'" % ' '.join(eb_args))
sys.argv = eb_args
# install EasyBuild with EasyBuild
from easybuild.main import main as easybuild_main
easybuild_main()
def stage2(tmpdir, templates, install_path, distribute_egg_dir, sourcepath):
"""STAGE 2: install EasyBuild to temporary dir with EasyBuild from stage 1."""
print('\n')
info("+++ STAGE 2: installing EasyBuild in %s with EasyBuild from stage 1...\n" % install_path)
preinstallopts = ''
if distribute_egg_dir is not None:
# inject path to distribute installed in stage 1 into $PYTHONPATH via preinstallopts
# other approaches are not reliable, since EasyBuildMeta easyblock unsets $PYTHONPATH;
# this is required for the easy_install from stage 1 to work
preinstallopts += "export PYTHONPATH=%s:$PYTHONPATH && " % distribute_egg_dir
# ensure that (latest) setuptools is installed as well alongside EasyBuild,
# since it is a required runtime dependency for recent vsc-base and EasyBuild versions
# this is necessary since we provide our own distribute installation during the bootstrap (cfr. stage0)
preinstallopts += "%s $(which easy_install) -U --prefix %%(installdir)s setuptools && " % sys.executable
# vsc-install is a runtime dependency for the EasyBuild unit test suite,
# and is easily picked up from stage1 rather than being actually installed, so force it
vsc_install = 'vsc-install'
if sourcepath:
vsc_install_tarball_paths = glob.glob(os.path.join(sourcepath, 'vsc-install*.tar.gz'))
if len(vsc_install_tarball_paths) == 1:
vsc_install = vsc_install_tarball_paths[0]
preinstallopts += "%s $(which easy_install) -U --prefix %%(installdir)s %s && " % (sys.executable, vsc_install)
templates.update({
'preinstallopts': preinstallopts,
})
# create easyconfig file
ebfile = os.path.join(tmpdir, 'EasyBuild-%s.eb' % templates['version'])
handle = open(ebfile, 'w')
templates.update({
'source_urls': '\n'.join(["'%s/%s/%s'," % (PYPI_SOURCE_URL, pkg[0], pkg) for pkg in EASYBUILD_PACKAGES]),
'sources': "%(vsc-install)s%(vsc-base)s%(easybuild-framework)s%(easybuild-easyblocks)s%(easybuild-easyconfigs)s" % templates,
'pythonpath': distribute_egg_dir,
})
handle.write(EASYBUILD_EASYCONFIG_TEMPLATE % templates)
handle.close()
# set command line arguments for eb
eb_args = ['eb', ebfile, '--allow-modules-tool-mismatch']
if print_debug:
eb_args.extend(['--debug', '--logtostdout'])
if forced_install:
info("Performing FORCED installation, as requested...")
eb_args.append('--force')
# make sure we don't leave any stuff behind in default path $HOME/.local/easybuild
# and set build and install path explicitely
if LooseVersion(templates['version']) < LooseVersion('1.3.0'):
os.environ['EASYBUILD_PREFIX'] = tmpdir
os.environ['EASYBUILD_BUILDPATH'] = tmpdir
if install_path is not None:
os.environ['EASYBUILD_INSTALLPATH'] = install_path
else:
# only for v1.3 and up
eb_args.append('--prefix=%s' % tmpdir)
eb_args.append('--buildpath=%s' % tmpdir)
if install_path is not None:
eb_args.append('--installpath=%s' % install_path)
if sourcepath is not None:
eb_args.append('--sourcepath=%s' % sourcepath)
# make sure parent modules path already exists (Lmod trips over a non-existing entry in $MODULEPATH)
if install_path is not None:
modules_path = os.path.join(install_path, 'modules', 'all')
if not os.path.exists(modules_path):
os.makedirs(modules_path)
debug("Created path %s" % modules_path)
debug("Running EasyBuild with arguments '%s'" % ' '.join(eb_args))
sys.argv = eb_args
# install EasyBuild with EasyBuild
from easybuild.main import main as easybuild_main
easybuild_main()
def stage2(tmpdir, templates, install_path, distribute_egg_dir, sourcepath):
"""STAGE 2: install EasyBuild to temporary dir with EasyBuild from stage 1."""
print('\n')
info("+++ STAGE 2: installing EasyBuild in %s with EasyBuild from stage 1...\n" % install_path)
preinstallopts = ''
if distribute_egg_dir is not None:
# inject path to distribute installed in stage 0 into $PYTHONPATH via preinstallopts
# other approaches are not reliable, since EasyBuildMeta easyblock unsets $PYTHONPATH;
# this is required for the easy_install from stage 0 to work
preinstallopts += "export PYTHONPATH=%s:$PYTHONPATH && " % distribute_egg_dir
# ensure that (latest) setuptools is installed as well alongside EasyBuild,
# since it is a required runtime dependency for recent vsc-base and EasyBuild versions
# this is necessary since we provide our own distribute installation during the bootstrap (cfr. stage0)
preinstallopts += "%s -m easy_install -U --prefix %%(installdir)s setuptools && " % sys.executable
# vsc-install is a runtime dependency for the EasyBuild unit test suite,
# and is easily picked up from stage1 rather than being actually installed, so force it
vsc_install = "'%s<0.11.4'" % VSC_INSTALL
if sourcepath:
vsc_install_tarball_paths = glob.glob(os.path.join(sourcepath, 'vsc-install*.tar.gz'))
if len(vsc_install_tarball_paths) == 1:
vsc_install = vsc_install_tarball_paths[0]
preinstallopts += "%s -m easy_install -U --prefix %%(installdir)s %s && " % (sys.executable, vsc_install)
templates.update({
'preinstallopts': preinstallopts,
})
# determine PyPI URLs for individual packages
pkg_urls = []
for pkg in EASYBUILD_PACKAGES:
# format of pkg entries in templates: "'<pkg_filename>',"
pkg_filename = templates[pkg][1:-2]
# the lines below implement a simplified version of the 'pypi_source_urls' and 'derive_alt_pypi_url' functions,
# which we can't leverage here, partially because of transitional changes in PyPI (#md5= -> #sha256=)
# determine download URL via PyPI's 'simple' API
pkg_simple = None
try:
pkg_simple = urllib2.urlopen('https://pypi.python.org/simple/%s' % pkg, timeout=10).read()
except (urllib2.URLError, urllib2.HTTPError) as err:
# failing to figure out the package download URl may be OK when source tarballs are provided
if sourcepath:
info("Ignoring failed attempt to determine '%s' download URL since source tarballs are provided" % pkg)
else:
raise err
if pkg_simple:
pkg_url_part_regex = re.compile('/(packages/[^#]+)/%s#' % pkg_filename)
res = pkg_url_part_regex.search(pkg_simple)
if res:
pkg_url_part = res.group(1)
else:
error_msg = "Failed to determine PyPI package URL for %s using pattern '%s': %s\n"
error(error_msg % (pkg, pkg_url_part_regex.pattern, pkg_simple))
pkg_url = 'https://pypi.python.org/' + pkg_url_part
pkg_urls.append(pkg_url)
templates.update({
'source_urls': '\n'.join(["'%s'," % pkg_url for pkg_url in pkg_urls]),
'sources': "%(vsc-install)s%(vsc-base)s%(easybuild-framework)s%(easybuild-easyblocks)s%(easybuild-easyconfigs)s" % templates,
'pythonpath': distribute_egg_dir,
})
# create easyconfig file
ebfile = os.path.join(tmpdir, 'EasyBuild-%s.eb' % templates['version'])
handle = open(ebfile, 'w')
ebfile_txt = EASYBUILD_EASYCONFIG_TEMPLATE % templates
handle.write(ebfile_txt)
handle.close()
debug("Contents of generated easyconfig file:\n%s" % ebfile_txt)
# set command line arguments for eb
eb_args = ['eb', ebfile, '--allow-modules-tool-mismatch']
if print_debug:
eb_args.extend(['--debug', '--logtostdout'])
if forced_install:
info("Performing FORCED installation, as requested...")
eb_args.append('--force')
# make sure we don't leave any stuff behind in default path $HOME/.local/easybuild
# and set build and install path explicitely
if LooseVersion(templates['version']) < LooseVersion('1.3.0'):
os.environ['EASYBUILD_PREFIX'] = tmpdir
os.environ['EASYBUILD_BUILDPATH'] = tmpdir
if install_path is not None:
os.environ['EASYBUILD_INSTALLPATH'] = install_path
else:
# only for v1.3 and up
eb_args.append('--prefix=%s' % tmpdir)
eb_args.append('--buildpath=%s' % tmpdir)
if install_path is not None:
eb_args.append('--installpath=%s' % install_path)
if sourcepath is not None:
eb_args.append('--sourcepath=%s' % sourcepath)
# make sure EasyBuild can find EasyBuild-*.eb easyconfig file when it needs to;
# (for example when HierarchicalMNS is used as module naming scheme,
# see https://github.com/easybuilders/easybuild-framework/issues/2393)
eb_args.append('--robot-paths=%s:' % tmpdir)
# make sure parent modules path already exists (Lmod trips over a non-existing entry in $MODULEPATH)
if install_path is not None:
modules_path = det_modules_path(install_path)
if not os.path.exists(modules_path):
os.makedirs(modules_path)
debug("Created path %s" % modules_path)
debug("Running EasyBuild with arguments '%s'" % ' '.join(eb_args))
sys.argv = eb_args
# location to 'eb' command (from stage 1) may be expected to be included in $PATH
# it usually is there after stage1, unless 'prep' is called again with another location
# (only when stage 0 is not skipped)
# cfr. https://github.com/easybuilders/easybuild-framework/issues/2279
curr_path = [x for x in os.environ.get('PATH', '').split(os.pathsep) if len(x) > 0]
os.environ['PATH'] = os.pathsep.join([os.path.join(tmpdir, STAGE1_SUBDIR, 'bin')] + curr_path)
debug("$PATH: %s" % os.environ['PATH'])
# install EasyBuild with EasyBuild
from easybuild.main import main as easybuild_main
easybuild_main()
if print_debug:
os.environ['EASYBUILD_DEBUG'] = '1'
# make sure the EasyBuild module was actually installed
# EasyBuild configuration options that are picked up from configuration files/environment may break the bootstrap,
# for example by having $EASYBUILD_VERSION defined or via a configuration file specifies a value for 'stop'...
from easybuild.tools.config import build_option, install_path, get_module_syntax
from easybuild.framework.easyconfig.easyconfig import ActiveMNS
eb_spec = {
'name': 'EasyBuild',
'hidden': False,
'toolchain': {'name': 'dummy', 'version': 'dummy'},
'version': templates['version'],
'versionprefix': '',
'versionsuffix': '',
'moduleclass': 'tools',
}
mod_path = os.path.join(install_path('mod'), build_option('suffix_modules_path'))
debug("EasyBuild module should have been installed to %s" % mod_path)
eb_mod_name = ActiveMNS().det_full_module_name(eb_spec)
debug("EasyBuild module name: %s" % eb_mod_name)
eb_mod_path = os.path.join(mod_path, eb_mod_name)
if get_module_syntax() == 'Lua':
eb_mod_path += '.lua'
if os.path.exists(eb_mod_path):
info("EasyBuild module installed: %s" % eb_mod_path)
else:
error("EasyBuild module not found at %s, define $EASYBUILD_BOOTSTRAP_DEBUG to debug" % eb_mod_path)
def stage2(tmpdir, templates, install_path, distribute_egg_dir, sourcepath):
"""STAGE 2: install EasyBuild to temporary dir with EasyBuild from stage 1."""
info("\n\n+++ STAGE 2: installing EasyBuild in %s with EasyBuild from stage 1...\n\n" % install_path)
# inject path to distribute installed in stage 1 into $PYTHONPATH via preinstallopts
# other approaches are not reliable, since EasyBuildMeta easyblock unsets $PYTHONPATH
if distribute_egg_dir is None:
preinstallopts = ''
else:
preinstallopts = 'PYTHONPATH=%s:$PYTHONPATH' % distribute_egg_dir
templates.update({
'preinstallopts': preinstallopts,
})
# create easyconfig file
ebfile = os.path.join(tmpdir, 'EasyBuild-%s.eb' % templates['version'])
f = open(ebfile, "w")
templates.update({
'source_urls': '\n'.join(["'%s/%s/%s'," % (PYPI_SOURCE_URL, pkg[0], pkg) for pkg in EASYBUILD_PACKAGES]),
'sources': "%(vsc-base)s%(easybuild-framework)s%(easybuild-easyblocks)s%(easybuild-easyconfigs)s" % templates,
'pythonpath': distribute_egg_dir,
})
f.write(EASYBUILD_EASYCONFIG_TEMPLATE % templates)
f.close()
# unset $MODULEPATH, we don't care about already installed modules
os.environ['MODULEPATH'] = ''
# set command line arguments for eb
eb_args = ['eb', ebfile, '--allow-modules-tool-mismatch']
if print_debug:
eb_args.extend(['--debug', '--logtostdout'])
# make sure we don't leave any stuff behind in default path $HOME/.local/easybuild
# and set build and install path explicitely
if LooseVersion(templates['version']) < LooseVersion('1.3.0'):
os.environ['EASYBUILD_PREFIX'] = tmpdir
os.environ['EASYBUILD_BUILDPATH'] = tmpdir
if install_path is not None:
os.environ['EASYBUILD_INSTALLPATH'] = install_path
else:
# only for v1.3 and up
eb_args.append('--prefix=%s' % tmpdir)
eb_args.append('--buildpath=%s' % tmpdir)
if install_path is not None:
eb_args.append('--installpath=%s' % install_path)
if sourcepath is not None:
eb_args.append('--sourcepath=%s' % sourcepath)
# make sure parent modules path already exists (Lmod trips over a non-existing entry in $MODULEPATH)
if install_path is not None:
modules_path = os.path.join(install_path, 'modules', 'all')
if not os.path.exists(modules_path):
os.makedirs(modules_path)
debug("Created path %s" % modules_path)
debug("Running EasyBuild with arguments '%s'" % ' '.join(eb_args))
sys.argv = eb_args
# install EasyBuild with EasyBuild
from easybuild.main import main as easybuild_main
easybuild_main()
def stage2(tmpdir, templates, install_path, distribute_egg_dir, sourcepath):
"""STAGE 2: install EasyBuild to temporary dir with EasyBuild from stage 1."""
print('\n')
info("+++ STAGE 2: installing EasyBuild in %s with EasyBuild from stage 1...\n" % install_path)
preinstallopts = ''
if distribute_egg_dir is not None:
# ensure that (latest) setuptools is installed as well alongside EasyBuild,
# since it is a required runtime dependency for recent vsc-base and EasyBuild versions
# this is necessary since we provide our own distribute installation during the bootstrap (cfr. stage0)
preinstallopts += "easy_install -U --prefix %(installdir)s setuptools && "
# vsc-install is a runtime dependency for the EasyBuild unit test suite,
# and is easily picked up from stage1 rather than being actually installed, so force it
vsc_install_tarball_paths = glob.glob(os.path.join(sourcepath, 'vsc-install*.tar.gz'))
if len(vsc_install_tarball_paths) == 1:
vsc_install = vsc_install_tarball_paths[0]
else:
vsc_install = 'vsc-install'
preinstallopts += "easy_install -U --prefix %%(installdir)s %s && " % vsc_install
templates.update({
'preinstallopts': preinstallopts,
})
# create easyconfig file
ebfile = os.path.join(tmpdir, 'EasyBuild-%s.eb' % templates['version'])
handle = open(ebfile, 'w')
templates.update({
'source_urls': '\n'.join(["'%s/%s/%s'," % (PYPI_SOURCE_URL, pkg[0], pkg) for pkg in EASYBUILD_PACKAGES]),
'sources': "%(vsc-install)s%(vsc-base)s%(easybuild-framework)s%(easybuild-easyblocks)s%(easybuild-easyconfigs)s" % templates,
'pythonpath': distribute_egg_dir,
})
handle.write(EASYBUILD_EASYCONFIG_TEMPLATE % templates)
handle.close()
# set command line arguments for eb
eb_args = ['eb', ebfile, '--allow-modules-tool-mismatch']
if print_debug:
eb_args.extend(['--debug', '--logtostdout'])
if forced_install:
info("Performing FORCED installation, as requested...")
eb_args.append('--force')
# make sure we don't leave any stuff behind in default path $HOME/.local/easybuild
# and set build and install path explicitely
if LooseVersion(templates['version']) < LooseVersion('1.3.0'):
os.environ['EASYBUILD_PREFIX'] = tmpdir
os.environ['EASYBUILD_BUILDPATH'] = tmpdir
if install_path is not None:
os.environ['EASYBUILD_INSTALLPATH'] = install_path
else:
# only for v1.3 and up
eb_args.append('--prefix=%s' % tmpdir)
eb_args.append('--buildpath=%s' % tmpdir)
if install_path is not None:
eb_args.append('--installpath=%s' % install_path)
if sourcepath is not None:
eb_args.append('--sourcepath=%s' % sourcepath)
# make sure parent modules path already exists (Lmod trips over a non-existing entry in $MODULEPATH)
if install_path is not None:
modules_path = os.path.join(install_path, 'modules', 'all')
if not os.path.exists(modules_path):
os.makedirs(modules_path)
debug("Created path %s" % modules_path)
debug("Running EasyBuild with arguments '%s'" % ' '.join(eb_args))
sys.argv = eb_args
# install EasyBuild with EasyBuild
from easybuild.main import main as easybuild_main
easybuild_main()
def stage2(tmpdir, templates, install_path, distribute_egg_dir, sourcepath):
"""STAGE 2: install EasyBuild to temporary dir with EasyBuild from stage 1."""
print('\n')
info("+++ STAGE 2: installing EasyBuild in %s with EasyBuild from stage 1...\n" % install_path)
preinstallopts = ''
if distribute_egg_dir is not None:
# inject path to distribute installed in stage 0 into $PYTHONPATH via preinstallopts
# other approaches are not reliable, since EasyBuildMeta easyblock unsets $PYTHONPATH;
# this is required for the easy_install from stage 0 to work
preinstallopts += "export PYTHONPATH=%s:$PYTHONPATH && " % distribute_egg_dir
# ensure that (latest) setuptools is installed as well alongside EasyBuild,
# since it is a required runtime dependency for recent vsc-base and EasyBuild versions
# this is necessary since we provide our own distribute installation during the bootstrap (cfr. stage0)
preinstallopts += "%s -m easy_install -U --prefix %%(installdir)s setuptools && " % sys.executable
# vsc-install is a runtime dependency for the EasyBuild unit test suite,
# and is easily picked up from stage1 rather than being actually installed, so force it
vsc_install = 'vsc-install'
if sourcepath:
vsc_install_tarball_paths = glob.glob(os.path.join(sourcepath, 'vsc-install*.tar.gz'))
if len(vsc_install_tarball_paths) == 1:
vsc_install = vsc_install_tarball_paths[0]
preinstallopts += "%s -m easy_install -U --prefix %%(installdir)s %s && " % (sys.executable, vsc_install)
templates.update({
'preinstallopts': preinstallopts,
})
# determine PyPI URLs for individual packages
pkg_urls = []
for pkg in EASYBUILD_PACKAGES:
# format of pkg entries in templates: "'<pkg_filename>',"
pkg_filename = templates[pkg][1:-2]
# the lines below implement a simplified version of the 'pypi_source_urls' and 'derive_alt_pypi_url' functions,
# which we can't leverage here, partially because of transitional changes in PyPI (#md5= -> #sha256=)
# determine download URL via PyPI's 'simple' API
pkg_simple = None
try:
pkg_simple = urllib2.urlopen('https://pypi.python.org/simple/%s' % pkg, timeout=10).read()
except (urllib2.URLError, urllib2.HTTPError) as err:
# failing to figure out the package download URl may be OK when source tarballs are provided
if sourcepath:
info("Ignoring failed attempt to determine '%s' download URL since source tarballs are provided" % pkg)
else:
raise err
if pkg_simple:
pkg_url_part_regex = re.compile('/(packages/[^#]+)/%s#' % pkg_filename)
res = pkg_url_part_regex.search(pkg_simple)
if res:
pkg_url_part = res.group(1)
else:
error("Failed to determine PyPI package URL for %s: %s\n" % (pkg, pkg_simple))
pkg_url = 'https://pypi.python.org/' + pkg_url_part
pkg_urls.append(pkg_url)
templates.update({
'source_urls': '\n'.join(["'%s'," % pkg_url for pkg_url in pkg_urls]),
'sources': "%(vsc-install)s%(vsc-base)s%(easybuild-framework)s%(easybuild-easyblocks)s%(easybuild-easyconfigs)s" % templates,
'pythonpath': distribute_egg_dir,
})
# create easyconfig file
ebfile = os.path.join(tmpdir, 'EasyBuild-%s.eb' % templates['version'])
handle = open(ebfile, 'w')
ebfile_txt = EASYBUILD_EASYCONFIG_TEMPLATE % templates
handle.write(ebfile_txt)
handle.close()
debug("Contents of generated easyconfig file:\n%s" % ebfile_txt)
# set command line arguments for eb
eb_args = ['eb', ebfile, '--allow-modules-tool-mismatch']
if print_debug:
eb_args.extend(['--debug', '--logtostdout'])
if forced_install:
info("Performing FORCED installation, as requested...")
eb_args.append('--force')
# make sure we don't leave any stuff behind in default path $HOME/.local/easybuild
# and set build and install path explicitely
if LooseVersion(templates['version']) < LooseVersion('1.3.0'):
os.environ['EASYBUILD_PREFIX'] = tmpdir
os.environ['EASYBUILD_BUILDPATH'] = tmpdir
if install_path is not None:
os.environ['EASYBUILD_INSTALLPATH'] = install_path
else:
# only for v1.3 and up
eb_args.append('--prefix=%s' % tmpdir)
eb_args.append('--buildpath=%s' % tmpdir)
if install_path is not None:
eb_args.append('--installpath=%s' % install_path)
if sourcepath is not None:
eb_args.append('--sourcepath=%s' % sourcepath)
# make sure parent modules path already exists (Lmod trips over a non-existing entry in $MODULEPATH)
if install_path is not None:
modules_path = det_modules_path(install_path)
if not os.path.exists(modules_path):
os.makedirs(modules_path)
debug("Created path %s" % modules_path)
debug("Running EasyBuild with arguments '%s'" % ' '.join(eb_args))
sys.argv = eb_args
# location to 'eb' command (from stage 1) may be expected to be included in $PATH
# it usually is there after stage1, unless 'prep' is called again with another location
# (only when stage 0 is not skipped)
# cfr. https://github.com/easybuilders/easybuild-framework/issues/2279
curr_path = [x for x in os.environ.get('PATH', '').split(os.pathsep) if len(x) > 0]
os.environ['PATH'] = os.pathsep.join([os.path.join(tmpdir, STAGE1_SUBDIR, 'bin')] + curr_path)
debug("$PATH: %s" % os.environ['PATH'])
# install EasyBuild with EasyBuild
from easybuild.main import main as easybuild_main
easybuild_main()
if print_debug:
os.environ['EASYBUILD_DEBUG'] = '1'
# make sure the EasyBuild module was actually installed
# EasyBuild configuration options that are picked up from configuration files/environment may break the bootstrap,
# for example by having $EASYBUILD_VERSION defined or via a configuration file specifies a value for 'stop'...
from easybuild.tools.config import build_option, install_path, get_module_syntax
from easybuild.framework.easyconfig.easyconfig import ActiveMNS
eb_spec = {
'name': 'EasyBuild',
'hidden': False,
'toolchain': {'name': 'dummy', 'version': 'dummy'},
'version': templates['version'],
'versionprefix': '',
'versionsuffix': '',
'moduleclass': 'tools',
}
mod_path = os.path.join(install_path('mod'), build_option('suffix_modules_path'))
debug("EasyBuild module should have been installed to %s" % mod_path)
eb_mod_name = ActiveMNS().det_full_module_name(eb_spec)
debug("EasyBuild module name: %s" % eb_mod_name)
eb_mod_path = os.path.join(mod_path, eb_mod_name)
if get_module_syntax() == 'Lua':
eb_mod_path += '.lua'
if os.path.exists(eb_mod_path):
info("EasyBuild module installed: %s" % eb_mod_path)
else:
error("EasyBuild module not found at %s, define $EASYBUILD_BOOTSTRAP_DEBUG to debug" % eb_mod_path)
def stage2(tmpdir, templates, install_path, distribute_egg_dir, sourcepath):
"""STAGE 2: install EasyBuild to temporary dir with EasyBuild from stage 1."""
print('\n')
info(
"+++ STAGE 2: installing EasyBuild in %s with EasyBuild from stage 1...\n"
% install_path)
preinstallopts = ''
if distribute_egg_dir is not None:
# inject path to distribute installed in stage 1 into $PYTHONPATH via preinstallopts
# other approaches are not reliable, since EasyBuildMeta easyblock unsets $PYTHONPATH;
# this is required for the easy_install from stage 1 to work
preinstallopts += "export PYTHONPATH=%s:$PYTHONPATH && " % distribute_egg_dir
# ensure that (latest) setuptools is installed as well alongside EasyBuild,
# since it is a required runtime dependency for recent vsc-base and EasyBuild versions
# this is necessary since we provide our own distribute installation during the bootstrap (cfr. stage0)
preinstallopts += "%s $(which easy_install) -U --prefix %%(installdir)s setuptools && " % sys.executable
# vsc-install is a runtime dependency for the EasyBuild unit test suite,
# and is easily picked up from stage1 rather than being actually installed, so force it
vsc_install = 'vsc-install'
if sourcepath:
vsc_install_tarball_paths = glob.glob(
os.path.join(sourcepath, 'vsc-install*.tar.gz'))
if len(vsc_install_tarball_paths) == 1:
vsc_install = vsc_install_tarball_paths[0]
preinstallopts += "%s $(which easy_install) -U --prefix %%(installdir)s %s && " % (
sys.executable, vsc_install)
templates.update({
'preinstallopts': preinstallopts,
})
# create easyconfig file
ebfile = os.path.join(tmpdir, 'EasyBuild-%s.eb' % templates['version'])
handle = open(ebfile, 'w')
templates.update({
'source_urls':
'\n'.join([
"'%s/%s/%s'," % (PYPI_SOURCE_URL, pkg[0], pkg)
for pkg in EASYBUILD_PACKAGES
]),
'sources':
"%(vsc-install)s%(vsc-base)s%(easybuild-framework)s%(easybuild-easyblocks)s%(easybuild-easyconfigs)s"
% templates,
'pythonpath':
distribute_egg_dir,
})
handle.write(EASYBUILD_EASYCONFIG_TEMPLATE % templates)
handle.close()
# set command line arguments for eb
eb_args = ['eb', ebfile, '--allow-modules-tool-mismatch']
if print_debug:
eb_args.extend(['--debug', '--logtostdout'])
if forced_install:
info("Performing FORCED installation, as requested...")
eb_args.append('--force')
# make sure we don't leave any stuff behind in default path $HOME/.local/easybuild
# and set build and install path explicitely
if LooseVersion(templates['version']) < LooseVersion('1.3.0'):
os.environ['EASYBUILD_PREFIX'] = tmpdir
os.environ['EASYBUILD_BUILDPATH'] = tmpdir
if install_path is not None:
os.environ['EASYBUILD_INSTALLPATH'] = install_path
else:
# only for v1.3 and up
eb_args.append('--prefix=%s' % tmpdir)
eb_args.append('--buildpath=%s' % tmpdir)
if install_path is not None:
eb_args.append('--installpath=%s' % install_path)
if sourcepath is not None:
eb_args.append('--sourcepath=%s' % sourcepath)
# make sure parent modules path already exists (Lmod trips over a non-existing entry in $MODULEPATH)
if install_path is not None:
modules_path = det_modules_path(install_path)
if not os.path.exists(modules_path):
os.makedirs(modules_path)
debug("Created path %s" % modules_path)
debug("Running EasyBuild with arguments '%s'" % ' '.join(eb_args))
sys.argv = eb_args
# location to 'eb' command (from stage 1) may be expected to be included in $PATH
# it usually is there after stage1, unless 'prep' is called again with another location
# (only when stage 0 is not skipped)
# cfr. https://github.com/easybuilders/easybuild-framework/issues/2279
curr_path = [
x for x in os.environ.get('PATH', '').split(os.pathsep) if len(x) > 0
]
os.environ['PATH'] = os.pathsep.join(
[os.path.join(tmpdir, STAGE1_SUBDIR, 'bin')] + curr_path)
debug("$PATH: %s" % os.environ['PATH'])
# install EasyBuild with EasyBuild
from easybuild.main import main as easybuild_main
easybuild_main()
# make sure the EasyBuild module was actually installed
# EasyBuild configuration options that are picked up from configuration files/environment may break the bootstrap,
# for example by having $EASYBUILD_VERSION defined or via a configuration file specifies a value for 'stop'...
from easybuild.tools.config import build_option, install_path, get_module_syntax
from easybuild.framework.easyconfig.easyconfig import ActiveMNS
eb_spec = {
'name': 'EasyBuild',
'hidden': False,
'toolchain': {
'name': 'dummy',
'version': 'dummy'
},
'version': templates['version'],
'versionprefix': '',
'versionsuffix': '',
'moduleclass': 'tools',
}
mod_path = os.path.join(install_path('mod'),
build_option('suffix_modules_path'))
debug("EasyBuild module should have been installed to %s" % mod_path)
eb_mod_name = ActiveMNS().det_full_module_name(eb_spec)
debug("EasyBuild module name: %s" % eb_mod_name)
eb_mod_path = os.path.join(mod_path, eb_mod_name)
if get_module_syntax() == 'Lua':
eb_mod_path += '.lua'
if os.path.exists(eb_mod_path):
info("EasyBuild module installed: %s" % eb_mod_path)
else:
error(
"EasyBuild module not found at %s, define $EASYBUILD_BOOTSTRAP_DEBUG to debug"
% eb_mod_path)
def stage2(tmpdir, templates, install_path, distribute_egg_dir, sourcepath):
"""STAGE 2: install EasyBuild to temporary dir with EasyBuild from stage 1."""
print('\n')
info("+++ STAGE 2: installing EasyBuild in %s with EasyBuild from stage 1...\n" % install_path)
preinstallopts = ''
if distribute_egg_dir is not None:
# inject path to distribute installed in stage 1 into $PYTHONPATH via preinstallopts
# other approaches are not reliable, since EasyBuildMeta easyblock unsets $PYTHONPATH;
# this is required for the easy_install from stage 1 to work
preinstallopts += "export PYTHONPATH=%s:$PYTHONPATH && " % distribute_egg_dir
# ensure that (latest) setuptools is installed as well alongside EasyBuild,
# since it is a required runtime dependency for recent vsc-base and EasyBuild versions
# this is necessary since we provide our own distribute installation during the bootstrap (cfr. stage0)
preinstallopts += "%s $(which easy_install) -U --prefix %%(installdir)s setuptools && " % sys.executable
# vsc-install is a runtime dependency for the EasyBuild unit test suite,
# and is easily picked up from stage1 rather than being actually installed, so force it
vsc_install = 'vsc-install'
if sourcepath:
vsc_install_tarball_paths = glob.glob(os.path.join(sourcepath, 'vsc-install*.tar.gz'))
if len(vsc_install_tarball_paths) == 1:
vsc_install = vsc_install_tarball_paths[0]
preinstallopts += "%s $(which easy_install) -U --prefix %%(installdir)s %s && " % (sys.executable, vsc_install)
templates.update({
'preinstallopts': preinstallopts,
})
# create easyconfig file
ebfile = os.path.join(tmpdir, 'EasyBuild-%s.eb' % templates['version'])
handle = open(ebfile, 'w')
templates.update({
'source_urls': '\n'.join(["'%s/%s/%s'," % (PYPI_SOURCE_URL, pkg[0], pkg) for pkg in EASYBUILD_PACKAGES]),
'sources': "%(vsc-install)s%(vsc-base)s%(easybuild-framework)s%(easybuild-easyblocks)s%(easybuild-easyconfigs)s" % templates,
'pythonpath': distribute_egg_dir,
})
handle.write(EASYBUILD_EASYCONFIG_TEMPLATE % templates)
handle.close()
# set command line arguments for eb
eb_args = ['eb', ebfile, '--allow-modules-tool-mismatch']
if print_debug:
eb_args.extend(['--debug', '--logtostdout'])
if forced_install:
info("Performing FORCED installation, as requested...")
eb_args.append('--force')
# make sure we don't leave any stuff behind in default path $HOME/.local/easybuild
# and set build and install path explicitely
if LooseVersion(templates['version']) < LooseVersion('1.3.0'):
os.environ['EASYBUILD_PREFIX'] = tmpdir
os.environ['EASYBUILD_BUILDPATH'] = tmpdir
if install_path is not None:
os.environ['EASYBUILD_INSTALLPATH'] = install_path
else:
# only for v1.3 and up
eb_args.append('--prefix=%s' % tmpdir)
eb_args.append('--buildpath=%s' % tmpdir)
if install_path is not None:
eb_args.append('--installpath=%s' % install_path)
if sourcepath is not None:
eb_args.append('--sourcepath=%s' % sourcepath)
# make sure parent modules path already exists (Lmod trips over a non-existing entry in $MODULEPATH)
if install_path is not None:
modules_path = det_modules_path(install_path)
if not os.path.exists(modules_path):
os.makedirs(modules_path)
debug("Created path %s" % modules_path)
debug("Running EasyBuild with arguments '%s'" % ' '.join(eb_args))
sys.argv = eb_args
# location to 'eb' command (from stage 1) may be expected to be included in $PATH
# it usually is there after stage1, unless 'prep' is called again with another location
# (only when stage 0 is not skipped)
# cfr. https://github.com/easybuilders/easybuild-framework/issues/2279
curr_path = [x for x in os.environ.get('PATH', '').split(os.pathsep) if len(x) > 0]
os.environ['PATH'] = os.pathsep.join([os.path.join(tmpdir, STAGE1_SUBDIR, 'bin')] + curr_path)
debug("$PATH: %s" % os.environ['PATH'])
# install EasyBuild with EasyBuild
from easybuild.main import main as easybuild_main
easybuild_main()
# make sure the EasyBuild module was actually installed
# EasyBuild configuration options that are picked up from configuration files/environment may break the bootstrap,
# for example by having $EASYBUILD_VERSION defined or via a configuration file specifies a value for 'stop'...
from easybuild.tools.config import build_option, install_path, get_module_syntax
from easybuild.framework.easyconfig.easyconfig import ActiveMNS
eb_spec = {
'name': 'EasyBuild',
'hidden': False,
'toolchain': {'name': 'dummy', 'version': 'dummy'},
'version': templates['version'],
'versionprefix': '',
'versionsuffix': '',
'moduleclass': 'tools',
}
mod_path = os.path.join(install_path('mod'), build_option('suffix_modules_path'))
debug("EasyBuild module should have been installed to %s" % mod_path)
eb_mod_name = ActiveMNS().det_full_module_name(eb_spec)
debug("EasyBuild module name: %s" % eb_mod_name)
eb_mod_path = os.path.join(mod_path, eb_mod_name)
if get_module_syntax() == 'Lua':
eb_mod_path += '.lua'
if os.path.exists(eb_mod_path):
info("EasyBuild module installed: %s" % eb_mod_path)
else:
error("EasyBuild module not found at %s, define $EASYBUILD_BOOTSTRAP_DEBUG to debug" % eb_mod_path)
def stage2(tmpdir, templates, install_path, distribute_egg_dir, sourcepath):
"""STAGE 2: install EasyBuild to temporary dir with EasyBuild from stage 1."""
info(
"\n\n+++ STAGE 2: installing EasyBuild in %s with EasyBuild from stage 1...\n\n"
% install_path)
# inject path to distribute installed in stage 1 into $PYTHONPATH via preinstallopts
# other approaches are not reliable, since EasyBuildMeta easyblock unsets $PYTHONPATH
if distribute_egg_dir is None:
preinstallopts = ''
else:
preinstallopts = 'PYTHONPATH=%s:$PYTHONPATH' % distribute_egg_dir
templates.update({
'preinstallopts': preinstallopts,
})
# create easyconfig file
ebfile = os.path.join(tmpdir, 'EasyBuild-%s.eb' % templates['version'])
f = open(ebfile, "w")
templates.update({
'source_urls':
'\n'.join([
"'%s/%s/%s'," % (PYPI_SOURCE_URL, pkg[0], pkg)
for pkg in EASYBUILD_PACKAGES
]),
'sources':
"%(vsc-base)s%(easybuild-framework)s%(easybuild-easyblocks)s%(easybuild-easyconfigs)s"
% templates,
'pythonpath':
distribute_egg_dir,
})
f.write(EASYBUILD_EASYCONFIG_TEMPLATE % templates)
f.close()
# unset $MODULEPATH, we don't care about already installed modules
os.environ['MODULEPATH'] = ''
# set command line arguments for eb
eb_args = ['eb', ebfile, '--allow-modules-tool-mismatch']
if print_debug:
eb_args.extend(['--debug', '--logtostdout'])
# make sure we don't leave any stuff behind in default path $HOME/.local/easybuild
# and set build and install path explicitely
if LooseVersion(templates['version']) < LooseVersion('1.3.0'):
os.environ['EASYBUILD_PREFIX'] = tmpdir
os.environ['EASYBUILD_BUILDPATH'] = tmpdir
if install_path is not None:
os.environ['EASYBUILD_INSTALLPATH'] = install_path
else:
# only for v1.3 and up
eb_args.append('--prefix=%s' % tmpdir)
eb_args.append('--buildpath=%s' % tmpdir)
if install_path is not None:
eb_args.append('--installpath=%s' % install_path)
if sourcepath is not None:
eb_args.append('--sourcepath=%s' % sourcepath)
# make sure parent modules path already exists (Lmod trips over a non-existing entry in $MODULEPATH)
if install_path is not None:
modules_path = os.path.join(install_path, 'modules', 'all')
if not os.path.exists(modules_path):
os.makedirs(modules_path)
debug("Created path %s" % modules_path)
debug("Running EasyBuild with arguments '%s'" % ' '.join(eb_args))
sys.argv = eb_args
# install EasyBuild with EasyBuild
from easybuild.main import main as easybuild_main
easybuild_main()
