def configure_step(self, cmd_prefix='', verbose=False, path=None):
"""
Configure step
- Make sure it is using the python installed by EB, set convenient variables to be used
by the configuration file, and set the proper download path. Also run a "sanity check"
to verify that the environment is in good shape
"""
# [Re]define the download directory and make sure that our python is used
self.log.info(
'Loading Python to get the correct paths to be used in %s' %
self.cfg['jhbuildrc_file'])
download_path = os.path.join(source_paths()[0], self.name[0].lower(),
self.name)
tarballdir = 'tarballdir = "%s"' % download_path
pythonpath = os.getenv('PYTHONPATH', None)
python_root = os.getenv('EBROOTPYTHON', None)
if python_root == None:
raise EasyBuildError("Python is not loaded")
try:
self.log.info('Appending PYTHONPATH and PYTHON definitions to %s' %
self.cfg['jhbuildrc_file'])
self.log.info('Appending "%s" to %s' %
(tarballdir, self.cfg['jhbuildrc_file']))
with open(self.cfg['jhbuildrc_file'], "a") as conf_file:
if pythonpath != None:
conf_file.write("addpath('PYTHONPATH', '%s')\n" %
pythonpath)
conf_file.write("os.environ['PYTHON'] = '%s/bin/python'\n" %
python_root)
conf_file.write(tarballdir + "\n")
except OSError, err:
raise EasyBuildError("Can't append to %s: %s",
self.cfg['jhbuildrc_file'], err)
def obtain_repo(self, url, force_download=False):
"""
Downloads the repo from the given url, then updates submodules
:param url: url of git repository
"""
srcpaths = source_paths()
if re.match(r"^(https?)://", url):
# URL detected, try and download it
reponame = url.split('/')[-1].split('.')[0]
# Try and find the repo in the sources directory first
foundfile = None
failedpaths = []
for path in srcpaths:
# create list of candidate filepaths
namepath = os.path.join(path, self.name)
letterpath = os.path.join(path, letter_dir_for(self.name),
self.name)
candidate_filepaths = [letterpath, namepath, path]
# look for repo in those locations
for cfp in candidate_filepaths:
fullpath = os.path.join(cfp, reponame)
if os.path.isdir(fullpath):
self.log.info("Found repo %s at %s", reponame,
fullpath)
foundfile = os.path.abspath(fullpath)
if force_download:
print_warning(
"Found file %s at %s, but re-downloading it anyway..."
% (reponame, foundfile))
foundfile = None
break
else:
failedpaths.append(fullpath)
if foundfile:
if self.dry_run:
self.dry_run_msg(" * %s found at %s", filename, foundfile)
return foundfile
else:
# Figure out where to clone the repo to
repopath = os.path.join(srcpaths[0], letter_dir_for(self.name),
self.name)
self.log.info("Creating path %s to clone repo to" % repopath)
mkdir(repopath, parents=True)
# Remove repo, if it has previously been downloaded (we must be forcing download)
fullpath = os.path.join(repopath, reponame)
if os.path.exists(fullpath):
rmtree2(fullpath)
try:
if self.download_repo(reponame, url, repopath):
return fullpath
except IOError, err:
raise EasyBuildError(
"Downloading file %s from url %s to %s failed: %s",
filename, url, fullpath, err)
def obtain_config_guess(download_source_path=None, search_source_paths=None):
"""
Locate or download an up-to-date config.guess
:param download_source_path: Path to download config.guess to
:param search_source_paths: Paths to search for config.guess
:return: Path to config.guess or None
"""
log = fancylogger.getLogger('obtain_config_guess')
eb_source_paths = source_paths()
if download_source_path is None:
download_source_path = eb_source_paths[0]
else:
log.deprecated("Specifying custom source path to download config.guess via 'download_source_path'", '5.0')
if search_source_paths is None:
search_source_paths = eb_source_paths
else:
log.deprecated("Specifying custom location to search for updated config.guess via 'search_source_paths'", '5.0')
config_guess = 'config.guess'
sourcepath_subdir = os.path.join('generic', 'eb_v%s' % EASYBLOCKS_VERSION, 'ConfigureMake')
config_guess_path = None
# check if config.guess has already been downloaded to source path
for path in search_source_paths:
cand_config_guess_path = os.path.join(path, sourcepath_subdir, config_guess)
if os.path.isfile(cand_config_guess_path) and check_config_guess(cand_config_guess_path):
force_download = build_option('force_download')
if force_download:
print_warning("Found file %s at %s, but re-downloading it anyway..."
% (config_guess, cand_config_guess_path))
else:
config_guess_path = cand_config_guess_path
log.info("Found %s at %s", config_guess, config_guess_path)
break
if not config_guess_path:
cand_config_guess_path = os.path.join(download_source_path, sourcepath_subdir, config_guess)
config_guess_url = CONFIG_GUESS_URL_STUB + CONFIG_GUESS_COMMIT_ID
if not download_file(config_guess, config_guess_url, cand_config_guess_path):
print_warning("Failed to download recent %s to %s", config_guess, cand_config_guess_path, log=log)
elif not check_config_guess(cand_config_guess_path):
print_warning("Verification failed for file %s, not using it!", cand_config_guess_path, log=log)
remove_file(cand_config_guess_path)
else:
config_guess_path = cand_config_guess_path
adjust_permissions(config_guess_path, stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH, add=True)
log.info("Verified %s at %s, using it if required", config_guess, config_guess_path)
return config_guess_path
def obtain_config_guess(self, download_source_path=None, search_source_paths=None):
"""
Locate or download an up-to-date config.guess for use with ConfigureMake
:param download_source_path: Path to download config.guess to
:param search_source_paths: Paths to search for config.guess
:return: Path to config.guess or None
"""
eb_source_paths = source_paths()
if download_source_path is None:
download_source_path = eb_source_paths[0]
if search_source_paths is None:
search_source_paths = eb_source_paths
config_guess = 'config.guess'
sourcepath_subdir = os.path.join('generic', 'eb_v%s' % EASYBLOCKS_VERSION, 'ConfigureMake')
config_guess_path = None
# check if config.guess has already been downloaded to source path
for path in eb_source_paths:
cand_config_guess_path = os.path.join(path, sourcepath_subdir, config_guess)
if os.path.isfile(cand_config_guess_path):
config_guess_path = cand_config_guess_path
self.log.info("Found recent %s at %s, using it if required", config_guess, config_guess_path)
break
# if not found, try to download it
if config_guess_path is None:
cand_config_guess_path = os.path.join(download_source_path, sourcepath_subdir, config_guess)
config_guess_url = CONFIG_GUESS_URL_STUB + CONFIG_GUESS_COMMIT_ID
downloaded_path = download_file(config_guess, config_guess_url, cand_config_guess_path)
if downloaded_path is not None:
# verify SHA256 checksum of download to avoid using a corrupted download
if verify_checksum(downloaded_path, CONFIG_GUESS_SHA256):
config_guess_path = downloaded_path
# add execute permissions
adjust_permissions(downloaded_path, stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH, add=True)
self.log.info("Downloaded recent %s to %s, using it if required", config_guess, config_guess_path)
else:
self.log.warning("Checksum failed for downloaded file %s, not using it!", downloaded_path)
remove_file(downloaded_path)
else:
self.log.warning("Failed to download recent %s to %s for use with ConfigureMake easyblock (if needed)",
config_guess, cand_config_guess_path)
return config_guess_path
def obtain_config_guess(self, download_source_path=None, search_source_paths=None):
"""
Locate or download an up-to-date config.guess for use with ConfigureMake
:param download_source_path: Path to download config.guess to
:param search_source_paths: Paths to search for config.guess
:return: Path to config.guess or None
"""
eb_source_paths = source_paths()
if download_source_path is None:
download_source_path = eb_source_paths[0]
if search_source_paths is None:
search_source_paths = eb_source_paths
config_guess = 'config.guess'
sourcepath_subdir = os.path.join('generic', 'eb_v%s' % EASYBLOCKS_VERSION, 'ConfigureMake')
config_guess_path = None
# check if config.guess has already been downloaded to source path
for path in eb_source_paths:
cand_config_guess_path = os.path.join(path, sourcepath_subdir, config_guess)
if os.path.isfile(cand_config_guess_path):
config_guess_path = cand_config_guess_path
self.log.info("Found recent %s at %s, using it if required", config_guess, config_guess_path)
break
# if not found, try to download it
if config_guess_path is None:
cand_config_guess_path = os.path.join(download_source_path, sourcepath_subdir, config_guess)
config_guess_url = CONFIG_GUESS_URL_STUB + CONFIG_GUESS_COMMIT_ID
downloaded_path = download_file(config_guess, config_guess_url, cand_config_guess_path)
if downloaded_path is not None:
# verify SHA256 checksum of download to avoid using a corrupted download
if verify_checksum(downloaded_path, CONFIG_GUESS_SHA256):
config_guess_path = downloaded_path
# add execute permissions
adjust_permissions(downloaded_path, stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH, add=True)
self.log.info("Downloaded recent %s to %s, using it if required", config_guess, config_guess_path)
else:
self.log.warning("Checksum failed for downloaded file %s, not using it!", downloaded_path)
remove_file(downloaded_path)
else:
self.log.warning("Failed to download recent %s to %s for use with ConfigureMake easyblock (if needed)",
config_guess, cand_config_guess_path)
return config_guess_path
def test_generaloption_config_file(self):
"""Test use of new-style configuration file."""
self.purge_environment()
config_file = os.path.join(self.tmpdir, 'testconfig.cfg')
testpath1 = os.path.join(self.tmpdir, 'test1')
testpath2 = os.path.join(self.tmpdir, 'testtwo')
# test with config file passed via command line
cfgtxt = '\n'.join([
'[config]',
'installpath = %s' % testpath2,
])
write_file(config_file, cfgtxt)
installpath_software = tempfile.mkdtemp(prefix='installpath-software')
args = [
'--configfiles', config_file,
'--debug',
'--buildpath', testpath1,
'--installpath-software', installpath_software,
]
options = init_config(args=args)
self.assertEqual(build_path(), testpath1) # via command line
self.assertEqual(source_paths(), [os.path.join(os.getenv('HOME'), '.local', 'easybuild', 'sources')]) # default
self.assertEqual(install_path(), installpath_software) # via cmdline arg
self.assertEqual(install_path('mod'), os.path.join(testpath2, 'modules')) # via config file
# copy test easyconfigs to easybuild/easyconfigs subdirectory of temp directory
# to check whether easyconfigs install path is auto-included in robot path
tmpdir = tempfile.mkdtemp(prefix='easybuild-easyconfigs-pkg-install-path')
mkdir(os.path.join(tmpdir, 'easybuild'), parents=True)
test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs')
shutil.copytree(test_ecs_dir, os.path.join(tmpdir, 'easybuild', 'easyconfigs'))
orig_sys_path = sys.path[:]
sys.path.insert(0, tmpdir) # prepend to give it preference over possible other installed easyconfigs pkgs
# test with config file passed via environment variable
# also test for existence of HOME and USER by adding paths to robot-paths
installpath_modules = tempfile.mkdtemp(prefix='installpath-modules')
cfgtxt = '\n'.join([
'[config]',
'buildpath = %s' % testpath1,
'sourcepath = %(DEFAULT_REPOSITORYPATH)s',
'repositorypath = %(DEFAULT_REPOSITORYPATH)s,somesubdir',
'robot-paths=/tmp/foo:%(sourcepath)s:%(HOME)s:/tmp/%(USER)s:%(DEFAULT_ROBOT_PATHS)s',
'installpath-modules=%s' % installpath_modules,
])
write_file(config_file, cfgtxt)
os.environ['EASYBUILD_CONFIGFILES'] = config_file
args = [
'--debug',
'--sourcepath', testpath2,
]
options = init_config(args=args)
topdir = os.path.join(os.getenv('HOME'), '.local', 'easybuild')
self.assertEqual(install_path(), os.path.join(topdir, 'software')) # default
self.assertEqual(install_path('mod'), installpath_modules), # via config file
self.assertEqual(source_paths(), [testpath2]) # via command line
self.assertEqual(build_path(), testpath1) # via config file
self.assertEqual(get_repositorypath(), [os.path.join(topdir, 'ebfiles_repo'), 'somesubdir']) # via config file
# hardcoded first entry
self.assertEqual(options.robot_paths[0], '/tmp/foo')
# resolved value for %(sourcepath)s template
self.assertEqual(options.robot_paths[1], os.path.join(os.getenv('HOME'), '.local', 'easybuild', 'ebfiles_repo'))
# resolved value for HOME constant
self.assertEqual(options.robot_paths[2], os.getenv('HOME'))
# resolved value that uses USER constant
self.assertEqual(options.robot_paths[3], os.path.join('/tmp', os.getenv('USER')))
# first path in DEFAULT_ROBOT_PATHS
self.assertEqual(options.robot_paths[4], os.path.join(tmpdir, 'easybuild', 'easyconfigs'))
testpath3 = os.path.join(self.tmpdir, 'testTHREE')
os.environ['EASYBUILD_SOURCEPATH'] = testpath2
args = [
'--debug',
'--installpath', testpath3,
]
options = init_config(args=args)
self.assertEqual(source_paths(), [testpath2]) # via environment variable $EASYBUILD_SOURCEPATHS
self.assertEqual(install_path(), os.path.join(testpath3, 'software')) # via command line
self.assertEqual(install_path('mod'), installpath_modules), # via config file
self.assertEqual(build_path(), testpath1) # via config file
del os.environ['EASYBUILD_CONFIGFILES']
sys.path[:] = orig_sys_path
def test_generaloption_config_file(self):
"""Test use of new-style configuration file."""
self.purge_environment()
config_file = os.path.join(self.tmpdir, 'testconfig.cfg')
testpath1 = os.path.join(self.tmpdir, 'test1')
testpath2 = os.path.join(self.tmpdir, 'testtwo')
# test with config file passed via command line
cfgtxt = '\n'.join([
'[config]',
'installpath = %s' % testpath2,
])
write_file(config_file, cfgtxt)
installpath_software = tempfile.mkdtemp(prefix='installpath-software')
args = [
'--configfiles',
config_file,
'--debug',
'--buildpath',
testpath1,
'--installpath-software',
installpath_software,
]
options = init_config(args=args)
self.assertEqual(build_path(), testpath1) # via command line
self.assertEqual(source_paths(), [
os.path.join(os.getenv('HOME'), '.local', 'easybuild', 'sources')
]) # default
self.assertEqual(install_path(),
installpath_software) # via cmdline arg
self.assertEqual(install_path('mod'),
os.path.join(testpath2, 'modules')) # via config file
# copy test easyconfigs to easybuild/easyconfigs subdirectory of temp directory
# to check whether easyconfigs install path is auto-included in robot path
tmpdir = tempfile.mkdtemp(
prefix='easybuild-easyconfigs-pkg-install-path')
mkdir(os.path.join(tmpdir, 'easybuild'), parents=True)
test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'easyconfigs')
shutil.copytree(test_ecs_dir,
os.path.join(tmpdir, 'easybuild', 'easyconfigs'))
orig_sys_path = sys.path[:]
sys.path.insert(
0, tmpdir
) # prepend to give it preference over possible other installed easyconfigs pkgs
# test with config file passed via environment variable
# also test for existence of HOME and USER by adding paths to robot-paths
installpath_modules = tempfile.mkdtemp(prefix='installpath-modules')
cfgtxt = '\n'.join([
'[config]',
'buildpath = %s' % testpath1,
'sourcepath = %(DEFAULT_REPOSITORYPATH)s',
'repositorypath = %(DEFAULT_REPOSITORYPATH)s,somesubdir',
'robot-paths=/tmp/foo:%(sourcepath)s:%(HOME)s:/tmp/%(USER)s:%(DEFAULT_ROBOT_PATHS)s',
'installpath-modules=%s' % installpath_modules,
])
write_file(config_file, cfgtxt)
os.environ['EASYBUILD_CONFIGFILES'] = config_file
args = [
'--debug',
'--sourcepath',
testpath2,
]
options = init_config(args=args)
topdir = os.path.join(os.getenv('HOME'), '.local', 'easybuild')
self.assertEqual(install_path(), os.path.join(topdir,
'software')) # default
self.assertEqual(install_path('mod'),
installpath_modules), # via config file
self.assertEqual(source_paths(), [testpath2]) # via command line
self.assertEqual(build_path(), testpath1) # via config file
self.assertEqual(get_repositorypath(),
[os.path.join(topdir, 'ebfiles_repo'), 'somesubdir'
]) # via config file
# hardcoded first entry
self.assertEqual(options.robot_paths[0], '/tmp/foo')
# resolved value for %(sourcepath)s template
self.assertEqual(
options.robot_paths[1],
os.path.join(os.getenv('HOME'), '.local', 'easybuild',
'ebfiles_repo'))
# resolved value for HOME constant
self.assertEqual(options.robot_paths[2], os.getenv('HOME'))
# resolved value that uses USER constant
self.assertEqual(options.robot_paths[3],
os.path.join('/tmp', os.getenv('USER')))
# first path in DEFAULT_ROBOT_PATHS
self.assertEqual(options.robot_paths[4],
os.path.join(tmpdir, 'easybuild', 'easyconfigs'))
testpath3 = os.path.join(self.tmpdir, 'testTHREE')
os.environ['EASYBUILD_SOURCEPATH'] = testpath2
args = [
'--debug',
'--installpath',
testpath3,
]
options = init_config(args=args)
self.assertEqual(
source_paths(),
[testpath2]) # via environment variable $EASYBUILD_SOURCEPATHS
self.assertEqual(install_path(),
os.path.join(testpath3,
'software')) # via command line
self.assertEqual(install_path('mod'),
installpath_modules), # via config file
self.assertEqual(build_path(), testpath1) # via config file
del os.environ['EASYBUILD_CONFIGFILES']
sys.path[:] = orig_sys_path
def test_generaloption_config_file(self):
"""Test use of new-style configuration file."""
self.purge_environment()
oldstyle_config_file = os.path.join(self.tmpdir, 'nooldconfig.py')
config_file = os.path.join(self.tmpdir, 'testconfig.cfg')
testpath1 = os.path.join(self.tmpdir, 'test1')
testpath2 = os.path.join(self.tmpdir, 'testtwo')
write_file(oldstyle_config_file, '')
# test with config file passed via command line
cfgtxt = '\n'.join([
'[config]',
'installpath = %s' % testpath2,
])
write_file(config_file, cfgtxt)
args = [
'--configfiles', config_file,
'--debug',
'--buildpath', testpath1,
]
options = init_config(args=args)
self.assertEqual(build_path(), testpath1) # via command line
self.assertEqual(source_paths(), [os.path.join(os.getenv('HOME'), '.local', 'easybuild', 'sources')]) # default
self.assertEqual(install_path(), os.path.join(testpath2, 'software')) # via config file
# test with config file passed via environment variable
cfgtxt = '\n'.join([
'[config]',
'buildpath = %s' % testpath1,
])
write_file(config_file, cfgtxt)
os.environ['EASYBUILD_CONFIGFILES'] = config_file
args = [
'--debug',
'--sourcepath', testpath2,
]
options = init_config(args=args)
self.assertEqual(install_path(), os.path.join(os.getenv('HOME'), '.local', 'easybuild', 'software')) # default
self.assertEqual(source_paths(), [testpath2]) # via command line
self.assertEqual(build_path(), testpath1) # via config file
testpath3 = os.path.join(self.tmpdir, 'testTHREE')
os.environ['EASYBUILD_SOURCEPATH'] = testpath2
args = [
'--debug',
'--installpath', testpath3,
]
options = init_config(args=args)
self.assertEqual(source_paths(), [testpath2]) # via environment variable $EASYBUILD_SOURCEPATHS
self.assertEqual(install_path(), os.path.join(testpath3, 'software')) # via command line
self.assertEqual(build_path(), testpath1) # via config file
del os.environ['EASYBUILD_CONFIGFILES']
def test_legacy_config_file(self):
"""Test finding/using legacy configuration files."""
self.purge_environment()
cfg_fn = self.configure(args=[])
self.assertTrue(cfg_fn.endswith('easybuild/easybuild_config.py'))
configtxt = """
build_path = '%(buildpath)s'
source_path = '%(sourcepath)s'
install_path = '%(installpath)s'
repository_path = '%(repopath)s'
repository = FileRepository(repository_path)
log_format = ('%(logdir)s', '%(logtmpl)s')
log_dir = '%(tmplogdir)s'
software_install_suffix = '%(softsuffix)s'
modules_install_suffix = '%(modsuffix)s'
"""
buildpath = os.path.join(self.tmpdir, 'my', 'test', 'build', 'path')
sourcepath = os.path.join(self.tmpdir, 'my', 'test', 'source', 'path')
installpath = os.path.join(self.tmpdir, 'my', 'test', 'install', 'path')
repopath = os.path.join(self.tmpdir, 'my', 'test', 'repo', 'path')
logdir = 'somedir'
logtmpl = 'test-eb-%(name)s%(version)s_date-%(date)s__time-%(time)s.log'
tmplogdir = os.path.join(self.tmpdir, 'my', 'test', 'tmplogdir')
softsuffix = 'myfavoritesoftware'
modsuffix = 'modulesgohere'
configdict = {
'buildpath': buildpath,
'sourcepath': sourcepath,
'installpath': installpath,
'repopath': repopath,
'logdir': logdir,
'logtmpl': logtmpl,
'tmplogdir': tmplogdir,
'softsuffix': softsuffix,
'modsuffix': modsuffix
}
# create user config file on default location
myconfigfile = os.path.join(self.tmpdir, '.easybuild', 'config.py')
if not os.path.exists(os.path.dirname(myconfigfile)):
os.makedirs(os.path.dirname(myconfigfile))
write_file(myconfigfile, configtxt % configdict)
# redefine home so we can test user config file on default location
home = os.environ.get('HOME', None)
os.environ['HOME'] = self.tmpdir
init_config()
cfg_fn = self.configure(args=[])
if home is not None:
os.environ['HOME'] = home
# check finding and use of config file
self.assertEqual(cfg_fn, myconfigfile)
self.assertEqual(build_path(), buildpath)
self.assertEqual(source_paths()[0], sourcepath)
self.assertEqual(install_path(), os.path.join(installpath, softsuffix))
self.assertEqual(install_path(typ='mod'), os.path.join(installpath, modsuffix))
repo = init_repository(get_repository(), get_repositorypath())
self.assertTrue(isinstance(repo, FileRepository))
self.assertEqual(repo.repo, repopath)
self.assertEqual(log_file_format(return_directory=True), logdir)
self.assertEqual(log_file_format(), logtmpl)
self.assertEqual(get_build_log_path(), tmplogdir)
# redefine config file entries for proper testing below
buildpath = os.path.join(self.tmpdir, 'my', 'custom', 'test', 'build', 'path')
sourcepath = os.path.join(self.tmpdir, 'my', 'custom', 'test', 'source', 'path')
installpath = os.path.join(self.tmpdir, 'my', 'custom', 'test', 'install', 'path')
repopath = os.path.join(self.tmpdir, 'my', 'custom', 'test', 'repo', 'path')
logdir = 'somedir_custom'
logtmpl = 'test-custom-eb-%(name)_%(date)s%(time)s__%(version)s.log'
tmplogdir = os.path.join(self.tmpdir, 'my', 'custom', 'test', 'tmplogdir')
softsuffix = 'myfavoritesoftware_custom'
modsuffix = 'modulesgohere_custom'
configdict = {
'buildpath': buildpath,
'sourcepath': sourcepath,
'installpath': installpath,
'repopath': repopath,
'logdir': logdir,
'logtmpl': logtmpl,
'tmplogdir': tmplogdir,
'softsuffix': softsuffix,
'modsuffix': modsuffix }
# create custom config file, and point to it
mycustomconfigfile = os.path.join(self.tmpdir, 'mycustomconfig.py')
if not os.path.exists(os.path.dirname(mycustomconfigfile)):
os.makedirs(os.path.dirname(mycustomconfigfile))
write_file(mycustomconfigfile, configtxt % configdict)
os.environ['EASYBUILDCONFIG'] = mycustomconfigfile
# reconfigure
init_config()
cfg_fn = self.configure(args=[])
# verify configuration
self.assertEqual(cfg_fn, mycustomconfigfile)
self.assertEqual(build_path(), buildpath)
self.assertEqual(source_paths()[0], sourcepath)
self.assertEqual(install_path(), os.path.join(installpath, softsuffix))
self.assertEqual(install_path(typ='mod'), os.path.join(installpath, modsuffix))
repo = init_repository(get_repository(), get_repositorypath())
self.assertTrue(isinstance(repo, FileRepository))
self.assertEqual(repo.repo, repopath)
self.assertEqual(log_file_format(return_directory=True), logdir)
self.assertEqual(log_file_format(), logtmpl)
self.assertEqual(get_build_log_path(), tmplogdir)
def test_legacy_env_vars(self):
"""Test legacy environment variables."""
self.purge_environment()
# build path
test_buildpath = os.path.join(self.tmpdir, 'build', 'path')
os.environ['EASYBUILDBUILDPATH'] = test_buildpath
self.configure(args=[])
self.assertEqual(build_path(), test_buildpath)
del os.environ['EASYBUILDBUILDPATH']
# source path(s)
test_sourcepaths = [
os.path.join(self.tmpdir, 'source', 'path'),
':'.join([
os.path.join(self.tmpdir, 'source', 'path1'),
os.path.join(self.tmpdir, 'source', 'path2'),
]),
':'.join([
os.path.join(self.tmpdir, 'source', 'path1'),
os.path.join(self.tmpdir, 'source', 'path2'),
os.path.join(self.tmpdir, 'source', 'path3'),
]),
]
for test_sourcepath in test_sourcepaths:
init_config()
os.environ['EASYBUILDSOURCEPATH'] = test_sourcepath
self.configure(args=[])
self.assertEqual(build_path(), os.path.join(os.path.expanduser('~'), '.local', 'easybuild',
DEFAULT_PATH_SUBDIRS['buildpath']))
self.assertEqual(source_paths(), test_sourcepath.split(':'))
del os.environ['EASYBUILDSOURCEPATH']
test_sourcepath = os.path.join(self.tmpdir, 'source', 'path')
# install path
init_config()
test_installpath = os.path.join(self.tmpdir, 'install', 'path')
os.environ['EASYBUILDINSTALLPATH'] = test_installpath
self.configure(args=[])
self.assertEqual(source_paths()[0], os.path.join(os.path.expanduser('~'), '.local', 'easybuild',
DEFAULT_PATH_SUBDIRS['sourcepath']))
self.assertEqual(install_path(), os.path.join(test_installpath, DEFAULT_PATH_SUBDIRS['subdir_software']))
self.assertEqual(install_path(typ='mod'), os.path.join(test_installpath,
DEFAULT_PATH_SUBDIRS['subdir_modules']))
del os.environ['EASYBUILDINSTALLPATH']
# prefix: should change build/install/source/repo paths
init_config()
test_prefixpath = os.path.join(self.tmpdir, 'prefix', 'path')
os.environ['EASYBUILDPREFIX'] = test_prefixpath
self.configure(args=[])
self.assertEqual(build_path(), os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['buildpath']))
self.assertEqual(source_paths()[0], os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['sourcepath']))
self.assertEqual(install_path(), os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['subdir_software']))
self.assertEqual(install_path(typ='mod'), os.path.join(test_prefixpath,
DEFAULT_PATH_SUBDIRS['subdir_modules']))
repo = init_repository(get_repository(), get_repositorypath())
self.assertTrue(isinstance(repo, FileRepository))
self.assertEqual(repo.repo, os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['repositorypath']))
# build/source/install path overrides prefix
init_config()
os.environ['EASYBUILDBUILDPATH'] = test_buildpath
self.configure(args=[])
self.assertEqual(build_path(), test_buildpath)
self.assertEqual(source_paths()[0], os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['sourcepath']))
self.assertEqual(install_path(), os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['subdir_software']))
self.assertEqual(install_path(typ='mod'), os.path.join(test_prefixpath,
DEFAULT_PATH_SUBDIRS['subdir_modules']))
repo = init_repository(get_repository(), get_repositorypath())
self.assertTrue(isinstance(repo, FileRepository))
self.assertEqual(repo.repo, os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['repositorypath']))
del os.environ['EASYBUILDBUILDPATH']
init_config()
os.environ['EASYBUILDSOURCEPATH'] = test_sourcepath
self.configure(args=[])
self.assertEqual(build_path(), os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['buildpath']))
self.assertEqual(source_paths()[0], test_sourcepath)
self.assertEqual(install_path(), os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['subdir_software']))
self.assertEqual(install_path(typ='mod'), os.path.join(test_prefixpath,
DEFAULT_PATH_SUBDIRS['subdir_modules']))
repo = init_repository(get_repository(), get_repositorypath())
self.assertTrue(isinstance(repo, FileRepository))
self.assertEqual(repo.repo, os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['repositorypath']))
del os.environ['EASYBUILDSOURCEPATH']
init_config()
os.environ['EASYBUILDINSTALLPATH'] = test_installpath
self.configure(args=[])
self.assertEqual(build_path(), os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['buildpath']))
self.assertEqual(source_paths()[0], os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['sourcepath']))
self.assertEqual(install_path(), os.path.join(test_installpath, DEFAULT_PATH_SUBDIRS['subdir_software']))
self.assertEqual(install_path(typ='mod'), os.path.join(test_installpath,
DEFAULT_PATH_SUBDIRS['subdir_modules']))
repo = init_repository(get_repository(), get_repositorypath())
self.assertTrue(isinstance(repo, FileRepository))
self.assertEqual(repo.repo, os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['repositorypath']))
def test_generaloption_config_file(self):
"""Test use of new-style configuration file."""
self.purge_environment()
oldstyle_config_file = os.path.join(self.tmpdir, 'nooldconfig.py')
config_file = os.path.join(self.tmpdir, 'testconfig.cfg')
testpath1 = os.path.join(self.tmpdir, 'test1')
testpath2 = os.path.join(self.tmpdir, 'testtwo')
write_file(oldstyle_config_file, '')
# test with config file passed via command line
cfgtxt = '\n'.join([
'[config]',
'installpath = %s' % testpath2,
])
write_file(config_file, cfgtxt)
args = [
'--configfiles',
config_file,
'--debug',
'--buildpath',
testpath1,
]
options = init_config(args=args)
self.assertEqual(build_path(), testpath1) # via command line
self.assertEqual(source_paths(), [
os.path.join(os.getenv('HOME'), '.local', 'easybuild', 'sources')
]) # default
self.assertEqual(install_path(),
os.path.join(testpath2,
'software')) # via config file
# test with config file passed via environment variable
cfgtxt = '\n'.join([
'[config]',
'buildpath = %s' % testpath1,
])
write_file(config_file, cfgtxt)
os.environ['EASYBUILD_CONFIGFILES'] = config_file
args = [
'--debug',
'--sourcepath',
testpath2,
]
options = init_config(args=args)
self.assertEqual(install_path(),
os.path.join(os.getenv('HOME'), '.local', 'easybuild',
'software')) # default
self.assertEqual(source_paths(), [testpath2]) # via command line
self.assertEqual(build_path(), testpath1) # via config file
testpath3 = os.path.join(self.tmpdir, 'testTHREE')
os.environ['EASYBUILD_SOURCEPATH'] = testpath2
args = [
'--debug',
'--installpath',
testpath3,
]
options = init_config(args=args)
self.assertEqual(
source_paths(),
[testpath2]) # via environment variable $EASYBUILD_SOURCEPATHS
self.assertEqual(install_path(),
os.path.join(testpath3,
'software')) # via command line
self.assertEqual(build_path(), testpath1) # via config file
del os.environ['EASYBUILD_CONFIGFILES']
def test_legacy_config_file(self):
"""Test finding/using legacy configuration files."""
self.purge_environment()
cfg_fn = self.configure(args=[])
self.assertTrue(cfg_fn.endswith('easybuild/easybuild_config.py'))
configtxt = """
build_path = '%(buildpath)s'
source_path = '%(sourcepath)s'
install_path = '%(installpath)s'
repository_path = '%(repopath)s'
repository = FileRepository(repository_path)
log_format = ('%(logdir)s', '%(logtmpl)s')
log_dir = '%(tmplogdir)s'
software_install_suffix = '%(softsuffix)s'
modules_install_suffix = '%(modsuffix)s'
"""
buildpath = os.path.join(self.tmpdir, 'my', 'test', 'build', 'path')
sourcepath = os.path.join(self.tmpdir, 'my', 'test', 'source', 'path')
installpath = os.path.join(self.tmpdir, 'my', 'test', 'install',
'path')
repopath = os.path.join(self.tmpdir, 'my', 'test', 'repo', 'path')
logdir = 'somedir'
logtmpl = 'test-eb-%(name)s%(version)s_date-%(date)s__time-%(time)s.log'
tmplogdir = os.path.join(self.tmpdir, 'my', 'test', 'tmplogdir')
softsuffix = 'myfavoritesoftware'
modsuffix = 'modulesgohere'
configdict = {
'buildpath': buildpath,
'sourcepath': sourcepath,
'installpath': installpath,
'repopath': repopath,
'logdir': logdir,
'logtmpl': logtmpl,
'tmplogdir': tmplogdir,
'softsuffix': softsuffix,
'modsuffix': modsuffix
}
# create user config file on default location
myconfigfile = os.path.join(self.tmpdir, '.easybuild', 'config.py')
if not os.path.exists(os.path.dirname(myconfigfile)):
os.makedirs(os.path.dirname(myconfigfile))
write_file(myconfigfile, configtxt % configdict)
# redefine home so we can test user config file on default location
home = os.environ.get('HOME', None)
os.environ['HOME'] = self.tmpdir
init_config()
cfg_fn = self.configure(args=[])
if home is not None:
os.environ['HOME'] = home
# check finding and use of config file
self.assertEqual(cfg_fn, myconfigfile)
self.assertEqual(build_path(), buildpath)
self.assertEqual(source_paths()[0], sourcepath)
self.assertEqual(install_path(), os.path.join(installpath, softsuffix))
self.assertEqual(install_path(typ='mod'),
os.path.join(installpath, modsuffix))
repo = init_repository(get_repository(), get_repositorypath())
self.assertTrue(isinstance(repo, FileRepository))
self.assertEqual(repo.repo, repopath)
self.assertEqual(log_file_format(return_directory=True), logdir)
self.assertEqual(log_file_format(), logtmpl)
self.assertEqual(get_build_log_path(), tmplogdir)
# redefine config file entries for proper testing below
buildpath = os.path.join(self.tmpdir, 'my', 'custom', 'test', 'build',
'path')
sourcepath = os.path.join(self.tmpdir, 'my', 'custom', 'test',
'source', 'path')
installpath = os.path.join(self.tmpdir, 'my', 'custom', 'test',
'install', 'path')
repopath = os.path.join(self.tmpdir, 'my', 'custom', 'test', 'repo',
'path')
logdir = 'somedir_custom'
logtmpl = 'test-custom-eb-%(name)_%(date)s%(time)s__%(version)s.log'
tmplogdir = os.path.join(self.tmpdir, 'my', 'custom', 'test',
'tmplogdir')
softsuffix = 'myfavoritesoftware_custom'
modsuffix = 'modulesgohere_custom'
configdict = {
'buildpath': buildpath,
'sourcepath': sourcepath,
'installpath': installpath,
'repopath': repopath,
'logdir': logdir,
'logtmpl': logtmpl,
'tmplogdir': tmplogdir,
'softsuffix': softsuffix,
'modsuffix': modsuffix
}
# create custom config file, and point to it
mycustomconfigfile = os.path.join(self.tmpdir, 'mycustomconfig.py')
if not os.path.exists(os.path.dirname(mycustomconfigfile)):
os.makedirs(os.path.dirname(mycustomconfigfile))
write_file(mycustomconfigfile, configtxt % configdict)
os.environ['EASYBUILDCONFIG'] = mycustomconfigfile
# reconfigure
init_config()
cfg_fn = self.configure(args=[])
# verify configuration
self.assertEqual(cfg_fn, mycustomconfigfile)
self.assertEqual(build_path(), buildpath)
self.assertEqual(source_paths()[0], sourcepath)
self.assertEqual(install_path(), os.path.join(installpath, softsuffix))
self.assertEqual(install_path(typ='mod'),
os.path.join(installpath, modsuffix))
repo = init_repository(get_repository(), get_repositorypath())
self.assertTrue(isinstance(repo, FileRepository))
self.assertEqual(repo.repo, repopath)
self.assertEqual(log_file_format(return_directory=True), logdir)
self.assertEqual(log_file_format(), logtmpl)
self.assertEqual(get_build_log_path(), tmplogdir)
def test_legacy_env_vars(self):
"""Test legacy environment variables."""
self.purge_environment()
# build path
test_buildpath = os.path.join(self.tmpdir, 'build', 'path')
os.environ['EASYBUILDBUILDPATH'] = test_buildpath
self.configure(args=[])
self.assertEqual(build_path(), test_buildpath)
del os.environ['EASYBUILDBUILDPATH']
# source path(s)
test_sourcepaths = [
os.path.join(self.tmpdir, 'source', 'path'),
':'.join([
os.path.join(self.tmpdir, 'source', 'path1'),
os.path.join(self.tmpdir, 'source', 'path2'),
]),
':'.join([
os.path.join(self.tmpdir, 'source', 'path1'),
os.path.join(self.tmpdir, 'source', 'path2'),
os.path.join(self.tmpdir, 'source', 'path3'),
]),
]
for test_sourcepath in test_sourcepaths:
init_config()
os.environ['EASYBUILDSOURCEPATH'] = test_sourcepath
self.configure(args=[])
self.assertEqual(
build_path(),
os.path.join(os.path.expanduser('~'), '.local', 'easybuild',
DEFAULT_PATH_SUBDIRS['buildpath']))
self.assertEqual(source_paths(), test_sourcepath.split(':'))
del os.environ['EASYBUILDSOURCEPATH']
test_sourcepath = os.path.join(self.tmpdir, 'source', 'path')
# install path
init_config()
test_installpath = os.path.join(self.tmpdir, 'install', 'path')
os.environ['EASYBUILDINSTALLPATH'] = test_installpath
self.configure(args=[])
self.assertEqual(
source_paths()[0],
os.path.join(os.path.expanduser('~'), '.local', 'easybuild',
DEFAULT_PATH_SUBDIRS['sourcepath']))
self.assertEqual(
install_path(),
os.path.join(test_installpath,
DEFAULT_PATH_SUBDIRS['subdir_software']))
self.assertEqual(
install_path(typ='mod'),
os.path.join(test_installpath,
DEFAULT_PATH_SUBDIRS['subdir_modules']))
del os.environ['EASYBUILDINSTALLPATH']
# prefix: should change build/install/source/repo paths
init_config()
test_prefixpath = os.path.join(self.tmpdir, 'prefix', 'path')
os.environ['EASYBUILDPREFIX'] = test_prefixpath
self.configure(args=[])
self.assertEqual(
build_path(),
os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['buildpath']))
self.assertEqual(
source_paths()[0],
os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['sourcepath']))
self.assertEqual(
install_path(),
os.path.join(test_prefixpath,
DEFAULT_PATH_SUBDIRS['subdir_software']))
self.assertEqual(
install_path(typ='mod'),
os.path.join(test_prefixpath,
DEFAULT_PATH_SUBDIRS['subdir_modules']))
repo = init_repository(get_repository(), get_repositorypath())
self.assertTrue(isinstance(repo, FileRepository))
self.assertEqual(
repo.repo,
os.path.join(test_prefixpath,
DEFAULT_PATH_SUBDIRS['repositorypath']))
# build/source/install path overrides prefix
init_config()
os.environ['EASYBUILDBUILDPATH'] = test_buildpath
self.configure(args=[])
self.assertEqual(build_path(), test_buildpath)
self.assertEqual(
source_paths()[0],
os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['sourcepath']))
self.assertEqual(
install_path(),
os.path.join(test_prefixpath,
DEFAULT_PATH_SUBDIRS['subdir_software']))
self.assertEqual(
install_path(typ='mod'),
os.path.join(test_prefixpath,
DEFAULT_PATH_SUBDIRS['subdir_modules']))
repo = init_repository(get_repository(), get_repositorypath())
self.assertTrue(isinstance(repo, FileRepository))
self.assertEqual(
repo.repo,
os.path.join(test_prefixpath,
DEFAULT_PATH_SUBDIRS['repositorypath']))
del os.environ['EASYBUILDBUILDPATH']
init_config()
os.environ['EASYBUILDSOURCEPATH'] = test_sourcepath
self.configure(args=[])
self.assertEqual(
build_path(),
os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['buildpath']))
self.assertEqual(source_paths()[0], test_sourcepath)
self.assertEqual(
install_path(),
os.path.join(test_prefixpath,
DEFAULT_PATH_SUBDIRS['subdir_software']))
self.assertEqual(
install_path(typ='mod'),
os.path.join(test_prefixpath,
DEFAULT_PATH_SUBDIRS['subdir_modules']))
repo = init_repository(get_repository(), get_repositorypath())
self.assertTrue(isinstance(repo, FileRepository))
self.assertEqual(
repo.repo,
os.path.join(test_prefixpath,
DEFAULT_PATH_SUBDIRS['repositorypath']))
del os.environ['EASYBUILDSOURCEPATH']
init_config()
os.environ['EASYBUILDINSTALLPATH'] = test_installpath
self.configure(args=[])
self.assertEqual(
build_path(),
os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['buildpath']))
self.assertEqual(
source_paths()[0],
os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['sourcepath']))
self.assertEqual(
install_path(),
os.path.join(test_installpath,
DEFAULT_PATH_SUBDIRS['subdir_software']))
self.assertEqual(
install_path(typ='mod'),
os.path.join(test_installpath,
DEFAULT_PATH_SUBDIRS['subdir_modules']))
repo = init_repository(get_repository(), get_repositorypath())
self.assertTrue(isinstance(repo, FileRepository))
self.assertEqual(
repo.repo,
os.path.join(test_prefixpath,
DEFAULT_PATH_SUBDIRS['repositorypath']))
def test_generaloption_config_file(self):
"""Test use of new-style configuration file."""
self.purge_environment()
oldstyle_config_file = os.path.join(self.tmpdir, 'nooldconfig.py')
config_file = os.path.join(self.tmpdir, 'testconfig.cfg')
testpath1 = os.path.join(self.tmpdir, 'test1')
testpath2 = os.path.join(self.tmpdir, 'testtwo')
write_file(oldstyle_config_file, '')
# test with config file passed via command line
cfgtxt = '\n'.join([
'[config]',
'installpath = %s' % testpath2,
])
write_file(config_file, cfgtxt)
args = [
'--configfiles', config_file,
'--debug',
'--buildpath', testpath1,
]
options = init_config(args=args)
self.assertEqual(build_path(), testpath1) # via command line
self.assertEqual(source_paths(), [os.path.join(os.getenv('HOME'), '.local', 'easybuild', 'sources')]) # default
self.assertEqual(install_path(), os.path.join(testpath2, 'software')) # via config file
# copy test easyconfigs to easybuild/easyconfigs subdirectory of temp directory
# to check whether easyconfigs install path is auto-included in robot path
tmpdir = tempfile.mkdtemp(prefix='easybuild-easyconfigs-pkg-install-path')
mkdir(os.path.join(tmpdir, 'easybuild'), parents=True)
test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs')
shutil.copytree(test_ecs_dir, os.path.join(tmpdir, 'easybuild', 'easyconfigs'))
orig_sys_path = sys.path[:]
sys.path.insert(0, tmpdir) # prepend to give it preference over possible other installed easyconfigs pkgs
# test with config file passed via environment variable
cfgtxt = '\n'.join([
'[config]',
'buildpath = %s' % testpath1,
'robot-paths = /tmp/foo:%(DEFAULT_ROBOT_PATHS)s',
])
write_file(config_file, cfgtxt)
os.environ['EASYBUILD_CONFIGFILES'] = config_file
args = [
'--debug',
'--sourcepath', testpath2,
]
options = init_config(args=args)
self.assertEqual(install_path(), os.path.join(os.getenv('HOME'), '.local', 'easybuild', 'software')) # default
self.assertEqual(source_paths(), [testpath2]) # via command line
self.assertEqual(build_path(), testpath1) # via config file
self.assertTrue('/tmp/foo' in options.robot_paths)
self.assertTrue(os.path.join(tmpdir, 'easybuild', 'easyconfigs') in options.robot_paths)
testpath3 = os.path.join(self.tmpdir, 'testTHREE')
os.environ['EASYBUILD_SOURCEPATH'] = testpath2
args = [
'--debug',
'--installpath', testpath3,
]
options = init_config(args=args)
self.assertEqual(source_paths(), [testpath2]) # via environment variable $EASYBUILD_SOURCEPATHS
self.assertEqual(install_path(), os.path.join(testpath3, 'software')) # via command line
self.assertEqual(build_path(), testpath1) # via config file
del os.environ['EASYBUILD_CONFIGFILES']
sys.path[:] = orig_sys_path