def test_find_eb_script(self):
"""Test find_eb_script function."""
self.assertTrue(os.path.exists(ft.find_eb_script('rpath_args.py')))
self.assertTrue(
os.path.exists(ft.find_eb_script('rpath_wrapper_template.sh.in')))
self.assertErrorRegex(EasyBuildError,
"Script 'no_such_script' not found",
ft.find_eb_script, 'no_such_script')
def prepare_rpath_wrappers(self, rpath_filter_dirs=None):
"""
Put RPATH wrapper script in place for compiler and linker commands
:param rpath_filter_dirs: extra directories to include in RPATH filter (e.g. build dir, tmpdir, ...)
"""
self.log.experimental(
"Using wrapper scripts for compiler/linker commands that enforce RPATH linking"
)
if get_os_type() == LINUX:
self.log.info("Putting RPATH wrappers in place...")
else:
raise EasyBuildError(
"RPATH linking is currently only supported on Linux")
wrapper_dir = os.path.join(tempfile.mkdtemp(), RPATH_WRAPPERS_SUBDIR)
# must also wrap compilers commands, required e.g. for Clang ('gcc' on OS X)?
c_comps, fortran_comps = self.compilers()
rpath_args_py = find_eb_script('rpath_args.py')
rpath_wrapper_template = find_eb_script('rpath_wrapper_template.sh.in')
# prepend location to wrappers to $PATH
setvar('PATH', '%s:%s' % (wrapper_dir, os.getenv('PATH')))
# figure out list of patterns to use in rpath filter
rpath_filter = build_option('rpath_filter')
if rpath_filter is None:
rpath_filter = ['/lib.*', '/usr.*']
self.log.debug(
"No general RPATH filter specified, falling back to default: %s",
rpath_filter)
rpath_filter = ','.join(rpath_filter +
['%s.*' % d for d in rpath_filter_dirs or []])
self.log.debug("Combined RPATH filter: '%s'" % rpath_filter)
# create wrappers
for cmd in nub(c_comps + fortran_comps + ['ld', 'ld.gold']):
orig_cmd = which(cmd)
if orig_cmd:
# bail out early if command already is a wrapped;
# this may occur when building extensions
if self.is_rpath_wrapper(orig_cmd):
self.log.info(
"%s already seems to be an RPATH wrapper script, not wrapping it again!",
orig_cmd)
continue
cmd_wrapper = os.path.join(wrapper_dir, cmd)
# make *very* sure we don't wrap around ourselves and create a fork bomb...
if os.path.exists(cmd_wrapper) and os.path.exists(
orig_cmd) and os.path.samefile(orig_cmd, cmd_wrapper):
raise EasyBuildError(
"Refusing the create a fork bomb, which(%s) == %s",
cmd, orig_cmd)
# enable debug mode in wrapper script by specifying location for log file
if build_option('debug'):
rpath_wrapper_log = os.path.join(
tempfile.gettempdir(), 'rpath_wrapper_%s.log' % cmd)
else:
rpath_wrapper_log = '/dev/null'
# complete template script and put it in place
cmd_wrapper_txt = read_file(rpath_wrapper_template) % {
'orig_cmd': orig_cmd,
'python': sys.executable,
'rpath_args_py': rpath_args_py,
'rpath_filter': rpath_filter,
'rpath_wrapper_log': rpath_wrapper_log,
}
write_file(cmd_wrapper, cmd_wrapper_txt)
adjust_permissions(cmd_wrapper, stat.S_IXUSR)
self.log.info("Wrapper script for %s: %s (log: %s)", orig_cmd,
which(cmd), rpath_wrapper_log)
else:
self.log.debug(
"Not installing RPATH wrapper for non-existing command '%s'",
cmd)
def prepare_rpath_wrappers(self,
rpath_filter_dirs=None,
rpath_include_dirs=None):
"""
Put RPATH wrapper script in place for compiler and linker commands
:param rpath_filter_dirs: extra directories to include in RPATH filter (e.g. build dir, tmpdir, ...)
"""
if get_os_type() == LINUX:
self.log.info("Putting RPATH wrappers in place...")
else:
raise EasyBuildError(
"RPATH linking is currently only supported on Linux")
if rpath_filter_dirs is None:
rpath_filter_dirs = []
# always include filter for 'stubs' library directory,
# cfr. https://github.com/easybuilders/easybuild-framework/issues/2683
rpath_filter_dirs.append('.*/lib(64)?/stubs/?')
# directory where all wrappers will be placed
wrappers_dir = os.path.join(tempfile.mkdtemp(), RPATH_WRAPPERS_SUBDIR)
# must also wrap compilers commands, required e.g. for Clang ('gcc' on OS X)?
c_comps, fortran_comps = self.compilers()
rpath_args_py = find_eb_script('rpath_args.py')
rpath_wrapper_template = find_eb_script('rpath_wrapper_template.sh.in')
# figure out list of patterns to use in rpath filter
rpath_filter = build_option('rpath_filter')
if rpath_filter is None:
rpath_filter = ['/lib.*', '/usr.*']
self.log.debug(
"No general RPATH filter specified, falling back to default: %s",
rpath_filter)
rpath_filter = ','.join(rpath_filter +
['%s.*' % d for d in rpath_filter_dirs])
self.log.debug("Combined RPATH filter: '%s'", rpath_filter)
rpath_include = ','.join(rpath_include_dirs or [])
self.log.debug("Combined RPATH include paths: '%s'", rpath_include)
# create wrappers
for cmd in nub(c_comps + fortran_comps + ['ld', 'ld.gold', 'ld.bfd']):
orig_cmd = which(cmd)
if orig_cmd:
# bail out early if command already is a wrapped;
# this may occur when building extensions
if self.is_rpath_wrapper(orig_cmd):
self.log.info(
"%s already seems to be an RPATH wrapper script, not wrapping it again!",
orig_cmd)
continue
# determine location for this wrapper
# each wrapper is placed in its own subdirectory to enable $PATH filtering per wrapper separately
# avoid '+' character in directory name (for example with 'g++' command), which can cause trouble
# (see https://github.com/easybuilders/easybuild-easyconfigs/issues/7339)
wrapper_dir_name = '%s_wrapper' % cmd.replace('+', 'x')
wrapper_dir = os.path.join(wrappers_dir, wrapper_dir_name)
cmd_wrapper = os.path.join(wrapper_dir, cmd)
# make *very* sure we don't wrap around ourselves and create a fork bomb...
if os.path.exists(cmd_wrapper) and os.path.exists(
orig_cmd) and os.path.samefile(orig_cmd, cmd_wrapper):
raise EasyBuildError(
"Refusing the create a fork bomb, which(%s) == %s",
cmd, orig_cmd)
# enable debug mode in wrapper script by specifying location for log file
if build_option('debug'):
rpath_wrapper_log = os.path.join(
tempfile.gettempdir(), 'rpath_wrapper_%s.log' % cmd)
else:
rpath_wrapper_log = '/dev/null'
# complete template script and put it in place
cmd_wrapper_txt = read_file(rpath_wrapper_template) % {
'orig_cmd': orig_cmd,
'python': sys.executable,
'rpath_args_py': rpath_args_py,
'rpath_filter': rpath_filter,
'rpath_include': rpath_include,
'rpath_wrapper_log': rpath_wrapper_log,
'wrapper_dir': wrapper_dir,
}
write_file(cmd_wrapper, cmd_wrapper_txt)
adjust_permissions(cmd_wrapper, stat.S_IXUSR)
self.log.info("Wrapper script for %s: %s (log: %s)", orig_cmd,
which(cmd), rpath_wrapper_log)
# prepend location to this wrapper to $PATH
setvar('PATH', '%s:%s' % (wrapper_dir, os.getenv('PATH')))
else:
self.log.debug(
"Not installing RPATH wrapper for non-existing command '%s'",
cmd)
def test_rpath_args_script(self):
"""Test rpath_args.py script"""
script = find_eb_script('rpath_args.py')
# simplest possible compiler command
out, ec = run_cmd("%s gcc '' -c foo.c" % script, simple=False)
self.assertEqual(ec, 0)
cmd_args = [
"'-Wl,-rpath=$ORIGIN/../lib'",
"'-Wl,-rpath=$ORIGIN/../lib64'",
"'-Wl,--disable-new-dtags'",
"'-c'",
"'foo.c'",
]
self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args))
# linker command, --enable-new-dtags should be filtered out
out, ec = run_cmd("%s ld '' --enable-new-dtags foo.o" % script, simple=False)
self.assertEqual(ec, 0)
expected = '\n'.join([
"CMD_ARGS=('foo.o')",
"RPATH_ARGS='--disable-new-dtags -rpath=$ORIGIN/../lib -rpath=$ORIGIN/../lib64'",
''
])
cmd_args = [
"'-rpath=$ORIGIN/../lib'",
"'-rpath=$ORIGIN/../lib64'",
"'--disable-new-dtags'",
"'foo.o'",
]
self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args))
# test passing no arguments
out, ec = run_cmd("%s gcc ''" % script, simple=False)
self.assertEqual(ec, 0)
cmd_args = [
"'-Wl,-rpath=$ORIGIN/../lib'",
"'-Wl,-rpath=$ORIGIN/../lib64'",
"'-Wl,--disable-new-dtags'",
]
self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args))
# test passing a single empty argument
out, ec = run_cmd("%s ld.gold '' ''" % script, simple=False)
self.assertEqual(ec, 0)
cmd_args = [
"'-rpath=$ORIGIN/../lib'",
"'-rpath=$ORIGIN/../lib64'",
"'--disable-new-dtags'",
"''",
]
self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args))
# single -L argument
out, ec = run_cmd("%s gcc '' foo.c -L/foo -lfoo" % script, simple=False)
self.assertEqual(ec, 0)
cmd_args = [
"'-Wl,-rpath=$ORIGIN/../lib'",
"'-Wl,-rpath=$ORIGIN/../lib64'",
"'-Wl,--disable-new-dtags'",
"'-Wl,-rpath=/foo'",
"'foo.c'",
"'-L/foo'",
"'-lfoo'",
]
self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args))
# relative paths passed to -L are *not* RPATH'ed in
out, ec = run_cmd("%s gcc '' foo.c -L../lib -lfoo" % script, simple=False)
self.assertEqual(ec, 0)
cmd_args = [
"'-Wl,-rpath=$ORIGIN/../lib'",
"'-Wl,-rpath=$ORIGIN/../lib64'",
"'-Wl,--disable-new-dtags'",
"'foo.c'",
"'-L../lib'",
"'-lfoo'",
]
self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args))
# single -L argument, with value separated by a space
out, ec = run_cmd("%s gcc '' foo.c -L /foo -lfoo" % script, simple=False)
self.assertEqual(ec, 0)
cmd_args = [
"'-Wl,-rpath=$ORIGIN/../lib'",
"'-Wl,-rpath=$ORIGIN/../lib64'",
"'-Wl,--disable-new-dtags'",
"'-Wl,-rpath=/foo'",
"'foo.c'",
"'-L/foo'",
"'-lfoo'",
]
self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args))
# multiple -L arguments, order should be preserved
out, ec = run_cmd("%s ld '' -L/foo foo.o -L/lib64 -lfoo -lbar -L/usr/lib -L/bar" % script, simple=False)
self.assertEqual(ec, 0)
cmd_args = [
"'-rpath=$ORIGIN/../lib'",
"'-rpath=$ORIGIN/../lib64'",
"'--disable-new-dtags'",
"'-rpath=/foo'",
"'-rpath=/lib64'",
"'-rpath=/usr/lib'",
"'-rpath=/bar'",
"'-L/foo'",
"'foo.o'",
"'-L/lib64'",
"'-lfoo'",
"'-lbar'",
"'-L/usr/lib'",
"'-L/bar'",
]
self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args))
# test specifying of custom rpath filter
out, ec = run_cmd("%s ld '/fo.*,/bar.*' -L/foo foo.o -L/lib64 -lfoo -L/bar -lbar" % script, simple=False)
self.assertEqual(ec, 0)
cmd_args = [
"'-rpath=$ORIGIN/../lib'",
"'-rpath=$ORIGIN/../lib64'",
"'--disable-new-dtags'",
"'-rpath=/lib64'",
"'-L/foo'",
"'foo.o'",
"'-L/lib64'",
"'-lfoo'",
"'-L/bar'",
"'-lbar'",
]
self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args))
# slightly trimmed down real-life example (compilation of XZ)
args = ' '.join([
'-fvisibility=hidden',
'-Wall',
'-O2',
'-xHost',
'-o .libs/lzmainfo',
'lzmainfo-lzmainfo.o lzmainfo-tuklib_progname.o lzmainfo-tuklib_exit.o',
'-L/icc/lib/intel64',
'-L/imkl/lib',
'-L/imkl/mkl/lib/intel64',
'-L/gettext/lib',
'../../src/liblzma/.libs/liblzma.so',
'-lrt -liomp5 -lpthread',
'-Wl,-rpath',
'-Wl,/example/software/XZ/5.2.2-intel-2016b/lib',
])
out, ec = run_cmd("%s icc '' %s" % (script, args), simple=False)
self.assertEqual(ec, 0)
cmd_args = [
"'-Wl,-rpath=$ORIGIN/../lib'",
"'-Wl,-rpath=$ORIGIN/../lib64'",
"'-Wl,--disable-new-dtags'",
"'-Wl,-rpath=/icc/lib/intel64'",
"'-Wl,-rpath=/imkl/lib'",
"'-Wl,-rpath=/imkl/mkl/lib/intel64'",
"'-Wl,-rpath=/gettext/lib'",
"'-fvisibility=hidden'",
"'-Wall'",
"'-O2'",
"'-xHost'",
"'-o' '.libs/lzmainfo'",
"'lzmainfo-lzmainfo.o' 'lzmainfo-tuklib_progname.o' 'lzmainfo-tuklib_exit.o'",
"'-L/icc/lib/intel64'",
"'-L/imkl/lib'",
"'-L/imkl/mkl/lib/intel64'",
"'-L/gettext/lib'",
"'../../src/liblzma/.libs/liblzma.so'",
"'-lrt' '-liomp5' '-lpthread'",
"'-Wl,-rpath'",
"'-Wl,/example/software/XZ/5.2.2-intel-2016b/lib'",
]
self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args))
# trimmed down real-life example involving quotes and escaped quotes (compilation of GCC)
args = [
'-DHAVE_CONFIG_H',
'-I.',
'-Ibuild',
'-I../../gcc',
'-DBASEVER="\\"5.4.0\\""',
'-DDATESTAMP="\\"\\""',
'-DPKGVERSION="\\"(GCC) \\""',
'-DBUGURL="\\"<http://gcc.gnu.org/bugs.html>\\""',
'-o build/version.o',
'../../gcc/version.c',
]
cmd = "%s g++ '' %s" % (script, ' '.join(args))
out, ec = run_cmd(cmd, simple=False)
self.assertEqual(ec, 0)
cmd_args = [
"'-Wl,-rpath=$ORIGIN/../lib'",
"'-Wl,-rpath=$ORIGIN/../lib64'",
"'-Wl,--disable-new-dtags'",
"'-DHAVE_CONFIG_H'",
"'-I.'",
"'-Ibuild'",
"'-I../../gcc'",
"'-DBASEVER=\"5.4.0\"'",
"'-DDATESTAMP=\"\"'",
"'-DPKGVERSION=\"(GCC) \"'",
"'-DBUGURL=\"<http://gcc.gnu.org/bugs.html>\"'",
"'-o' 'build/version.o'",
"'../../gcc/version.c'",
]
self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args))
# verify that no -rpath arguments are injected when command is run in 'version check' mode
cmd = "%s g++ '' -v" % script
out, ec = run_cmd(cmd, simple=False)
self.assertEqual(ec, 0)
self.assertEqual(out.strip(), "CMD_ARGS=('-v')")
def test_find_eb_script(self):
"""Test find_eb_script function."""
self.assertTrue(os.path.exists(ft.find_eb_script('rpath_args.py')))
self.assertTrue(os.path.exists(ft.find_eb_script('rpath_wrapper_template.sh.in')))
self.assertErrorRegex(EasyBuildError, "Script 'no_such_script' not found", ft.find_eb_script, 'no_such_script')
def prepare_rpath_wrappers(self, rpath_filter_dirs=None):
"""
Put RPATH wrapper script in place for compiler and linker commands
:param rpath_filter_dirs: extra directories to include in RPATH filter (e.g. build dir, tmpdir, ...)
"""
self.log.experimental("Using wrapper scripts for compiler/linker commands that enforce RPATH linking")
if get_os_type() == LINUX:
self.log.info("Putting RPATH wrappers in place...")
else:
raise EasyBuildError("RPATH linking is currently only supported on Linux")
wrapper_dir = os.path.join(tempfile.mkdtemp(), RPATH_WRAPPERS_SUBDIR)
# must also wrap compilers commands, required e.g. for Clang ('gcc' on OS X)?
c_comps, fortran_comps = self.compilers()
rpath_args_py = find_eb_script('rpath_args.py')
rpath_wrapper_template = find_eb_script('rpath_wrapper_template.sh.in')
# prepend location to wrappers to $PATH
setvar('PATH', '%s:%s' % (wrapper_dir, os.getenv('PATH')))
# figure out list of patterns to use in rpath filter
rpath_filter = build_option('rpath_filter')
if rpath_filter is None:
rpath_filter = ['/lib.*', '/usr.*']
self.log.debug("No general RPATH filter specified, falling back to default: %s", rpath_filter)
rpath_filter = ','.join(rpath_filter + ['%s.*' % d for d in rpath_filter_dirs or []])
self.log.debug("Combined RPATH filter: '%s'" % rpath_filter)
# create wrappers
for cmd in nub(c_comps + fortran_comps + ['ld', 'ld.gold']):
orig_cmd = which(cmd)
if orig_cmd:
# bail out early if command already is a wrapped;
# this may occur when building extensions
if self.is_rpath_wrapper(orig_cmd):
self.log.info("%s already seems to be an RPATH wrapper script, not wrapping it again!", orig_cmd)
continue
cmd_wrapper = os.path.join(wrapper_dir, cmd)
# make *very* sure we don't wrap around ourselves and create a fork bomb...
if os.path.exists(cmd_wrapper) and os.path.exists(orig_cmd) and os.path.samefile(orig_cmd, cmd_wrapper):
raise EasyBuildError("Refusing the create a fork bomb, which(%s) == %s", cmd, orig_cmd)
# enable debug mode in wrapper script by specifying location for log file
if build_option('debug'):
rpath_wrapper_log = os.path.join(tempfile.gettempdir(), 'rpath_wrapper_%s.log' % cmd)
else:
rpath_wrapper_log = '/dev/null'
# complete template script and put it in place
cmd_wrapper_txt = read_file(rpath_wrapper_template) % {
'orig_cmd': orig_cmd,
'python': sys.executable,
'rpath_args_py': rpath_args_py,
'rpath_filter': rpath_filter,
'rpath_wrapper_log': rpath_wrapper_log,
}
write_file(cmd_wrapper, cmd_wrapper_txt)
adjust_permissions(cmd_wrapper, stat.S_IXUSR)
self.log.info("Wrapper script for %s: %s (log: %s)", orig_cmd, which(cmd), rpath_wrapper_log)
else:
self.log.debug("Not installing RPATH wrapper for non-existing command '%s'", cmd)
def prepare_rpath_wrappers(self, rpath_filter_dirs=None, rpath_include_dirs=None):
"""
Put RPATH wrapper script in place for compiler and linker commands
:param rpath_filter_dirs: extra directories to include in RPATH filter (e.g. build dir, tmpdir, ...)
"""
if get_os_type() == LINUX:
self.log.info("Putting RPATH wrappers in place...")
else:
raise EasyBuildError("RPATH linking is currently only supported on Linux")
if rpath_filter_dirs is None:
rpath_filter_dirs = []
# always include filter for 'stubs' library directory,
# cfr. https://github.com/easybuilders/easybuild-framework/issues/2683
rpath_filter_dirs.append('.*/lib(64)?/stubs/?')
# directory where all wrappers will be placed
wrappers_dir = os.path.join(tempfile.mkdtemp(), RPATH_WRAPPERS_SUBDIR)
# must also wrap compilers commands, required e.g. for Clang ('gcc' on OS X)?
c_comps, fortran_comps = self.compilers()
rpath_args_py = find_eb_script('rpath_args.py')
rpath_wrapper_template = find_eb_script('rpath_wrapper_template.sh.in')
# figure out list of patterns to use in rpath filter
rpath_filter = build_option('rpath_filter')
if rpath_filter is None:
rpath_filter = ['/lib.*', '/usr.*']
self.log.debug("No general RPATH filter specified, falling back to default: %s", rpath_filter)
rpath_filter = ','.join(rpath_filter + ['%s.*' % d for d in rpath_filter_dirs])
self.log.debug("Combined RPATH filter: '%s'", rpath_filter)
rpath_include = ','.join(rpath_include_dirs or [])
self.log.debug("Combined RPATH include paths: '%s'", rpath_include)
# create wrappers
for cmd in nub(c_comps + fortran_comps + ['ld', 'ld.gold', 'ld.bfd']):
orig_cmd = which(cmd)
if orig_cmd:
# bail out early if command already is a wrapped;
# this may occur when building extensions
if self.is_rpath_wrapper(orig_cmd):
self.log.info("%s already seems to be an RPATH wrapper script, not wrapping it again!", orig_cmd)
continue
# determine location for this wrapper
# each wrapper is placed in its own subdirectory to enable $PATH filtering per wrapper separately
# avoid '+' character in directory name (for example with 'g++' command), which can cause trouble
# (see https://github.com/easybuilders/easybuild-easyconfigs/issues/7339)
wrapper_dir_name = '%s_wrapper' % cmd.replace('+', 'x')
wrapper_dir = os.path.join(wrappers_dir, wrapper_dir_name)
cmd_wrapper = os.path.join(wrapper_dir, cmd)
# make *very* sure we don't wrap around ourselves and create a fork bomb...
if os.path.exists(cmd_wrapper) and os.path.exists(orig_cmd) and os.path.samefile(orig_cmd, cmd_wrapper):
raise EasyBuildError("Refusing the create a fork bomb, which(%s) == %s", cmd, orig_cmd)
# enable debug mode in wrapper script by specifying location for log file
if build_option('debug'):
rpath_wrapper_log = os.path.join(tempfile.gettempdir(), 'rpath_wrapper_%s.log' % cmd)
else:
rpath_wrapper_log = '/dev/null'
# complete template script and put it in place
cmd_wrapper_txt = read_file(rpath_wrapper_template) % {
'orig_cmd': orig_cmd,
'python': sys.executable,
'rpath_args_py': rpath_args_py,
'rpath_filter': rpath_filter,
'rpath_include': rpath_include,
'rpath_wrapper_log': rpath_wrapper_log,
'wrapper_dir': wrapper_dir,
}
write_file(cmd_wrapper, cmd_wrapper_txt)
adjust_permissions(cmd_wrapper, stat.S_IXUSR)
self.log.info("Wrapper script for %s: %s (log: %s)", orig_cmd, which(cmd), rpath_wrapper_log)
# prepend location to this wrapper to $PATH
setvar('PATH', '%s:%s' % (wrapper_dir, os.getenv('PATH')))
else:
self.log.debug("Not installing RPATH wrapper for non-existing command '%s'", cmd)