def _make_exe(tmpdir_factory, name, contents=None):
path = str(tmpdir_factory.mktemp('%s_exe' % name).join(name))
if contents is not None:
with open(path, 'w') as f:
f.write('#!/bin/sh\n%s\n' % contents)
set_executable(path)
return path
def write_spconfig(package, dirty, shebang):
# Set-up the environment
spack.build_environment.setup_package(package, dirty)
cmd = [str(which('cmake'))] + package.std_cmake_args + package.cmake_args()
env = dict()
paths = os.environ['PATH'].split(':')
paths = [item for item in paths if 'spack/env' not in item]
env['PATH'] = ':'.join(paths)
env['SPACK_TRANSITIVE_INCLUDE_PATH'] = spack_transitive_include_path()
env['CMAKE_PREFIX_PATH'] = os.environ['CMAKE_PREFIX_PATH']
env['CC'] = os.environ['SPACK_CC']
env['CXX'] = os.environ['SPACK_CXX']
env['FC'] = os.environ['SPACK_FC']
setup_fname = 'spconfig.py'
with open(setup_fname, 'w') as fout:
fout.write(
r"""#!%s
#
import sys
import os
import subprocess
def cmdlist(str):
return list(x.strip().replace("'",'') for x in str.split('\n') if x)
env = dict(os.environ)
""" % shebang)
env_vars = sorted(list(env.keys()))
for name in env_vars:
val = env[name]
if name.find('PATH') < 0:
fout.write('env[%s] = %s\n' % (repr(name), repr(val)))
else:
if name == 'SPACK_TRANSITIVE_INCLUDE_PATH':
sep = ';'
else:
sep = ':'
fout.write(
'env[%s] = "%s".join(cmdlist("""\n' % (repr(name), sep))
for part in val.split(sep):
fout.write(' %s\n' % part)
fout.write('"""))\n')
fout.write('\nif sys.argv[1:2] in [["--build"], ["-E"]]:\n')
fout.write(' cmd = ["%s"]\n' % cmd[0])
fout.write('else:\n')
fout.write(' cmd = cmdlist("""\n')
fout.write(' %s\n' % cmd[0])
for arg in cmd[1:]:
fout.write(' %s\n' % arg)
fout.write(' """)\n')
fout.write('cmd += sys.argv[1:]\n')
fout.write('\nproc = subprocess.Popen(cmd, env=env)\nproc.wait()\n')
set_executable(setup_fname)
def test_compiler_flags_use_real_version(working_env, monkeypatch, tmpdir):
# Create compiler
gcc = str(tmpdir.join('gcc'))
with open(gcc, 'w') as f:
f.write("""#!/bin/bash
echo "4.4.4"
""") # Version for which c++11 flag is -std=c++0x
fs.set_executable(gcc)
# Add compiler to config
compiler_info = {
'spec': 'gcc@foo',
'paths': {
'cc': gcc,
'cxx': None,
'f77': None,
'fc': None,
},
'flags': {},
'operating_system': 'fake',
'target': 'fake',
'modules': ['turn_on'],
'environment': {},
'extra_rpaths': [],
}
compiler_dict = {'compiler': compiler_info}
# Run and confirm output
compilers = spack.compilers.get_compilers([compiler_dict])
assert len(compilers) == 1
compiler = compilers[0]
flag = compiler.cxx11_flag
assert flag == '-std=c++0x'
def test_cray_frontend_compiler_detection(
compiler, version, tmpdir, monkeypatch, working_env
):
"""Test that the Cray frontend properly finds compilers form modules"""
# setup the fake compiler directory
compiler_dir = tmpdir.join(compiler)
compiler_exe = compiler_dir.join('cc').ensure()
fs.set_executable(str(compiler_exe))
# mock modules
def _module(cmd, *args):
module_name = '%s/%s' % (compiler, version)
module_contents = 'prepend-path PATH %s' % compiler_dir
if cmd == 'avail':
return module_name if compiler in args[0] else ''
if cmd == 'show':
return module_contents if module_name in args else ''
monkeypatch.setattr(spack.operating_systems.cray_frontend, 'module',
_module)
# remove PATH variable
os.environ.pop('PATH', None)
# get a CrayFrontend object
cray_fe_os = CrayFrontend()
paths = cray_fe_os.compiler_search_paths
assert paths == [str(compiler_dir)]
def make_mock_compiler():
"""Make a directory containing a fake, but detectable compiler."""
mock_compiler_dir = mkdtemp()
bin_dir = os.path.join(mock_compiler_dir, 'bin')
mkdirp(bin_dir)
gcc_path = os.path.join(bin_dir, 'gcc')
gxx_path = os.path.join(bin_dir, 'g++')
gfortran_path = os.path.join(bin_dir, 'gfortran')
with open(gcc_path, 'w') as f:
f.write("""\
#!/bin/sh
for arg in "$@"; do
if [ "$arg" = -dumpversion ]; then
echo '%s'
fi
done
""" % test_version)
# Create some mock compilers in the temporary directory
set_executable(gcc_path)
shutil.copy(gcc_path, gxx_path)
shutil.copy(gcc_path, gfortran_path)
return mock_compiler_dir
def test_read_unicode(tmpdir, working_env):
script_name = 'print_unicode.py'
# read the unicode back in and see whether things work
if is_windows:
script = ex.Executable('%s %s' % (sys.executable, script_name))
else:
script = ex.Executable('./%s' % script_name)
with tmpdir.as_cwd():
os.environ['LD_LIBRARY_PATH'] = spack.main.spack_ld_library_path
# make a script that prints some unicode
with open(script_name, 'w') as f:
f.write('''#!{0}
from __future__ import print_function
import sys
if sys.version_info < (3, 0, 0):
reload(sys)
sys.setdefaultencoding('utf8')
print(u'\\xc3')
'''.format(sys.executable))
# make it executable
fs.set_executable(script_name)
filter_shebangs_in_directory('.', [script_name])
assert u'\xc3' == script(output=str).strip()
def test_get_compiler_link_paths_load_env(working_env, monkeypatch, tmpdir):
gcc = str(tmpdir.join('gcc'))
with open(gcc, 'w') as f:
f.write("""#!/bin/bash
if [[ $ENV_SET == "1" && $MODULE_LOADED == "1" ]]; then
echo '""" + no_flag_output + """'
fi
""")
fs.set_executable(gcc)
# Set module load to turn compiler on
def module(*args):
if args[0] == 'show':
return ''
elif args[0] == 'load':
os.environ['MODULE_LOADED'] = "1"
monkeypatch.setattr(spack.util.module_cmd, 'module', module)
compiler = MockCompiler()
compiler.environment = {'set': {'ENV_SET': '1'}}
compiler.modules = ['turn_on']
dirs = compiler._get_compiler_link_paths([gcc])
assert dirs == no_flag_dirs
def test_compiler_get_real_version_fails(working_env, monkeypatch, tmpdir):
# Test variables
test_version = '2.2.2'
# Create compiler
gcc = str(tmpdir.join('gcc'))
with open(gcc, 'w') as f:
f.write("""#!/bin/bash
if [[ $CMP_ON == "1" ]]; then
echo "$CMP_VER"
fi
""")
fs.set_executable(gcc)
# Add compiler to config
compiler_info = {
'spec': 'gcc@foo',
'paths': {
'cc': gcc,
'cxx': None,
'f77': None,
'fc': None,
},
'flags': {},
'operating_system': 'fake',
'target': 'fake',
'modules': ['turn_on'],
'environment': {
'set': {'CMP_VER': test_version},
},
'extra_rpaths': [],
}
compiler_dict = {'compiler': compiler_info}
# Set module load to turn compiler on
def module(*args):
if args[0] == 'show':
return ''
elif args[0] == 'load':
os.environ['SPACK_TEST_CMP_ON'] = "1"
monkeypatch.setattr(spack.util.module_cmd, 'module', module)
# Make compiler fail when getting implicit rpaths
def _call(*args, **kwargs):
raise ProcessError("Failed intentionally")
monkeypatch.setattr(spack.util.executable.Executable, '__call__', _call)
# Run and no change to environment
compilers = spack.compilers.get_compilers([compiler_dict])
assert len(compilers) == 1
compiler = compilers[0]
try:
_ = compiler.get_real_version()
assert False
except ProcessError:
# Confirm environment does not change after failed call
assert 'SPACK_TEST_CMP_ON' not in os.environ
def write_spconfig(package):
# Set-up the environment
spack.build_environment.setup_package(package)
cmd = [str(which('cmake'))] + package.std_cmake_args + package.cmake_args()
env = dict()
paths = os.environ['PATH'].split(':')
paths = [item for item in paths if 'spack/env' not in item]
env['PATH'] = ':'.join(paths)
env['SPACK_TRANSITIVE_INCLUDE_PATH'] = spack_transitive_include_path()
env['CMAKE_PREFIX_PATH'] = os.environ['CMAKE_PREFIX_PATH']
env['CC'] = os.environ['SPACK_CC']
env['CXX'] = os.environ['SPACK_CXX']
env['FC'] = os.environ['SPACK_FC']
setup_fname = 'spconfig.py'
with open(setup_fname, 'w') as fout:
fout.write(
r"""#!%s
#
import sys
import os
import subprocess
def cmdlist(str):
return list(x.strip().replace("'",'') for x in str.split('\n') if x)
env = dict(os.environ)
""" % sys.executable)
env_vars = sorted(list(env.keys()))
for name in env_vars:
val = env[name]
if string.find(name, 'PATH') < 0:
fout.write('env[%s] = %s\n' % (repr(name), repr(val)))
else:
if name == 'SPACK_TRANSITIVE_INCLUDE_PATH':
sep = ';'
else:
sep = ':'
fout.write(
'env[%s] = "%s".join(cmdlist("""\n' % (repr(name), sep))
for part in string.split(val, sep):
fout.write(' %s\n' % part)
fout.write('"""))\n')
fout.write('\ncmd = cmdlist("""\n')
fout.write('%s\n' % cmd[0])
for arg in cmd[1:]:
fout.write(' %s\n' % arg)
fout.write('""") + sys.argv[1:]\n')
fout.write('\nproc = subprocess.Popen(cmd, env=env)\nproc.wait()\n')
set_executable(setup_fname)
def test_version_git_nonsense_output(tmpdir, working_env):
git = str(tmpdir.join("git"))
with open(git, "w") as f:
f.write("""#!/bin/sh
echo --|not a hash|----
""")
fs.set_executable(git)
os.environ["PATH"] = str(tmpdir)
assert spack.spack_version == get_version()
def test_get_version_bad_git(tmpdir, working_env):
bad_git = str(tmpdir.join("git"))
with open(bad_git, "w") as f:
f.write("""#!/bin/sh
exit 1
""")
fs.set_executable(bad_git)
os.environ["PATH"] = str(tmpdir)
assert spack.spack_version == get_version()
def test_get_version_no_match_git(tmpdir, working_env):
git = str(tmpdir.join("git"))
with open(git, "w") as f:
f.write("""#!/bin/sh
echo v0.13.3
""")
fs.set_executable(git)
os.environ["PATH"] = str(tmpdir)
assert spack.spack_version == get_version()
def test_get_version_match_git(tmpdir, working_env):
git = str(tmpdir.join("git"))
with open(git, "w") as f:
f.write("""#!/bin/sh
echo v0.13.3-912-g3519a1762
""")
fs.set_executable(git)
os.environ["PATH"] = str(tmpdir)
assert "0.13.3-912-3519a1762" == get_version()
def test_version_git_fails(tmpdir, working_env):
git = str(tmpdir.join("git"))
with open(git, "w") as f:
f.write("""#!/bin/sh
echo 26552533be04e83e66be2c28e0eb5011cb54e8fa
exit 1
""")
fs.set_executable(git)
os.environ["PATH"] = str(tmpdir)
assert spack.spack_version == get_version()
def test_which_relative_path_with_slash(tmpdir, working_env):
tmpdir.ensure('exe')
path = str(tmpdir.join('exe'))
os.environ['PATH'] = ''
with tmpdir.as_cwd():
no_exe = ex.which('./exe')
assert no_exe is None
fs.set_executable(path)
exe = ex.which('./exe')
assert exe.path == path
def test_git_sha_output(tmpdir, working_env):
git = str(tmpdir.join("git"))
sha = '26552533be04e83e66be2c28e0eb5011cb54e8fa'
with open(git, "w") as f:
f.write("""#!/bin/sh
echo {0}
""".format(sha))
fs.set_executable(git)
os.environ["PATH"] = str(tmpdir)
expected = "{0} ({1})".format(spack.spack_version, sha)
assert expected == get_version()
def test_which(tmpdir):
os.environ["PATH"] = str(tmpdir)
assert ex.which("spack-test-exe") is None
with pytest.raises(ex.CommandNotFoundError):
ex.which("spack-test-exe", required=True)
with tmpdir.as_cwd():
fs.touch("spack-test-exe")
fs.set_executable('spack-test-exe')
exe = ex.which("spack-test-exe")
assert exe is not None
assert exe.path == str(tmpdir.join("spack-test-exe"))
def test_compiler_get_real_version(working_env, monkeypatch, tmpdir):
# Test variables
test_version = '2.2.2'
# Create compiler
gcc = str(tmpdir.join('gcc'))
with open(gcc, 'w') as f:
f.write("""#!/bin/bash
if [[ $CMP_ON == "1" ]]; then
echo "$CMP_VER"
fi
""")
fs.set_executable(gcc)
# Add compiler to config
compiler_info = {
'spec': 'gcc@foo',
'paths': {
'cc': gcc,
'cxx': None,
'f77': None,
'fc': None,
},
'flags': {},
'operating_system': 'fake',
'target': 'fake',
'modules': ['turn_on'],
'environment': {
'set': {
'CMP_VER': test_version
},
},
'extra_rpaths': [],
}
compiler_dict = {'compiler': compiler_info}
# Set module load to turn compiler on
def module(*args):
if args[0] == 'show':
return ''
elif args[0] == 'load':
os.environ['CMP_ON'] = "1"
monkeypatch.setattr(spack.util.module_cmd, 'module', module)
# Run and confirm output
compilers = spack.compilers.get_compilers([compiler_dict])
assert len(compilers) == 1
compiler = compilers[0]
version = compiler.get_real_version()
assert version == test_version
def test_which_with_slash_ignores_path(tmpdir, working_env):
tmpdir.ensure('exe')
tmpdir.ensure('bin{0}exe'.format(os.path.sep))
path = str(tmpdir.join('exe'))
wrong_path = str(tmpdir.join('bin', 'exe'))
os.environ['PATH'] = os.path.dirname(wrong_path)
fs.set_executable(path)
fs.set_executable(wrong_path)
with tmpdir.as_cwd():
exe = ex.which('./exe')
assert exe.path == path
def test_find_gpg(cmd_name, version, tmpdir, mock_gnupghome, monkeypatch):
TEMPLATE = ('#!/bin/sh\n' 'echo "{version}"\n')
with tmpdir.as_cwd():
for fname in (cmd_name, 'gpgconf'):
with open(fname, 'w') as f:
f.write(TEMPLATE.format(version=version))
fs.set_executable(fname)
monkeypatch.setitem(os.environ, "PATH", str(tmpdir))
if version == 'undetectable' or version.endswith('1.3.4'):
with pytest.raises(spack.util.gpg.SpackGPGError):
spack.util.gpg.init(force=True)
else:
spack.util.gpg.init(force=True)
assert spack.util.gpg.GPG is not None
assert spack.util.gpg.GPGCONF is not None
def test_which_relative_path_with_slash(tmpdir, working_env):
tmpdir.ensure("exe")
path = str(tmpdir.join("exe"))
os.environ['PATH'] = ''
with tmpdir.as_cwd():
no_exe = ex.which('.{0}exe'.format(os.path.sep))
assert no_exe is None
if sys.platform == "win32":
# These checks are for 'executable' files, Windows
# determines this by file extension.
path += ".exe"
tmpdir.ensure('exe.exe')
else:
fs.set_executable(path)
exe = ex.which('.{0}exe'.format(os.path.sep))
assert exe.path == path
def test_find_gpg(cmd_name, version, tmpdir, mock_gnupghome, monkeypatch):
TEMPLATE = ('#!/bin/sh\n' 'echo "{version}"\n')
with tmpdir.as_cwd():
for fname in (cmd_name, 'gpgconf'):
with open(fname, 'w') as f:
f.write(TEMPLATE.format(version=version))
fs.set_executable(fname)
monkeypatch.setitem(os.environ, "PATH", str(tmpdir))
if version == 'undetectable' or version.endswith('1.3.4'):
with pytest.raises(spack.util.gpg.SpackGPGError):
spack.util.gpg.ensure_gpg(reevaluate=True)
else:
spack.util.gpg.ensure_gpg(reevaluate=True)
gpg_exe = spack.util.gpg.get_global_gpg_instance().gpg_exe
assert isinstance(gpg_exe, spack.util.executable.Executable)
gpgconf_exe = spack.util.gpg.get_global_gpg_instance().gpgconf_exe
assert isinstance(gpgconf_exe, spack.util.executable.Executable)
def test_sbang_with_specific_shebang(tmpdir, shebang, returncode, expected):
script = str(tmpdir.join("script"))
# write a script out with <shebang> on second line
with open(script, "w") as f:
f.write("#!/bin/sh {sbang}\n{shebang}\n".format(
sbang=spack.paths.sbang_script, shebang=shebang))
fs.set_executable(script)
# test running the script in debug, which prints what would be executed
exe = which(script)
out = exe(output=str, fail_on_error=False, env={"SBANG_DEBUG": "1"})
# check error status and output vs. expected
assert exe.returncode == returncode
if expected is not None:
expected += " " + script
assert expected == out.strip()
def test_which_with_slash_ignores_path(tmpdir, working_env):
tmpdir.ensure('exe')
tmpdir.ensure('bin{0}exe'.format(os.path.sep))
path = str(tmpdir.join('exe'))
wrong_path = str(tmpdir.join('bin', 'exe'))
os.environ['PATH'] = os.path.dirname(wrong_path)
with tmpdir.as_cwd():
if sys.platform == "win32":
# For Windows, need to create files with .exe after any assert is none tests
tmpdir.ensure('exe.exe')
tmpdir.ensure('bin{0}exe.exe'.format(os.path.sep))
path = path + ".exe"
wrong_path = wrong_path + ".exe"
else:
fs.set_executable(path)
fs.set_executable(wrong_path)
exe = ex.which('.{0}exe'.format(os.path.sep))
assert exe.path == path
def test_which(tmpdir):
os.environ["PATH"] = str(tmpdir)
assert ex.which("spack-test-exe") is None
with pytest.raises(ex.CommandNotFoundError):
ex.which("spack-test-exe", required=True)
path = str(tmpdir.join("spack-test-exe"))
with tmpdir.as_cwd():
if sys.platform == "win32":
# For Windows, need to create files with .exe after any assert is none tests
tmpdir.ensure("spack-test-exe.exe")
path += ".exe"
else:
fs.touch("spack-test-exe")
fs.set_executable("spack-test-exe")
exe = ex.which("spack-test-exe")
assert exe is not None
assert exe.path == path
def test_read_unicode(tmpdir):
script_name = 'print_unicode.py'
with tmpdir.as_cwd():
# make a script that prints some unicode
with open(script_name, 'w') as f:
f.write('''#!{0}
from __future__ import print_function
import sys
if sys.version_info < (3, 0, 0):
reload(sys)
sys.setdefaultencoding('utf8')
print(u'\\xc3')
'''.format(sys.executable))
# make it executable
fs.set_executable(script_name)
# read the unicode back in and see whether things work
script = ex.Executable('./%s' % script_name)
assert u'\xc3' == script(output=str).strip()
def _commands(parser, args):
"""This is the 'regular' command, which can be called multiple times.
See ``commands()`` below for ``--update-completion`` handling.
"""
formatter = formatters[args.format]
# check header first so we don't open out files unnecessarily
if args.header and not os.path.exists(args.header):
tty.die("No such file: '%s'" % args.header)
if args.update:
tty.msg('Updating file: %s' % args.update)
with open(args.update, 'w') as f:
prepend_header(args, f)
formatter(args, f)
if args.update_completion:
fs.set_executable(args.update)
else:
prepend_header(args, sys.stdout)
formatter(args, sys.stdout)
def _commands(parser, args):
"""This is the 'regular' command, which can be called multiple times.
See ``commands()`` below for ``--update-completion`` handling.
"""
formatter = formatters[args.format]
# check header first so we don't open out files unnecessarily
if args.header and not os.path.exists(args.header):
tty.die("No such file: '%s'" % args.header)
# if we're updating an existing file, only write output if a command
# or the header is newer than the file.
if args.update:
if os.path.exists(args.update):
files = [
spack.cmd.get_module(command).__file__.rstrip('c') # pyc -> py
for command in spack.cmd.all_commands()
]
if args.header:
files.append(args.header)
last_update = os.path.getmtime(args.update)
if not any(os.path.getmtime(f) > last_update for f in files):
tty.msg('File is up to date: %s' % args.update)
return
tty.msg('Updating file: %s' % args.update)
with open(args.update, 'w') as f:
prepend_header(args, f)
formatter(args, f)
if args.update_completion:
fs.set_executable(args.update)
else:
prepend_header(args, sys.stdout)
formatter(args, sys.stdout)
def prepare_executable(name):
real = str(tmpdir.join('cc').ensure())
fs.set_executable(real)
setattr(compiler, name, real)
