def setup_run_environment(self, env):
# Set the environment variables after copying tree
env.set('FSLDIR', self.prefix)
fslsetup = join_path(self.prefix, 'etc', 'fslconf', 'fsl.sh')
if os.path.isfile(fslsetup):
env.extend(EnvironmentModifications.from_sourcing_file(fslsetup))
def setup_environment(self, spack_env, run_env):
if not self.stage.source_path:
self.stage.fetch()
self.stage.expand_archive()
spack_env.set('FSLDIR', self.stage.source_path)
# Here, run-time environment variables are being set manually.
# Normally these would be added to the modulefile at build-time
# by sourcing fsl.sh, but incorrect paths were being set, pointing to
# the staging directory rather than the install directory.
run_env.set('FSLDIR', self.prefix)
run_env.set('FSLOUTPUTTYPE', 'NIFTI_GZ')
run_env.set('FSLMULTIFILEQUIT', 'TRUE')
run_env.set('FSLTCLSH', self.prefix.bin.fsltclsh)
run_env.set('FSLWISH', self.prefix.bin.fslwish)
run_env.set('FSLLOCKDIR', '')
run_env.set('FSLMACHINELIST', '')
run_env.set('FSLREMOTECALL', '')
run_env.set('FSLGECUDAQ', 'cuda.q')
run_env.prepend_path('PATH', self.prefix)
# Below is for sourcing purposes during building
fslsetup = join_path(self.stage.source_path, 'etc', 'fslconf',
'fsl.sh')
if os.path.isfile(fslsetup):
spack_env.extend(EnvironmentModifications.from_sourcing_file(
fslsetup))
def setup_environment(self, spack_env, run_env):
if not self.stage.source_path:
self.stage.fetch()
self.stage.expand_archive()
spack_env.set('FSLDIR', self.stage.source_path)
# Here, run-time environment variables are being set manually.
# Normally these would be added to the modulefile at build-time
# by sourcing fsl.sh, but incorrect paths were being set, pointing to
# the staging directory rather than the install directory.
run_env.set('FSLDIR', self.prefix)
run_env.set('FSLOUTPUTTYPE', 'NIFTI_GZ')
run_env.set('FSLMULTIFILEQUIT', 'TRUE')
run_env.set('FSLTCLSH', self.prefix.bin.fsltclsh)
run_env.set('FSLWISH', self.prefix.bin.fslwish)
run_env.set('FSLLOCKDIR', '')
run_env.set('FSLMACHINELIST', '')
run_env.set('FSLREMOTECALL', '')
run_env.set('FSLGECUDAQ', 'cuda.q')
run_env.prepend_path('PATH', self.prefix)
# Below is for sourcing purposes during building
fslsetup = join_path(self.stage.source_path, 'etc', 'fslconf',
'fsl.sh')
if os.path.isfile(fslsetup):
spack_env.extend(EnvironmentModifications.from_sourcing_file(
fslsetup))
def setup_run_environment(self, env):
try:
env.extend(EnvironmentModifications.from_sourcing_file(
join_path(self.spec.prefix, 'headas-config_spack.sh'), clean=True
))
except Exception as e:
msg = 'unexpected error when sourcing HEASOFT setup [{0}]'
tty.warn(msg.format(str(e)))
def test_source_files(files_to_be_sourced):
"""Tests the construction of a list of environment modifications that are
the result of sourcing a file.
"""
env = EnvironmentModifications()
for filename in files_to_be_sourced:
if filename.endswith('sourceme_parameters.sh'):
env.extend(
EnvironmentModifications.from_sourcing_file(
filename, 'intel64'))
else:
env.extend(EnvironmentModifications.from_sourcing_file(filename))
modifications = env.group_by_name()
# This is sensitive to the user's environment; can include
# spurious entries for things like PS1
#
# TODO: figure out how to make a bit more robust.
assert len(modifications) >= 5
# Set new variables
assert len(modifications['NEW_VAR']) == 1
assert isinstance(modifications['NEW_VAR'][0], SetEnv)
assert modifications['NEW_VAR'][0].value == 'new'
assert len(modifications['FOO']) == 1
assert isinstance(modifications['FOO'][0], SetEnv)
assert modifications['FOO'][0].value == 'intel64'
# Unset variables
assert len(modifications['EMPTY_PATH_LIST']) == 1
assert isinstance(modifications['EMPTY_PATH_LIST'][0], UnsetEnv)
# Modified variables
assert len(modifications['UNSET_ME']) == 1
assert isinstance(modifications['UNSET_ME'][0], SetEnv)
assert modifications['UNSET_ME'][0].value == 'overridden'
assert len(modifications['PATH_LIST']) == 3
assert isinstance(modifications['PATH_LIST'][0], RemovePath)
assert modifications['PATH_LIST'][0].value == '/path/third'
assert isinstance(modifications['PATH_LIST'][1], AppendPath)
assert modifications['PATH_LIST'][1].value == '/path/fourth'
assert isinstance(modifications['PATH_LIST'][2], PrependPath)
assert modifications['PATH_LIST'][2].value == '/path/first'
def test_blacklist_lmod_variables():
# Construct the list of environment modifications
file = os.path.join(datadir, 'sourceme_lmod.sh')
env = EnvironmentModifications.from_sourcing_file(file)
# Check that variables related to lmod are not in there
modifications = env.group_by_name()
assert not any(x.startswith('LMOD_') for x in modifications)
def setup_run_environment(self, env):
bashrc = self.prefix.etc.bashrc
try:
env.extend(EnvironmentModifications.from_sourcing_file(
bashrc, clean=True
))
except Exception as e:
msg = 'unexpected error when sourcing OpenFOAM bashrc [{0}]'
tty.warn(msg.format(str(e)))
def setup_run_environment(self, env):
"""Sets the run environment (post-installation).
The environment comes from running:
.. code-block:: console
$ . $WM_PROJECT_DIR/etc/bashrc
"""
bashrc = join_path(self.projectdir, 'etc', 'bashrc')
minimal = True
if os.path.isfile(bashrc):
# post-install: source the installed bashrc
try:
mods = EnvironmentModifications.from_sourcing_file(
bashrc,
clean=True, # Remove duplicate entries
blacklist=[ # Blacklist these
# Inadvertent changes
# -------------------
'PS1', # Leave untouched
'MANPATH', # Leave untouched
# Unneeded bits
# -------------
# 'FOAM_SETTINGS', # Do not use with modules
# 'FOAM_INST_DIR', # Old
# 'FOAM_(APP|ETC|SRC|SOLVERS|UTILITIES)',
# 'FOAM_TUTORIALS', # May be useful
# 'WM_OSTYPE', # Purely optional value
# Third-party cruft - only used for orig compilation
# -----------------
'[A-Z].*_ARCH_PATH',
# '(KAHIP|METIS|SCOTCH)_VERSION',
# User-specific
# -------------
'FOAM_RUN',
'(FOAM|WM)_.*USER_.*',
],
whitelist=[ # Whitelist these
'MPI_ARCH_PATH', # Can be required for compilation
])
env.extend(mods)
minimal = False
tty.info('OpenFOAM bashrc env: {0}'.format(bashrc))
except Exception:
minimal = True
if minimal:
# pre-build or minimal environment
self.setup_minimal_environment(env)
def setup_run_environment(self, env):
"""Adds environment variables to the generated module file.
These environment variables come from running:
.. code-block:: console
$ source {prefix}/{component}/{version}/env/vars.sh
"""
env.extend(
EnvironmentModifications.from_sourcing_file(
join_path(self.component_path, 'env', 'vars.sh')))
def setup_run_environment(self, env):
"""Adds environment variables to the generated module file.
These environment variables come from running:
.. code-block:: console
$ source {prefix}/setvars.sh --force
"""
env.extend(
EnvironmentModifications.from_sourcing_file(
join(self.prefix, self.component_dir, 'latest/env/vars.sh')))
def setup_build_environment(self, env):
if not self.stage.source_path:
self.stage.fetch()
self.stage.expand_archive()
env.set('FSLDIR', self.stage.source_path)
# Below is for sourcing purposes during building
fslsetup = join_path(self.stage.source_path, 'etc', 'fslconf',
'fsl.sh')
if os.path.isfile(fslsetup):
env.extend(EnvironmentModifications.from_sourcing_file(fslsetup))
def _read_environment_file(self, filename):
"""
Read and parse the environment file.
Given an environment file, we want to read it, split by semicolons
and new lines, and then parse down to the subset of SPACK_* variables.
We assume that all spack prefix variables are not secrets, and unlike
the install_manifest.json, we don't (at least to start) parse the values
to remove path prefixes specific to user systems.
"""
if not os.path.exists(filename):
return
mods = EnvironmentModifications.from_sourcing_file(filename)
env = {}
mods.apply_modifications(env)
return env
def setup_environment(self, spack_env, run_env):
"""Add environment variables to the generated module file.
These environment variables come from running:
.. code-block:: console
$ . $WM_PROJECT_DIR/etc/bashrc
"""
# NOTE: Spack runs setup_environment twice.
# 1) pre-build to set up the build environment
# 2) post-install to determine runtime environment variables
# The etc/bashrc is only available (with corrrect content)
# post-installation.
bashrc = join_path(self.projectdir, 'etc', 'bashrc')
minimal = True
if os.path.isfile(bashrc):
# post-install: source the installed bashrc
try:
mods = EnvironmentModifications.from_sourcing_file(
bashrc,
clean=True, # Remove duplicate entries
blacklist=[ # Blacklist these
# Inadvertent changes
# -------------------
'PS1', # Leave unaffected
'MANPATH', # Leave unaffected
# Unneeded bits
# -------------
# 'FOAM_SETTINGS', # Do not use with modules
# 'FOAM_INST_DIR', # Old
# 'FOAM_(APP|ETC|SRC|SOLVERS|UTILITIES)',
# 'FOAM_TUTORIALS', # can be useful
# 'WM_OSTYPE', # Purely optional value
# Third-party cruft - only used for orig compilation
# -----------------
'[A-Z].*_ARCH_PATH',
# '(KAHIP|METIS|SCOTCH)_VERSION',
# User-specific
# -------------
'FOAM_RUN',
'(FOAM|WM)_.*USER_.*',
],
whitelist=[ # Whitelist these
'MPI_ARCH_PATH', # Can be needed for compilation
])
run_env.extend(mods)
spack_env.extend(mods)
minimal = False
tty.info('OpenFOAM bashrc env: {0}'.format(bashrc))
except Exception:
minimal = True
if minimal:
# pre-build or minimal environment
tty.info('OpenFOAM minimal env {0}'.format(self.prefix))
run_env.set('FOAM_PROJECT_DIR', self.projectdir)
run_env.set('WM_PROJECT_DIR', self.projectdir)
spack_env.set('FOAM_PROJECT_DIR', self.projectdir)
spack_env.set('WM_PROJECT_DIR', self.projectdir)
for d in ['wmake', self.archbin]: # bin added automatically
run_env.prepend_path('PATH', join_path(self.projectdir, d))
spack_env.prepend_path('PATH', join_path(self.projectdir, d))
def setup_run_environment(self, env):
filename = self.prefix.etc.join('profile.d').join('conda.sh')
env.extend(EnvironmentModifications.from_sourcing_file(filename))
def setup_environment(self, spack_env, run_env):
"""Add environment variables to the generated module file.
These environment variables come from running:
.. code-block:: console
$ . $WM_PROJECT_DIR/etc/bashrc
"""
# NOTE: Spack runs setup_environment twice.
# 1) pre-build to set up the build environment
# 2) post-install to determine runtime environment variables
# The etc/bashrc is only available (with corrrect content)
# post-installation.
bashrc = join_path(self.projectdir, 'etc', 'bashrc')
minimal = True
if os.path.isfile(bashrc):
# post-install: source the installed bashrc
try:
mods = EnvironmentModifications.from_sourcing_file(
bashrc,
clean=True, # Remove duplicate entries
blacklist=[ # Blacklist these
# Inadvertent changes
# -------------------
'PS1', # Leave unaffected
'MANPATH', # Leave unaffected
# Unneeded bits
# -------------
'FOAM_INST_DIR', # Possibly incorrect
'FOAM_(APP|ETC|SRC|SOLVERS|UTILITIES)',
'FOAM_TEST_.*_DIR',
'WM_NCOMPPROCS',
# 'FOAM_TUTORIALS', # can be useful
# Lots of third-party cruft
# -------------------------
'[A-Z].*_(BIN|LIB|INCLUDE)_DIR',
'[A-Z].*_SYSTEM',
'WM_THIRD_PARTY_.*',
'(BISON|FLEX|CMAKE|ZLIB)_DIR',
'(METIS|PARMETIS|PARMGRIDGEN|SCOTCH)_DIR',
# User-specific
# -------------
'FOAM_RUN',
'(FOAM|WM)_.*USER_.*',
],
whitelist=[ # Whitelist these
'MPI_ARCH_PATH', # Can be needed for compilation
'PYTHON_BIN_DIR',
])
run_env.extend(mods)
minimal = False
tty.info('foam-extend env: {0}'.format(bashrc))
except Exception:
minimal = True
if minimal:
# pre-build or minimal environment
tty.info('foam-extend minimal env {0}'.format(self.prefix))
run_env.set('FOAM_INST_DIR', os.path.dirname(self.projectdir)),
run_env.set('FOAM_PROJECT_DIR', self.projectdir)
run_env.set('WM_PROJECT_DIR', self.projectdir)
for d in ['wmake', self.archbin]: # bin added automatically
run_env.prepend_path('PATH', join_path(self.projectdir, d))
def setup_environment(self, spack_env, run_env):
"""Add environment variables to the generated module file.
These environment variables come from running:
.. code-block:: console
$ . $WM_PROJECT_DIR/etc/bashrc
"""
# NOTE: Spack runs setup_environment twice.
# 1) pre-build to set up the build environment
# 2) post-install to determine runtime environment variables
# The etc/bashrc is only available (with corrrect content)
# post-installation.
bashrc = join_path(self.projectdir, 'etc', 'bashrc')
minimal = True
if os.path.isfile(bashrc):
# post-install: source the installed bashrc
try:
mods = EnvironmentModifications.from_sourcing_file(
bashrc,
clean=True, # Remove duplicate entries
blacklist=[ # Blacklist these
# Inadvertent changes
# -------------------
'PS1', # Leave unaffected
'MANPATH', # Leave unaffected
# Unneeded bits
# -------------
'FOAM_INST_DIR', # Possibly incorrect
'FOAM_(APP|ETC|SRC|SOLVERS|UTILITIES)',
'FOAM_TEST_.*_DIR',
'WM_NCOMPPROCS',
# 'FOAM_TUTORIALS', # can be useful
# Lots of third-party cruft
# -------------------------
'[A-Z].*_(BIN|LIB|INCLUDE)_DIR',
'[A-Z].*_SYSTEM',
'WM_THIRD_PARTY_.*',
'(BISON|FLEX|CMAKE|ZLIB)_DIR',
'(METIS|PARMETIS|PARMGRIDGEN|SCOTCH)_DIR',
# User-specific
# -------------
'FOAM_RUN',
'(FOAM|WM)_.*USER_.*',
],
whitelist=[ # Whitelist these
'MPI_ARCH_PATH', # Can be needed for compilation
'PYTHON_BIN_DIR',
])
run_env.extend(mods)
minimal = False
tty.info('foam-extend env: {0}'.format(bashrc))
except Exception:
minimal = True
if minimal:
# pre-build or minimal environment
tty.info('foam-extend minimal env {0}'.format(self.prefix))
run_env.set('FOAM_INST_DIR', os.path.dirname(self.projectdir)),
run_env.set('FOAM_PROJECT_DIR', self.projectdir)
run_env.set('WM_PROJECT_DIR', self.projectdir)
for d in ['wmake', self.archbin]: # bin added automatically
run_env.prepend_path('PATH', join_path(self.projectdir, d))
