def runtime(self, _from='0'):
"""Generate the set of instructions to install the runtime specific
components from a build in a previous stage.
# Examples
```python
g = generic_cmake(...)
Stage0 += g
Stage1 += g.runtime()
```
"""
if self.prefix:
instructions = []
if self.__url:
instructions.append(comment(self.__url, reformat=False))
elif self.__repository:
instructions.append(comment(self.__repository, reformat=False))
instructions.append(
copy(_from=_from, src=self.prefix, dest=self.prefix))
return '\n'.join(str(x) for x in instructions)
else:
return
def runtime(self, _from='0'):
"""Generate the set of instructions to install the runtime specific
components from a build in a previous stage.
# Examples
```python
h = hdf5(...)
Stage0 += h
Stage1 += h.runtime()
```
"""
instructions = []
instructions.append(comment('HDF5'))
instructions.append(packages(ospackages=self.__runtime_ospackages))
instructions.append(copy(_from=_from, src=self.prefix,
dest=self.prefix))
if self.ldconfig:
instructions.append(shell(
commands=[self.ldcache_step(
directory=posixpath.join(self.prefix, 'lib'))]))
instructions.append(environment(variables=self.environment_step()))
return '\n'.join(str(x) for x in instructions)
def __str__(self):
"""String representation of the building block"""
scif_recipe = self.__scif_recipe()
if (self.__native
and hpccm.config.g_ctype == container_type.SINGULARITY):
# Take advantage of Singularity's native support for SCI-F.
return '\n'.join(str(x) for x in scif_recipe)
else:
# Generate an external SCI-F recipe file and manually call scif
# Temporarily switch container format to Singularity to write
# the SCI-F recipe file
preserved_ctype = hpccm.config.g_ctype
hpccm.config.set_container_format('singularity')
logging.info('Writing {}'.format(self.__scif_file))
with open(self.__scif_file, 'w') as f:
f.write('\n\n'.join(str(x) for x in scif_recipe))
# Restore original container format
hpccm.config.g_ctype = preserved_ctype
# Container instructions to copy the SCI-F recipe file
# into the container and then run scif
c_scif_file = posixpath.join('/scif/recipes',
os.path.basename(self.__scif_file))
instructions = []
instructions.append(comment('SCI-F "{}"'.format(self.__name)))
instructions.append(copy(src=self.__scif_file, dest=c_scif_file))
instructions.append(
shell(_arguments=self.__arguments,
chdir=False,
commands=['scif install {}'.format(c_scif_file)]))
return '\n'.join(str(x) for x in instructions)
def __str__(self):
"""String representation of the building block"""
instructions = []
if self.__directory:
instructions.append(comment('HDF5'))
else:
instructions.append(comment(
'HDF5 version {}'.format(self.__version)))
instructions.append(packages(ospackages=self.__ospackages))
if self.__directory:
# Use source from local build context
instructions.append(
copy(src=self.__directory,
dest=os.path.join(self.__wd, self.__directory)))
instructions.append(shell(commands=self.__commands))
if self.__environment_variables:
instructions.append(environment(
variables=self.__environment_variables))
return '\n'.join(str(x) for x in instructions)
def __str__(self):
"""String representation of the building block"""
instructions = []
instructions.append(
comment('PGI compiler version {}'.format(self.__version)))
if self.__tarball:
# Use tarball from local build context
instructions.append(
copy(src=self.__tarball,
dest=os.path.join(self.__wd,
os.path.basename(self.__tarball))))
else:
# Downloading, so need wget
self.__ospackages.append('wget')
if self.__ospackages:
instructions.append(packages(ospackages=self.__ospackages))
instructions.append(shell(commands=self.__commands))
instructions.append(environment(variables=self.__environment()))
return '\n'.join(str(x) for x in instructions)
def runtime(self, _from='0'):
"""Generate the set of instructions to install the runtime specific
components from a build in a previous stage.
# Example
```python
c = charm(...)
Stage0 += c
Stage1 += c.runtime()
```
"""
instructions = []
instructions.append(comment('Charm++'))
instructions.append(copy(_from=_from, src=self.__installdir,
dest=self.__installdir))
if self.ldconfig:
instructions.append(shell(
commands=[self.ldcache_step(
directory=posixpath.join(self.__prefix, 'lib_so'))]))
if self.__environment_variables:
instructions.append(environment(
variables=self.__environment_variables))
return '\n'.join(str(x) for x in instructions)
def runtime(self, _from='0'):
"""Generate the set of instructions to install the runtime specific
components from a build in a previous stage.
# Examples
```python
g = generic_autotools(...)
Stage0 += g
Stage1 += g.runtime()
```
"""
if self.prefix:
instructions = []
if self.__comment:
if self.url:
instructions.append(comment(self.url, reformat=False))
elif self.repository:
instructions.append(
comment(self.repository, reformat=False))
instructions.append(
copy(_from=_from, src=self.prefix, dest=self.prefix))
if self.ldconfig:
instructions.append(
shell(commands=[
self.ldcache_step(directory=posixpath.join(
self.prefix, self.__libdir))
]))
if self.runtime_environment_variables:
instructions.append(
environment(variables=self.environment_step(runtime=True)))
if self.annotate:
instructions.append(label(metadata=self.annotate_step()))
return '\n'.join(str(x) for x in instructions)
else: # pragma: no cover
return
def runtime(self, _from='0'):
"""Generate the set of instructions to install the runtime specific
components from a build in a previous stage.
# Examples
```python
p = pnetcdf(...)
Stage0 += p
Stage1 += p.runtime()
```
"""
instructions = []
instructions.append(comment('PnetCDF'))
instructions.append(copy(_from=_from, src=self.prefix,
dest=self.prefix))
if self.ldconfig:
instructions.append(shell(
commands=[self.ldcache_step(
directory=os.path.join(self.prefix, 'lib'))]))
if self.__environment_variables:
instructions.append(environment(
variables=self.__environment_variables))
return '\n'.join(str(x) for x in instructions)
def test_single_bash(self):
"""Single source file specified"""
c = copy(src='a', dest='b')
self.assertEqual(str(c), '')
def test_single_singularity(self):
"""Single source file specified"""
c = copy(src='a', dest='b')
self.assertEqual(str(c), '%files\n a b')
def test_single_docker(self):
"""Single source file specified"""
c = copy(src='a', dest='b')
self.assertEqual(str(c), 'COPY a b')
def test_mkdir_files_singularity(self):
"""mkdir folder with setup, multiple files"""
c = copy(src=['a', 'b'], dest='/opt/foo', _mkdir=True)
self.assertEqual(str(c),
'%setup\n mkdir -p ${SINGULARITY_ROOTFS}/opt/foo\n%files\n a /opt/foo\n b /opt/foo')
def runtime(self, _from='0'):
"""Generate the set of instructions to install the runtime specific
components from a build in a previous stage.
# Examples
```python
a = arm_allinea_compiler(...)
Stage0 += a
Stage1 += a.runtime()
```
"""
self.rt += comment('Arm Allinea Studio')
paths = []
# Redistributable libraries from redistributables.txt
# The allowed list of redistributable libraries does not
# include all Arm Allinea Studio libraries that get typically
# linked; consider using '-static-arm-libs'.
# OpenMP and Fortran runtime libraries
compiler_redist_path = posixpath.join(
self.__prefix,
'arm-linux-compiler-{0}_Generic-AArch64_{1}_aarch64-linux'.format(
self.__version, self.__directory_string), 'lib')
paths.append(compiler_redist_path)
self.rt += copy(_from=_from,
src=[
posixpath.join(compiler_redist_path, lib)
for lib in [
'libgomp.so', 'libiomp5.so', 'libomp.so',
'libflang.so', 'libflangrti.so'
]
],
dest=posixpath.join(compiler_redist_path, ''))
# Performance libraries
microarch_string = {
'generic': 'Generic-AArch64',
'generic-sve': 'Generic-SVE',
'neoverse-n1': 'Neoverse-N1',
'thunderx2t99': 'ThunderX2CN99'
}
for microarch in self.__microarchitectures:
armpl_arm_redist_path = posixpath.join(
self.__prefix,
'armpl-{0}.0_{1}_{2}_arm-linux-compiler_{0}_aarch64-linux'.
format(self.__version, microarch_string[microarch],
self.__directory_string), 'lib')
paths.append(armpl_arm_redist_path)
self.rt += copy(
_from=_from,
src=[
posixpath.join(armpl_arm_redist_path, lib) for lib in
['libamath.so', 'libamath_dummy.so', 'libastring.so']
],
dest=posixpath.join(armpl_arm_redist_path, ''))
armpl_gcc_redist_path = posixpath.join(
self.__prefix,
'armpl-{0}.0_{1}_{2}_gcc_{3}_aarch64-linux'.format(
self.__version, microarch_string[microarch],
self.__directory_string, self.__gcc_version), 'lib')
paths.append(armpl_gcc_redist_path)
self.rt += copy(
_from=_from,
src=[
posixpath.join(armpl_gcc_redist_path, lib) for lib in
['libamath.so', 'libamath_dummy.so', 'libastring.so']
],
dest=posixpath.join(armpl_gcc_redist_path, ''))
paths.append('$LD_LIBRARY_PATH') # tack on existing value at end
self.runtime_environment_variables['LD_LIBRARY_PATH'] = ':'.join(paths)
self.rt += environment(variables=self.environment_step(runtime=True))
return str(self.rt)
def test_from_docker(self):
"""Docker --from syntax"""
c = copy(src='a', dest='b', _from='dev')
self.assertEqual(str(c), 'COPY --from=dev a b')
def test_files_docker(self):
"""Pairs of files specified"""
c = copy(files={'a1': 'b1', 'a2': 'b2', 'a3': 'b3'})
self.assertEqual(str(c), 'COPY a1 b1\nCOPY a2 b2\nCOPY a3 b3')
def test_appfiles_multiple_singularity(self):
"""Multiple app-specific source files specified"""
c = copy(src=['a1', 'a2', 'a3'], dest='b', _app='foo')
self.assertEqual(str(c), '%appfiles foo\n a1 b\n a2 b\n a3 b')
def __init__(self, **kwargs):
"""Initialize building block"""
super(nvshmem, self).__init__(**kwargs)
self.__binary_tarball = kwargs.pop('binary_tarball', None)
self.__cuda = kwargs.pop('cuda', '/usr/local/cuda')
self.__gdrcopy = kwargs.pop('gdrcopy', None)
self.__hydra = kwargs.pop('hydra', False)
self.__make_variables = kwargs.pop('make_variables', {})
self.__mpi = kwargs.pop('mpi', None)
self.__ospackages = kwargs.pop('ospackages', ['make', 'wget'])
self.__prefix = kwargs.pop('prefix', '/usr/local/nvshmem')
self.__shmem = kwargs.pop('shmem', None)
self.__src_directory = kwargs.pop('src_directory', None)
self.__version = kwargs.pop('version', '1.0.1-0')
self.__wd = kwargs.get('wd', hpccm.config.g_wd) # working directory
# Set the download specific parameters
self.__download()
kwargs['url'] = self.url
if self.__src_directory:
kwargs['directory'] = self.__src_directory
# Setup the environment variables
self.environment_variables['CPATH'] = '{}:$CPATH'.format(
posixpath.join(self.__prefix, 'include'))
self.environment_variables['LIBRARY_PATH'] = '{}:$LIBRARY_PATH'.format(
posixpath.join(self.__prefix, 'lib'))
self.environment_variables['PATH'] = '{}:$PATH'.format(
posixpath.join(self.__prefix, 'bin'))
# Add packages
if self.__hydra:
self.__ospackages.append('automake')
if self.__version and not self.__binary_tarball and not self.package:
self += comment('NVSHMEM {}'.format(self.__version))
else:
self += comment('NVSHMEM')
self += packages(ospackages=self.__ospackages)
if self.__binary_tarball:
# Shorthand for the tarball file inside the container
tarball = posixpath.join(self.__wd,
os.path.basename(self.__binary_tarball))
self += copy(src=self.__binary_tarball, dest=tarball)
self += shell(commands=[
# Untar binary package
self.untar_step(
tarball=tarball,
# remove the leading directory, e.g., install in
# /usr/local/nvshmem not
# /usr/local/nvshmem/nvshmem_<version>_<arch>.
args=['--strip-components=1'],
directory=self.__prefix),
# Install Hydra process launcher
'{0}/scripts/install_hydra.sh {1} {0}'.
format(self.__prefix, self.__wd) if self.__hydra else None,
# Remove temporary files and cleanup
self.cleanup_step(items=[tarball])
])
self += environment(variables=self.environment_variables)
else:
# Build from source
# Set the build options
self.__configure()
self.__bb = generic_build(
build=[
'{} make -j$(nproc) install'.format(
self.__build_environment),
'./scripts/install_hydra.sh {1} {0}'.format(
self.__prefix, self.__wd) if self.__hydra else None
],
comment=False,
devel_environment=self.environment_variables,
prefix=self.__prefix,
runtime_environment=self.environment_variables,
**kwargs)
self += self.__bb
def test_post_multiple_singularity(self):
"""move file during post"""
c = copy(src=['a', 'b'], dest='/opt', _post=True)
self.assertEqual(str(c),
'%files\n a /\n b /\n%post\n mv /a /opt/a\n mv /b /opt/b')
def test_chown_singularity(self):
"""Docker --chown syntax"""
c = copy(_chown='alice:alice', src='foo', dest='bar')
self.assertEqual(str(c), '%files\n foo bar')
def test_chown_docker(self):
"""Docker --chown syntax"""
c = copy(_chown='alice:alice', src='foo', dest='bar')
self.assertEqual(str(c), 'COPY --chown=alice:alice foo bar')
def test_multiple_docker(self):
"""Multiple source files specified"""
c = copy(src=['a1', 'a2', 'a3'], dest='b')
self.assertEqual(str(c), 'COPY a1 \\\n a2 \\\n a3 \\\n b/')
def test_from_singularity32(self):
"""Singularity from syntax"""
c = copy(src='a', dest='b', _from='dev')
self.assertEqual(str(c), '%files from dev\n a b')
def test_multiple_singularity(self):
"""Multiple source files specified"""
c = copy(src=['a1', 'a2', 'a3'], dest='b')
self.assertEqual(str(c), '%files\n a1 b\n a2 b\n a3 b')
def test_appfiles_docker(self):
"""app-parameter is ignored in Docker"""
c = copy(src=['a1', 'a2', 'a3'], dest='b', _app='foo')
self.assertEqual(str(c), 'COPY a1 \\\n a2 \\\n a3 \\\n b/')
def test_files_singularity(self):
"""Pairs of files specified"""
c = copy(files={'a1': 'b1', 'a2': 'b2', 'a3': 'b3'})
self.assertEqual(str(c), '%files\n a1 b1\n a2 b2\n a3 b3')
def test_empty(self):
"""No source or destination specified"""
c = copy()
self.assertEqual(str(c), '')
def test_from_singularity(self):
"""Docker --from syntax"""
c = copy(src='a', dest='b', _from='dev')
self.assertEqual(str(c), '%files\n a b')
def test_invalid_ctype(self):
"""Invalid container type specified"""
c = copy(src='a', dest='b')
with self.assertRaises(RuntimeError):
str(c)
def test_appfiles_files_singularity(self):
"""Pairs of app-specific files specified"""
c = copy(files={'a1': 'b1', 'a2': 'b2', 'a3': 'b3'}, _app='foo')
self.assertEqual(str(c),
'%appfiles foo\n a1 b1\n a2 b2\n a3 b3')
def test_mkdir_file_singularity(self):
"""mkdir folder with setup, single file"""
c = copy(src='a', dest='/opt/foo/a', _mkdir=True)
self.assertEqual(str(c),
'%setup\n mkdir -p ${SINGULARITY_ROOTFS}/opt/foo\n%files\n a /opt/foo/a')