def test_merge_docker(self):
"""merge primitives"""
l = []
l.append(label(metadata={'ONE': 1, 'TWO': 2}))
l.append(label(metadata={'THREE': 3}))
merged = l[0].merge(l)
self.assertEqual(str(merged), '''LABEL ONE=1 \\
THREE=3 \\
TWO=2''')
l.append(label(metadata={'ONE': 'uno'}))
key_overwrite = l[0].merge(l)
self.assertEqual(str(key_overwrite), '''LABEL ONE=uno \\
THREE=3 \\
TWO=2''')
def test_applabel_multiple_singularity(self):
"""Multiple app-specific labels specified"""
l = label(metadata={'ONE': 1, 'TWO': 2, 'THREE': 3}, _app='foo')
self.assertEqual(str(l), '''%applabels foo
ONE 1
THREE 3
TWO 2''')
def runtime(self, _from='0'):
"""Generate the set of instructions to install the runtime specific
components from a build in a previous stage.
# Examples
```python
g = generic_build(...)
Stage0 += g
Stage1 += g.runtime()
```
"""
if self.__prefix:
instructions = []
if self.__comment:
if self.url:
instructions.append(comment(self.url, reformat=False))
elif self.repository:
instructions.append(comment(self.repository, reformat=False))
instructions.append(copy(_from=_from, src=self.__prefix,
dest=self.__prefix))
if self.ldconfig:
instructions.append(shell(commands=[self.ldcache_step(
directory=posixpath.join(self.__prefix, self.__libdir))]))
if self.runtime_environment_variables:
instructions.append(environment(
variables=self.environment_step(runtime=True)))
if self.annotate:
instructions.append(label(metadata=self.annotate_step()))
return '\n'.join(str(x) for x in instructions)
else: #pragma: no cover
return
def __instructions(self):
"""Fill in container instructions"""
# Mellanox OFED
for version in self.__mlnx_versions:
self += mlnx_ofed(annotate=False,
oslabel=self.__mlnx_oslabel,
packages=self.__mlnx_packages,
prefix=posixpath.join(self.__prefix, version),
symlink=self.__symlink,
version=version)
# Inbox OFED
if self.__inbox:
self += ofed(prefix=posixpath.join(self.__prefix, 'inbox'),
symlink=self.__symlink)
self += shell(commands=[
'ln -s {0} {1}'.format(posixpath.join(self.__prefix, 'inbox'),
posixpath.join(self.__prefix, '5.0-0'))
])
# Annotations
self.add_annotation('mlnx_versions', ', '.join(self.__mlnx_versions))
self.add_annotation('inbox', self.__inbox)
self += label(metadata=self.annotate_step())
def test_allsections_singularity(self):
"""One of each SCI-f section type"""
s = scif(name='foo')
s += copy(src='file', dest='/tmp/file')
s += comment('My app')
s += environment(variables={'ONE': '1'})
s += label(metadata={'A': 'B'})
s += runscript(commands=['default_program'])
s += shell(commands=['build_cmds'])
s += shell(commands=['test_program'], _test=True)
self.assertEqual(
str(s), r'''%appenv foo
export ONE=1
%appfiles foo
file /tmp/file
%apphelp foo
My app
%appinstall foo
for f in /.singularity.d/env/*; do . $f; done
build_cmds
%applabels foo
A B
%apprun foo
exec default_program "$@"
%apptest foo
test_program''')
def __str__(self):
"""String representation of the building block"""
instructions = [
comment(__doc__, reformat=False),
packages(ospackages=self.__ospackages),
packages(yum=['xz'], apt=['xz-utils']),
lmod(version='7.8.6'),
shell(commands=self.__commands),
environment(
variables={
'MODULEPATH':
'/opt/easybuild/modules/all:/home/easybuild/.local/'
'easybuild/modules/all:$MODULEPATH',
'FORCE_UNSAFE_CONFIGURE':
'1',
# https://github.com/docker-library/python/blob/edde349541e11f66dcc79cde1674317d065ddbdd/3.6/Dockerfile#L8
'LANG':
'C.UTF-8'
}),
label(metadata={'PACKAGE_MANAGER': 'easybuild'})
]
# Without the FORCE_UNSAFE_CONFIGURE env var some spack package
# installations may fail due to running as root.
return '\n'.join(str(x) for x in instructions)
def test_multiple_singularity(self):
"""Multiple labels specified"""
l = label(metadata={'ONE': 1, 'TWO': 2, 'THREE': 3})
self.assertEqual(str(l), '''%labels
ONE 1
THREE 3
TWO 2''')
def __instructions(self):
self += comment('OSU benchmarks version {}'.format(self.__version))
self += packages(ospackages=self.__ospackages)
self += shell(commands=self.__commands)
if self.__environment_variables:
self += environment(variables=self.__environment_variables)
if self.__labels:
self += label(metadata=self.__labels)
def test_merge_singularity(self):
"""merge primitives"""
l = []
l.append(label(metadata={'ONE': 1, 'TWO': 2}))
l.append(label(metadata={'THREE': 3}))
merged = l[0].merge(l)
self.assertEqual(str(merged), '''%labels
ONE 1
THREE 3
TWO 2''')
l.append(label(metadata={'ONE': 'uno'}))
key_overwrite = l[0].merge(l)
self.assertEqual(str(key_overwrite), '''%labels
ONE uno
THREE 3
TWO 2''')
def __instructions(self):
"""Fill in container instructions"""
if self.__comment:
if self.url:
self += comment(self.url, reformat=False)
elif self.repository:
self += comment(self.repository, reformat=False)
self += shell(_arguments=self.__run_arguments,
commands=self.__commands)
self += environment(variables=self.environment_step())
self += label(metadata=self.annotate_step())
def runtime(self, _from='0'):
instructions = []
instructions.append(
comment('OSU benchmarks version {}'.format(self.__version)))
instructions.append(
copy(_from=_from, src=self.prefix, dest=self.prefix))
if self.__environment_variables:
instructions.append(
environment(variables=self.__environment_variables))
if self.__labels:
instructions.append(label(metadata=self.__labels))
return '\n'.join(str(x) for x in instructions)
def runtime(self, _from='0'):
instructions = [
comment('OpenGeoSys build from repo {0}, branch {1}'.format(
self.__repo, self.__branch)),
copy(_from=_from, src=self.__prefix, dest=self.__prefix)
]
if self.__environment_variables:
instructions.append(
environment(variables=self.__environment_variables))
if self.__labels:
instructions.append(label(metadata=self.__labels))
return '\n'.join(str(x) for x in instructions)
def __instructions(self):
self += comment('OpenGeoSys build from repo {0}, branch {1}'.format(
self.__repo, self.__branch))
self += packages(ospackages=self.__ospackages)
self += shell(
commands=self.__commands,
_arguments='--mount=type=bind,target=/scif/apps/ogs/src,rw')
self += runscript(commands=['ogs'])
if self.__environment_variables:
self += environment(variables=self.__environment_variables)
if self.__labels:
self += label(metadata=self.__labels)
def __instructions(self):
self += comment(__doc__, reformat=False)
# https://github.com/bincrafters/community/issues/880
self += packages(ospackages=['pkg-config'])
# For building curl:
self += packages(ospackages=['autoconf-archive', 'libtool'])
conan_version = "1.22.2"
self += pip(pip='pip3', packages=['conan=={}'.format(conan_version)])
self += shell(commands=self.__commands)
if self.__user_home != '':
self += environment(
variables={'CONAN_USER_HOME': self.__user_home})
self += label(
metadata={
'org.opengeosys.pm': 'conan',
'org.opengeosys.pm.conan.version': conan_version
})
if self.__user_home != '':
self += label(metadata={
'org.opengeosys.pm.conan.user_home': self.__user_home
})
def runtime(self, _from='0'):
"""Generate the set of instructions to install the runtime specific
components from a build in a previous stage.
# Examples
```python
g = generic_build(...)
Stage0 += g
Stage1 += g.runtime()
```
"""
if self.__prefix:
if self.__comment:
if self.url:
self.rt += comment(self.url, reformat=False)
elif self.repository:
self.rt += comment(self.repository, reformat=False)
if self.__runtime:
for src in self.__runtime:
if '*' in posixpath.basename(src):
# When using COPY with more than one source file,
# the destination must be a directory and end with
# a /
dest = posixpath.dirname(src) + '/'
else:
dest = src
self.rt += copy(_from=_from, src=src, dest=dest)
else:
# Copy the entire prefix
self.rt += copy(_from=_from,
src=self.__prefix,
dest=self.__prefix)
if self.ldconfig:
self.rt += shell(commands=[
self.ldcache_step(
directory=posixpath.join(self.__prefix, self.__libdir))
])
if self.runtime_environment_variables:
self.rt += environment(variables=self.environment_step(
runtime=True))
if self.annotate:
self.rt += label(metadata=self.annotate_step())
return str(self.rt)
else: #pragma: no cover
return
def __instructions(self):
self += comment(__doc__, reformat=False)
self += packages(ospackages=['ccache'])
self += shell(commands=[
'mkdir -p {0} && chmod 777 {0}'.format(self.__cache_dir)
])
self += environment(
variables={
'CCACHE_DIR': self.__cache_dir,
'CCACHE_MAXSIZE': self.__cache_size,
'CCACHE_SLOPPINESS': 'pch_defines,time_macros'
})
self += label(metadata={
'ccache.dir': self.__cache_dir,
'ccache.size': self.__cache_size
})
def test_allsections_docker(self):
"""One of each SCI-f section type"""
# See comment in the test_defaults_docker test case
scif_file = tempfile.NamedTemporaryFile(delete=False, suffix='.scif')
s = scif(name='foo', file=scif_file.name)
s += copy(src='file', dest='/tmp/file')
s += comment('My app')
s += environment(variables={'ONE': '1'})
s += label(metadata={'A': 'B'})
s += runscript(commands=['default_program'])
s += shell(commands=['build_cmds'])
s += shell(commands=['test_program'], _test=True)
str(s) # Force writing the SCI-F recipe file
# slurp file content
with open(scif_file.name) as f:
content = f.read()
try:
os.unlink(scif_file.name)
except WindowsError:
# WindowsError: [Error 32] The process cannot access the file
# because it is being used by another process
pass
self.assertEqual(
content, r'''%appenv foo
export ONE=1
%appfiles foo
file /tmp/file
%apphelp foo
My app
%appinstall foo
build_cmds
%applabels foo
A B
%apprun foo
exec default_program "$@"
%apptest foo
test_program''')
def __instructions(self):
"""Fill in container instructions"""
self += comment('Mellanox OFED version {}'.format(self.__version))
if self.__prefix:
self += packages(ospackages=self.__deppackages + self.__ospackages)
else:
self += packages(ospackages=self.__ospackages)
self += packages(
apt_keys=[self.__key],
apt_repositories=[
'https://linux.mellanox.com/public/repo/mlnx_ofed/{0}/{1}/mellanox_mlnx_ofed.list'
.format(self.__version, self.__oslabel)
],
download=bool(self.__prefix),
extract=self.__prefix,
ospackages=self.__packages,
yum_keys=[self.__key],
yum_repositories=[
'https://linux.mellanox.com/public/repo/mlnx_ofed/{0}/{1}/mellanox_mlnx_ofed.repo'
.format(self.__version, self.__oslabel)
])
if self.__prefix:
commands = []
if self.__symlink:
commands.append('mkdir -p {0} && cd {0}'.format(
posixpath.join(self.__prefix, 'lib')))
# Prune the symlink directory itself and any debug
# libraries
commands.append(
'find .. -path ../lib -prune -o -name "*valgrind*" -prune -o -name "lib*.so*" -exec ln -s {} \;'
)
commands.append(
'cd {0} && ln -s usr/bin bin && ln -s usr/include include'.
format(self.__prefix))
# Suppress warnings from libibverbs
commands.append('mkdir -p /etc/libibverbs.d')
self += shell(commands=commands)
self += label(metadata=self.annotate_step())
def __instructions(self):
"""Fill in container instructions"""
if self.__comment:
if self.url:
self += comment(self.url, reformat=False)
elif self.repository:
self += comment(self.repository, reformat=False)
elif self.package:
self += comment(self.package, reformat=False)
if self.package:
self += copy(src=self.package,
dest=posixpath.join(self.__wd,
os.path.basename(self.package)))
self += shell(_arguments=self.__run_arguments,
commands=self.__commands)
self += environment(variables=self.environment_step())
self += label(metadata=self.annotate_step())
def test_invalid_ctype(self):
"""Invalid container type specified"""
l = label(metadata={'A': 'B'})
with self.assertRaises(RuntimeError):
str(l)
def test_single_bash(self):
"""Single label specified"""
l = label(metadata={'A': 'B'})
self.assertEqual(str(l), '')
def test_applabels_docker(self):
"""applabels not implemented in Docker"""
l = label(metadata={'ONE': 1, 'TWO': 2, 'THREE': 3}, _app='foo')
self.assertEqual(str(l), '''LABEL ONE=1 \\
THREE=3 \\
TWO=2''')
def test_empty(self):
"""No label specified"""
l = label()
self.assertEqual(str(l), '')
def test_multiple_docker(self):
"""Multiple labels specified"""
l = label(metadata={'ONE': 1, 'TWO': 2, 'THREE': 3})
self.assertEqual(str(l), '''LABEL ONE=1 \\
THREE=3 \\
TWO=2''')
def test_single_singularity(self):
"""Single label specified"""
l = label(metadata={'A': 'B'})
self.assertEqual(str(l), '%labels\n A B')
def test_single_docker(self):
"""Single label specified"""
l = label(metadata={'A': 'B'})
self.assertEqual(str(l), 'LABEL A=B')
