def test_merge_docker(self): """merge primitives""" l = [] l.append(label(metadata={'ONE': 1, 'TWO': 2})) l.append(label(metadata={'THREE': 3})) merged = l[0].merge(l) self.assertEqual(str(merged), '''LABEL ONE=1 \\ THREE=3 \\ TWO=2''') l.append(label(metadata={'ONE': 'uno'})) key_overwrite = l[0].merge(l) self.assertEqual(str(key_overwrite), '''LABEL ONE=uno \\ THREE=3 \\ TWO=2''')
def test_applabel_multiple_singularity(self): """Multiple app-specific labels specified""" l = label(metadata={'ONE': 1, 'TWO': 2, 'THREE': 3}, _app='foo') self.assertEqual(str(l), '''%applabels foo ONE 1 THREE 3 TWO 2''')
def runtime(self, _from='0'): """Generate the set of instructions to install the runtime specific components from a build in a previous stage. # Examples ```python g = generic_build(...) Stage0 += g Stage1 += g.runtime() ``` """ if self.__prefix: instructions = [] if self.__comment: if self.url: instructions.append(comment(self.url, reformat=False)) elif self.repository: instructions.append(comment(self.repository, reformat=False)) instructions.append(copy(_from=_from, src=self.__prefix, dest=self.__prefix)) if self.ldconfig: instructions.append(shell(commands=[self.ldcache_step( directory=posixpath.join(self.__prefix, self.__libdir))])) if self.runtime_environment_variables: instructions.append(environment( variables=self.environment_step(runtime=True))) if self.annotate: instructions.append(label(metadata=self.annotate_step())) return '\n'.join(str(x) for x in instructions) else: #pragma: no cover return
def __instructions(self): """Fill in container instructions""" # Mellanox OFED for version in self.__mlnx_versions: self += mlnx_ofed(annotate=False, oslabel=self.__mlnx_oslabel, packages=self.__mlnx_packages, prefix=posixpath.join(self.__prefix, version), symlink=self.__symlink, version=version) # Inbox OFED if self.__inbox: self += ofed(prefix=posixpath.join(self.__prefix, 'inbox'), symlink=self.__symlink) self += shell(commands=[ 'ln -s {0} {1}'.format(posixpath.join(self.__prefix, 'inbox'), posixpath.join(self.__prefix, '5.0-0')) ]) # Annotations self.add_annotation('mlnx_versions', ', '.join(self.__mlnx_versions)) self.add_annotation('inbox', self.__inbox) self += label(metadata=self.annotate_step())
def test_allsections_singularity(self): """One of each SCI-f section type""" s = scif(name='foo') s += copy(src='file', dest='/tmp/file') s += comment('My app') s += environment(variables={'ONE': '1'}) s += label(metadata={'A': 'B'}) s += runscript(commands=['default_program']) s += shell(commands=['build_cmds']) s += shell(commands=['test_program'], _test=True) self.assertEqual( str(s), r'''%appenv foo export ONE=1 %appfiles foo file /tmp/file %apphelp foo My app %appinstall foo for f in /.singularity.d/env/*; do . $f; done build_cmds %applabels foo A B %apprun foo exec default_program "$@" %apptest foo test_program''')
def __str__(self): """String representation of the building block""" instructions = [ comment(__doc__, reformat=False), packages(ospackages=self.__ospackages), packages(yum=['xz'], apt=['xz-utils']), lmod(version='7.8.6'), shell(commands=self.__commands), environment( variables={ 'MODULEPATH': '/opt/easybuild/modules/all:/home/easybuild/.local/' 'easybuild/modules/all:$MODULEPATH', 'FORCE_UNSAFE_CONFIGURE': '1', # https://github.com/docker-library/python/blob/edde349541e11f66dcc79cde1674317d065ddbdd/3.6/Dockerfile#L8 'LANG': 'C.UTF-8' }), label(metadata={'PACKAGE_MANAGER': 'easybuild'}) ] # Without the FORCE_UNSAFE_CONFIGURE env var some spack package # installations may fail due to running as root. return '\n'.join(str(x) for x in instructions)
def test_multiple_singularity(self): """Multiple labels specified""" l = label(metadata={'ONE': 1, 'TWO': 2, 'THREE': 3}) self.assertEqual(str(l), '''%labels ONE 1 THREE 3 TWO 2''')
def __instructions(self): self += comment('OSU benchmarks version {}'.format(self.__version)) self += packages(ospackages=self.__ospackages) self += shell(commands=self.__commands) if self.__environment_variables: self += environment(variables=self.__environment_variables) if self.__labels: self += label(metadata=self.__labels)
def test_merge_singularity(self): """merge primitives""" l = [] l.append(label(metadata={'ONE': 1, 'TWO': 2})) l.append(label(metadata={'THREE': 3})) merged = l[0].merge(l) self.assertEqual(str(merged), '''%labels ONE 1 THREE 3 TWO 2''') l.append(label(metadata={'ONE': 'uno'})) key_overwrite = l[0].merge(l) self.assertEqual(str(key_overwrite), '''%labels ONE uno THREE 3 TWO 2''')
def __instructions(self): """Fill in container instructions""" if self.__comment: if self.url: self += comment(self.url, reformat=False) elif self.repository: self += comment(self.repository, reformat=False) self += shell(_arguments=self.__run_arguments, commands=self.__commands) self += environment(variables=self.environment_step()) self += label(metadata=self.annotate_step())
def runtime(self, _from='0'): instructions = [] instructions.append( comment('OSU benchmarks version {}'.format(self.__version))) instructions.append( copy(_from=_from, src=self.prefix, dest=self.prefix)) if self.__environment_variables: instructions.append( environment(variables=self.__environment_variables)) if self.__labels: instructions.append(label(metadata=self.__labels)) return '\n'.join(str(x) for x in instructions)
def runtime(self, _from='0'): instructions = [ comment('OpenGeoSys build from repo {0}, branch {1}'.format( self.__repo, self.__branch)), copy(_from=_from, src=self.__prefix, dest=self.__prefix) ] if self.__environment_variables: instructions.append( environment(variables=self.__environment_variables)) if self.__labels: instructions.append(label(metadata=self.__labels)) return '\n'.join(str(x) for x in instructions)
def __instructions(self): self += comment('OpenGeoSys build from repo {0}, branch {1}'.format( self.__repo, self.__branch)) self += packages(ospackages=self.__ospackages) self += shell( commands=self.__commands, _arguments='--mount=type=bind,target=/scif/apps/ogs/src,rw') self += runscript(commands=['ogs']) if self.__environment_variables: self += environment(variables=self.__environment_variables) if self.__labels: self += label(metadata=self.__labels)
def __instructions(self): self += comment(__doc__, reformat=False) # https://github.com/bincrafters/community/issues/880 self += packages(ospackages=['pkg-config']) # For building curl: self += packages(ospackages=['autoconf-archive', 'libtool']) conan_version = "1.22.2" self += pip(pip='pip3', packages=['conan=={}'.format(conan_version)]) self += shell(commands=self.__commands) if self.__user_home != '': self += environment( variables={'CONAN_USER_HOME': self.__user_home}) self += label( metadata={ 'org.opengeosys.pm': 'conan', 'org.opengeosys.pm.conan.version': conan_version }) if self.__user_home != '': self += label(metadata={ 'org.opengeosys.pm.conan.user_home': self.__user_home })
def runtime(self, _from='0'): """Generate the set of instructions to install the runtime specific components from a build in a previous stage. # Examples ```python g = generic_build(...) Stage0 += g Stage1 += g.runtime() ``` """ if self.__prefix: if self.__comment: if self.url: self.rt += comment(self.url, reformat=False) elif self.repository: self.rt += comment(self.repository, reformat=False) if self.__runtime: for src in self.__runtime: if '*' in posixpath.basename(src): # When using COPY with more than one source file, # the destination must be a directory and end with # a / dest = posixpath.dirname(src) + '/' else: dest = src self.rt += copy(_from=_from, src=src, dest=dest) else: # Copy the entire prefix self.rt += copy(_from=_from, src=self.__prefix, dest=self.__prefix) if self.ldconfig: self.rt += shell(commands=[ self.ldcache_step( directory=posixpath.join(self.__prefix, self.__libdir)) ]) if self.runtime_environment_variables: self.rt += environment(variables=self.environment_step( runtime=True)) if self.annotate: self.rt += label(metadata=self.annotate_step()) return str(self.rt) else: #pragma: no cover return
def __instructions(self): self += comment(__doc__, reformat=False) self += packages(ospackages=['ccache']) self += shell(commands=[ 'mkdir -p {0} && chmod 777 {0}'.format(self.__cache_dir) ]) self += environment( variables={ 'CCACHE_DIR': self.__cache_dir, 'CCACHE_MAXSIZE': self.__cache_size, 'CCACHE_SLOPPINESS': 'pch_defines,time_macros' }) self += label(metadata={ 'ccache.dir': self.__cache_dir, 'ccache.size': self.__cache_size })
def test_allsections_docker(self): """One of each SCI-f section type""" # See comment in the test_defaults_docker test case scif_file = tempfile.NamedTemporaryFile(delete=False, suffix='.scif') s = scif(name='foo', file=scif_file.name) s += copy(src='file', dest='/tmp/file') s += comment('My app') s += environment(variables={'ONE': '1'}) s += label(metadata={'A': 'B'}) s += runscript(commands=['default_program']) s += shell(commands=['build_cmds']) s += shell(commands=['test_program'], _test=True) str(s) # Force writing the SCI-F recipe file # slurp file content with open(scif_file.name) as f: content = f.read() try: os.unlink(scif_file.name) except WindowsError: # WindowsError: [Error 32] The process cannot access the file # because it is being used by another process pass self.assertEqual( content, r'''%appenv foo export ONE=1 %appfiles foo file /tmp/file %apphelp foo My app %appinstall foo build_cmds %applabels foo A B %apprun foo exec default_program "$@" %apptest foo test_program''')
def __instructions(self): """Fill in container instructions""" self += comment('Mellanox OFED version {}'.format(self.__version)) if self.__prefix: self += packages(ospackages=self.__deppackages + self.__ospackages) else: self += packages(ospackages=self.__ospackages) self += packages( apt_keys=[self.__key], apt_repositories=[ 'https://linux.mellanox.com/public/repo/mlnx_ofed/{0}/{1}/mellanox_mlnx_ofed.list' .format(self.__version, self.__oslabel) ], download=bool(self.__prefix), extract=self.__prefix, ospackages=self.__packages, yum_keys=[self.__key], yum_repositories=[ 'https://linux.mellanox.com/public/repo/mlnx_ofed/{0}/{1}/mellanox_mlnx_ofed.repo' .format(self.__version, self.__oslabel) ]) if self.__prefix: commands = [] if self.__symlink: commands.append('mkdir -p {0} && cd {0}'.format( posixpath.join(self.__prefix, 'lib'))) # Prune the symlink directory itself and any debug # libraries commands.append( 'find .. -path ../lib -prune -o -name "*valgrind*" -prune -o -name "lib*.so*" -exec ln -s {} \;' ) commands.append( 'cd {0} && ln -s usr/bin bin && ln -s usr/include include'. format(self.__prefix)) # Suppress warnings from libibverbs commands.append('mkdir -p /etc/libibverbs.d') self += shell(commands=commands) self += label(metadata=self.annotate_step())
def __instructions(self): """Fill in container instructions""" if self.__comment: if self.url: self += comment(self.url, reformat=False) elif self.repository: self += comment(self.repository, reformat=False) elif self.package: self += comment(self.package, reformat=False) if self.package: self += copy(src=self.package, dest=posixpath.join(self.__wd, os.path.basename(self.package))) self += shell(_arguments=self.__run_arguments, commands=self.__commands) self += environment(variables=self.environment_step()) self += label(metadata=self.annotate_step())
def test_invalid_ctype(self): """Invalid container type specified""" l = label(metadata={'A': 'B'}) with self.assertRaises(RuntimeError): str(l)
def test_single_bash(self): """Single label specified""" l = label(metadata={'A': 'B'}) self.assertEqual(str(l), '')
def test_applabels_docker(self): """applabels not implemented in Docker""" l = label(metadata={'ONE': 1, 'TWO': 2, 'THREE': 3}, _app='foo') self.assertEqual(str(l), '''LABEL ONE=1 \\ THREE=3 \\ TWO=2''')
def test_empty(self): """No label specified""" l = label() self.assertEqual(str(l), '')
def test_multiple_docker(self): """Multiple labels specified""" l = label(metadata={'ONE': 1, 'TWO': 2, 'THREE': 3}) self.assertEqual(str(l), '''LABEL ONE=1 \\ THREE=3 \\ TWO=2''')
def test_single_singularity(self): """Single label specified""" l = label(metadata={'A': 'B'}) self.assertEqual(str(l), '%labels\n A B')
def test_single_docker(self): """Single label specified""" l = label(metadata={'A': 'B'}) self.assertEqual(str(l), 'LABEL A=B')