def test_parallel(self):
"""Parallel count specified"""
cm = CMakeBuild()
cm.configure_step(directory='/tmp/src')
build = cm.build_step(parallel=7)
self.assertEqual(build,
'cmake --build /tmp/src/build --target all -- -j7')
def singleStageBuild(stage):
stage += label(metadata={'MAINTAINER': 'Simeon Ehrig'})
stage += label(metadata={'EMAIL': '*****@*****.**'})
stage += label(metadata={'Version': str(container_version)})
stage += environment(variables={'GOL_VERSION': str(container_version)})
# copy example inside container
stage += copy(src='notebook', dest='/')
# copy and build the pnwriter library
stage += copy(src='pngwriter', dest='/opt')
stage += packages(ospackages=['libpng-dev'])
cmake = CMakeBuild(prefix='/notebook/pngwriter')
stage += shell(commands=[
cmake.configure_step(directory='/opt/pngwriter',
opts=['-DBUILD_SHARED_LIBS=ON']),
cmake.build_step(target='install'), 'rm -rf /opt/pngwriter'
])
# Copy notebook examples and pngwriter lib to the host's /tmp file system to obtain a writable file system.
stage += runscript(commands=[
'if [ ! -d /tmp/GOL-xeus-cling-cuda ]; then \n'
' mkdir /tmp/GOL-xeus-cling-cuda &&'
' cp -r /notebook/ /tmp/GOL-xeus-cling-cuda\n fi',
'cd /tmp/GOL-xeus-cling-cuda/notebook', 'jupyter-lab'
])
print(stage.__str__())
def test_configure_opts(self):
"""Configure options specified"""
cm = CMakeBuild()
configure = cm.configure_step(
directory='/tmp/src',
opts=['-DCMAKE_BUILD_TYPE=Debug', '-DWITH_FOO=ON'])
self.assertEqual(
configure,
'mkdir -p /tmp/src/build && cd /tmp/src/build && cmake -DCMAKE_BUILD_TYPE=Debug -DWITH_FOO=ON /tmp/src'
)
def test_toolchain(self):
"""Toolchain specified"""
cm = CMakeBuild()
tc = toolchain(CC='mycc', CXX='mycxx', FC='myfc', F77='myf77',
F90='myf90', CFLAGS='-g -O3', CPPFLAGS='-DFOO -DBAR',
CXXFLAGS='-g -O3', FCFLAGS='-g -O3', FFLAGS='-g -O3',
FLIBS='-ldl',
LD_LIBRARY_PATH='/opt/mysw/lib:/opt/yoursw/lib',
LDFLAGS='-Wl,--start-group foo.o bar.o -Wl,--endgroup',
LIBS='-ldl -lpthread')
configure = cm.configure_step(directory='/tmp/src', toolchain=tc)
self.assertEqual(configure,
'''mkdir -p /tmp/src/build && cd /tmp/src/build && CC=mycc CFLAGS='-g -O3' CPPFLAGS='-DFOO -DBAR' CXX=mycxx CXXFLAGS='-g -O3' F77=myf77 F90=myf90 FC=myfc FCFLAGS='-g -O3' FFLAGS='-g -O3' FLIBS=-ldl LD_LIBRARY_PATH=/opt/mysw/lib:/opt/yoursw/lib LDFLAGS='-Wl,--start-group foo.o bar.o -Wl,--endgroup' LIBS='-ldl -lpthread' cmake -DCMAKE_INSTALL_PREFIX=/usr/local /tmp/src''')
def __init__(self, **kwargs):
"""Initialize building block"""
# Trouble getting MRO with kwargs working correctly, so just call
# the parent class constructors manually for now.
#super(catalyst, self).__init__(**kwargs)
CMakeBuild.__init__(self, **kwargs)
ldconfig.__init__(self, **kwargs)
rm.__init__(self, **kwargs)
tar.__init__(self, **kwargs)
wget.__init__(self, **kwargs)
self.cmake_opts = kwargs.get('cmake_opts', [])
self.__edition = kwargs.get(
'edition', 'Base-Enable-Python-Essentials-Extras-Rendering-Base')
self.__ospackages = kwargs.get('ospackages', [])
self.prefix = kwargs.get('prefix', '/usr/local/catalyst')
self.__runtime_ospackages = [] # Filled in by __distro()
# Input toolchain, i.e., what to use when building
self.__toolchain = kwargs.get('toolchain', toolchain())
self.__version = kwargs.get('version', '5.6.0')
self.__url = r'https://www.paraview.org/paraview-downloads/download.php?submit=Download\&version={0}\&type=catalyst\&os=Sources\&downloadFile={1}'
self.__commands = [] # Filled in by __setup()
self.__environment_variables = {
'PATH': '{}:$PATH'.format(os.path.join(self.prefix, 'bin'))
}
self.__wd = '/var/tmp' # working directory
# Validate edition choice
if self.__edition not in [
'Base', 'Base-Essentials', 'Base-Essentials-Extras',
'Base-Essentials-Extras-Rendering-Base', 'Base-Enable-Python',
'Base-Enable-Python-Essentials',
'Base-Enable-Python-Essentials-Extras',
'Base-Enable-Python-Essentials-Extras-Rendering-Base'
]:
logging.warning('Invalid Catalyst edition "{0}", defaulting to '
'Base-Essentials'.format(self.__edition))
self.__edition = 'Base-Essentials'
self.__basename = 'Catalyst-v{0}-{1}'.format(self.__version,
self.__edition)
# Set the Linux distribution specific parameters
self.__distro()
# Construct the series of steps to execute
self.__setup()
def test_defaults(self):
"""Default values"""
cm = CMakeBuild()
# configure step
configure = cm.configure_step(directory='/tmp/src')
self.assertEqual(configure, 'mkdir -p /tmp/src/build && cd /tmp/src/build && cmake -DCMAKE_INSTALL_PREFIX=/usr/local /tmp/src')
# build step
build = cm.build_step()
self.assertEqual(build, 'cmake --build /tmp/src/build --target all -- -j$(nproc)')
# build some target
install = cm.build_step(target='foo')
self.assertEqual(install, 'cmake --build /tmp/src/build --target foo -- -j$(nproc)')
def test_configure_opts(self):
"""Configure options specified"""
cm = CMakeBuild(opts=['-DWITH_BAR=ON'], prefix='')
# Function arguments override constructor
configure = cm.configure_step(
directory='/tmp/src',
opts=['-DCMAKE_BUILD_TYPE=Debug', '-DWITH_FOO=ON']
)
self.assertEqual(configure,
'mkdir -p /tmp/src/build && cd /tmp/src/build && cmake -DCMAKE_BUILD_TYPE=Debug -DWITH_FOO=ON /tmp/src')
# Use constructor arguments
configure = cm.configure_step(directory='/tmp/src')
self.assertEqual(configure,
'mkdir -p /tmp/src/build && cd /tmp/src/build && cmake -DWITH_BAR=ON /tmp/src')
def test_directory(self):
"""Build directory specified"""
cm = CMakeBuild()
# Relative build dir
configure = cm.configure_step(directory='/tmp/src',
build_directory='../build')
self.assertEqual(
configure,
'mkdir -p /tmp/src/../build && cd /tmp/src/../build && cmake /tmp/src'
)
# Absolute build dir
configure = cm.configure_step(directory='/tmp/src',
build_directory='/tmp/build')
self.assertEqual(
configure,
'mkdir -p /tmp/build && cd /tmp/build && cmake /tmp/src')
def test_directory(self):
"""Build directory specified"""
cm = CMakeBuild()
# Relative build dir
configure = cm.configure_step(directory='/tmp/src',
build_directory='../build')
self.assertEqual(configure,
'mkdir -p /tmp/src/../build && cd /tmp/src/../build && cmake -DCMAKE_INSTALL_PREFIX=/usr/local /tmp/src')
# Absolute build dir
configure = cm.configure_step(directory='/tmp/src',
build_directory='/tmp/build')
self.assertEqual(configure,
'mkdir -p /tmp/build && cd /tmp/build && cmake -DCMAKE_INSTALL_PREFIX=/usr/local /tmp/src')
# No build directory
configure = cm.configure_step()
self.assertEqual(configure,
'cmake -DCMAKE_INSTALL_PREFIX=/usr/local ..')
def get_stage(container):
# generate baseimage
hpccm.config.set_container_format(container)
# version >3.2 is necessary for multi-stage build
if container == 'singularity':
hpccm.config.set_singularity_version('3.3')
Stage0 = hpccm.Stage()
Stage0 += baseimage(image='ubuntu:bionic')
# copy project from outside in the container
if container == 'singularity':
Stage0 += copy(src='../hello_world_tool', dest='/opt/')
else:
# docker: cannot copy files from outsite the build context
# so, we need to move the build context one level up
Stage0 += copy(src='./hello_world_tool', dest='/opt/hello_world_tool')
# install compiler tools
Stage0 += cmake(eula=True, version='3.14.5')
Stage0 += packages(ospackages=['g++', 'make', 'wget', 'build-essential'])
# build and install project
cmb = CMakeBuild()
cm = []
cm.append(cmb.configure_step(build_directory='/opt/build_hello_world_tool',
directory='/opt/hello_world_tool/'))
cm.append(cmb.build_step(target='install'))
Stage0 += shell(commands=cm)
Stage0 += shell(commands=build_openssl(name='openssl-1.1.1c',
build_dir='/opt/openssl_build'))
# script that runs when
# - singularity uses the run parameter or the image runs directly
# - docker uses the run parameter without arguments
Stage0 += runscript(commands=['hello_world_tool'])
return Stage0
def test_parallel(self):
"""Parallel count specified"""
cm = CMakeBuild(parallel=4)
cm.configure_step(directory='/tmp/src')
# Function arguments override constructor
build = cm.build_step(parallel=7)
self.assertEqual(build,
'cmake --build /tmp/src/build --target all -- -j7')
# Use constructor arguments
build = cm.build_step()
self.assertEqual(build,
'cmake --build /tmp/src/build --target all -- -j4')
def add_alpaka_dep_layer(stage: hpccm.Stage,
ubuntu_version: str,
cuda_support: bool,
extra_compiler: List[str],
alpaka=False) -> bool:
"""Add all dependencies to an hpccm stage that are necessary to build and run Alpaka.
:param stage: At least a baseimage
:type stage: hpccm.Stage
:param ubuntu_version: Ubuntu version number: '16.04' or '18.04'
:type ubuntu_version: str
:param cuda_support: Set True, if the Stage supports CUDA
:type cuda_support: bool
:param extra_compiler: List of compilers, which are installed additional to the system compiler. Supported are: gcc:[5-9], clang:[5.0-7.0, 8-9]
:type extra_compiler: str
:param alpaka: install alpaka in /usr/local
:type alpaka: bool
:returns: Returns True if function was successful
:rtype: bool
"""
if ubuntu_version != '16.04' and ubuntu_version != '18.04':
print('not supported Ubuntu version: ' + ubuntu_version,
file=sys.stderr)
print('supported are: 16.04, 18.04', file=sys.stderr)
return False
apt_package_list = [
'gcc', 'g++', 'make', 'software-properties-common', 'wget',
'libc6-dev', 'libomp-dev', 'unzip', 'git'
]
if ubuntu_version == '16.04':
apt_package_list.append('gnupg-agent')
if ubuntu_version == '18.04':
apt_package_list.append('gpg-agent')
stage += packages(ospackages=apt_package_list)
stage += cmake(eula=True, version='3.16.0')
# install extra compiler
if extra_compiler is not None:
for com in extra_compiler:
if com.startswith('gcc'):
stage += gnu(extra_repository=True, version=com[len('gcc:'):])
if com.startswith('clang'):
add_clang(stage, ubuntu_version, version=com[len('clang:'):])
#install boost
stage += shell(
commands=['add-apt-repository -y ppa:mhier/libboost-latest'])
stage += packages(ospackages=['boost1.67'])
if cuda_support:
stage += environment(
variables={
'LD_LIBRARY_PATH': '$LD_LIBRARY_PATH:/usr/local/cuda/lib64'
})
# alpaka use a function direct from the cuda driver library
# in the container, the cuda libraries are not at the default path
stage += environment(
variables={
'LIBRARY_PATH': '$LIBRARY_PATH:/usr/local/cuda/lib64/stubs'
})
stage += environment(
variables={'CMAKE_PREFIX_PATH': '/usr/local/cuda/lib64/stubs/'})
if alpaka:
git_alpaka = git()
cmake_alpaka = CMakeBuild()
alpaka_commands = []
alpaka_commands.append(
git_alpaka.clone_step(
repository='https://github.com/alpaka-group/alpaka.git',
path='/opt'))
alpaka_commands.append(
cmake_alpaka.configure_step(
build_directory='build',
directory='/opt/alpaka',
opts=['-Dalpaka_BUILD_EXAMPLES=OFF', '-DBUILD_TESTING=OFF']))
alpaka_commands.append(cmake_alpaka.build_step(target='install'))
alpaka_commands.append('rm -rf /opt/alpaka')
stage += shell(commands=alpaka_commands)
return True
def get_stage(container):
# generate baseimage
hpccm.config.set_container_format(container)
# version >3.2 is necessary for multi-stage build
if container == 'singularity':
hpccm.config.set_singularity_version('3.3')
Stage0 = hpccm.Stage()
# the stages need "names" so that they can reference each other
Stage0 += baseimage(image='ubuntu:bionic', _as='Stage0')
# copy project from outside in the container
if container == 'singularity':
Stage0 += copy(src='../hello_world_tool', dest='/opt/')
else:
# docker: cannot copy files from outsite the build context
# so, we need to move the build context one level up
Stage0 += copy(src='./hello_world_tool', dest='/opt/hello_world_tool')
# install compiler tools
Stage0 += cmake(eula=True, version='3.14.5')
Stage0 += packages(ospackages=['g++', 'make', 'wget', 'build-essential'])
# build and install project
cmb = CMakeBuild(prefix="/opt/hello_install/")
cm = []
cm.append(
cmb.configure_step(build_directory='/opt/build_hello_world_tool',
directory='/opt/hello_world_tool/',
opts=['-DCMAKE_INSTALL_RPATH=/usr/local/lib/']))
cm.append(cmb.build_step(target='install'))
Stage0 += shell(commands=cm)
Stage0 += shell(commands=build_openssl(name='openssl-1.1.1c',
build_dir='/opt/openssl_build'))
# add release stage
Stage1 = hpccm.Stage()
Stage1 += baseimage(image='ubuntu:bionic', _as='Stage1')
Stage1 += copy(_from='Stage0',
src='/opt/hello_install/',
dest='/usr/local/')
Stage1 += copy(_from='Stage0',
src='/opt/openssl_install/',
dest='/usr/local/')
# the commands merge the bin, lib ect. folders of hello_install and openssl_install
# in the /usr/local folder
if container == "singularity":
Stage1 += shell(commands=[
'cp -rl /usr/local/hello_install/* /usr/local/',
'cp -rl /usr/local/openssl_install/* /usr/local/',
'rm -r /usr/local/hello_install/',
'rm -r /usr/local/openssl_install/'
])
# script that runs when
# - singularity uses the run parameter or the image runs directly
# - docker uses the run parameter without arguments
Stage1 += runscript(commands=['hello_world_tool'])
return [Stage0, Stage1]
Stage0 += environment(
variables={
'PYTHONPATH':
'$PYTHONPATH:/usr/lib/python{}/site-packages:/usr/local/lib/python{}/dist-packages:/builds/gmxapi/build:/builds/sample_restraint/build/src/pythonmodule'
.format(python_version, python_version),
'PATH':
'$PATH:/usr/local/gromacs/bin'
})
################################################
# GROMACS
################################################
Stage0 += comment("GROMACS 2019")
cm = CMakeBuild()
build_cmds = [
git().clone_step(repository='https://github.com/gromacs/gromacs',
branch='release-2019',
path='/builds/gromacs',
directory='src'),
cm.configure_step(directory='/builds/gromacs/src',
build_directory='/builds/gromacs/build',
opts=[
'-DCMAKE_BUILD_TYPE=Release',
'-DCUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda',
'-DGMX_BUILD_OWN_FFTW=ON',
'-DGMX_GPU={}'.format(gpu), '-DGMX_MPI=OFF',
'-DGMX_OPENMP=ON', '-DGMXAPI=ON',
'-DMPIEXEC_PREFLAGS=--allow-run-as-root'
]),
def main():
parser = argparse.ArgumentParser(
description=
'Simple script for generating a singularity recipe for the GOL example.'
)
parser.add_argument(
'--build_prefix',
type=str,
default='/tmp/GOL_example',
help=
'Define the path in which all projects will be built (default: /tmp/GOL_example).'
)
parser.add_argument('-v ',
'--version',
action='store_true',
help='print version of the container')
args = parser.parse_args()
if args.version:
print(container_version)
sys.exit(0)
hpccm.config.set_container_format('singularity')
hpccm.config.set_singularity_version('3.3')
stage = hpccm.Stage()
stage += label(metadata={'GOL_MAINTAINER': 'Simeon Ehrig'})
stage += label(metadata={'GOL_EMAIL': '*****@*****.**'})
stage += label(metadata={'GOL_Version': str(container_version)})
# copy example inside container
stage += copy(src='notebook', dest='/')
stage += copy(src='jupyter_notebook_config.py', dest='/')
# copy and build the pnwriter library
stage += packages(ospackages=['libpng-dev'])
png = []
png_git = git()
png.append(
png_git.clone_step(
repository='https://github.com/pngwriter/pngwriter.git',
branch='dev',
path='/opt/'))
png_cmake = CMakeBuild(prefix='/notebook/pngwriter')
png.append(
png_cmake.configure_step(directory='/opt/pngwriter',
opts=['-DBUILD_SHARED_LIBS=ON']))
png.append(png_cmake.build_step(target='install'))
png.append('rm -rf /opt/pngwriter')
stage += shell(commands=png)
# Copy notebook examples and pngwriter lib to the host's /tmp file system to obtain a writable file system.
stage += runscript(commands=[
'if [ ! -d /tmp/GOL-xeus-cling-cuda ]; then \n'
' mkdir /tmp/GOL-xeus-cling-cuda &&'
' cp -r /notebook/ /tmp/GOL-xeus-cling-cuda &&'
' ln -s /tmp/GOL-xeus-cling-cuda/notebook/pngwriter'
' /tmp/GOL-xeus-cling-cuda/notebook/GTC_presentations/simulation/ \n fi',
'cd /tmp/GOL-xeus-cling-cuda/notebook',
'jupyter-notebook --config=/jupyter_notebook_config.py'
])
# Add the bootstrap manually because hpccm does not support .sregistry,
recipe = stage.__str__()
recipe = 'Bootstrap: library\nFrom: sehrig/default/xeus-cling-cuda:2.3\n\n' + recipe
print(recipe)
gnu = gnu(fortran=False)
Stage0 += gnu
Stage0 += packages(ospackages=['ca-certificates', 'cmake', 'git'])
ofed = ofed()
Stage0 += ofed
ompi = openmpi(configure_opts=['--enable-mpi-cxx'],
parallel=32,
prefix="/opt/openmpi",
version='3.0.0')
Stage0 += ompi
cm = CMakeBuild()
build_cmds = [
git().clone_step(repository='https://github.com/gromacs/gromacs',
branch='v' + gromacs_version,
path='/gromacs',
directory='src'),
cm.configure_step(directory='/gromacs/src',
build_directory='/gromacs/build',
opts=[
'-DCMAKE_BUILD_TYPE=Release',
'-DCMAKE_INSTALL_PREFIX=/gromacs/install',
'-DCUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda',
'-DGMX_BUILD_OWN_FFTW=ON', '-DGMX_GPU=ON',
'-DGMX_MPI=OFF', '-DGMX_OPENMP=ON',
'-DGMX_PREFER_STATIC_LIBS=ON',
'-DMPIEXEC_PREFLAGS=--allow-run-as-root',
Stage0 += packages(
ospackages=['autoconf', 'automake', 'ca-certificates', 'cmake', 'git'])
ofed = ofed()
Stage0 += ofed
ompi = openmpi(configure_opts=['--enable-mpi-cxx'],
prefix='/opt/openmpi',
parallel=32,
version='3.0.0')
Stage0 += ompi
# build QUDA
g = git()
cm = CMakeBuild()
quda_cmds = [
g.clone_step(repository='https://github.com/lattice/quda',
branch='v0.8.0',
path='/quda',
directory='src'),
cm.configure_step(
directory='/quda/src',
build_directory='/quda/build',
opts=[
'-DCMAKE_BUILD_TYPE=RELEASE', '-DQUDA_DIRAC_CLOVER=ON',
'-DQUDA_DIRAC_DOMAIN_WALL=ON', '-DQUDA_DIRAC_STAGGERED=ON',
'-DQUDA_DIRAC_TWISTED_CLOVER=ON', '-DQUDA_DIRAC_TWISTED_MASS=ON',
'-DQUDA_DIRAC_WILSON=ON', '-DQUDA_FORCE_GAUGE=ON',
'-DQUDA_FORCE_HISQ=ON', '-DQUDA_GPU_ARCH={}'.format(gpu_arch),
'-DQUDA_INTERFACE_MILC=ON', '-DQUDA_INTERFACE_QDP=ON',
def main():
parser = argparse.ArgumentParser(
description=
'Simple script for generating a singularity recipe for the GOL exercise.'
)
parser.add_argument(
'--build_prefix',
type=str,
default='/tmp/GOL_example',
help=
'Define the path in which all projects will be built (default: /tmp/GOL_example).'
)
parser.add_argument('-j',
type=str,
help='number of build threads for make (default: -j)')
parser.add_argument(
'-l',
type=str,
help='number of linker threads for the cling build (default: -j)')
parser.add_argument('-v ',
'--version',
action='store_true',
help='print version of the container')
args = parser.parse_args()
if args.version:
print(container_version)
sys.exit(0)
if args.j:
threads = int(args.j)
if threads < 1:
raise ValueError('-j have to be greater than 0')
else:
threads = None
if args.l:
linker_threads = int(args.l)
if linker_threads < 1:
raise ValueError('-l have to be greater than 0')
else:
linker_threads = None
xcc_gen = gn.XCC_gen(build_prefix=args.build_prefix,
threads=threads,
linker_threads=linker_threads)
stage = xcc_gen.gen_release_single_stage()
stage += label(metadata={'EXAMPLE_CONTAINER_MAINTAINER': 'Simeon Ehrig'})
stage += label(metadata={'EXAMPLE_CONTAINER_EMAIL': '*****@*****.**'})
stage += label(
metadata={'EXAMPLE_CONTAINER_Version': str(container_version)})
# disable the xsrf check, which avoid some problems in Firefox
stage += copy(src='jupyter_notebook_config.py', dest='/')
# copy and build the pnwriter library
stage += packages(ospackages=['libpng-dev'])
png_git = git()
stage += png_git.clone_step(
repository='https://github.com/pngwriter/pngwriter.git',
branch='dev',
path='/opt/')
png_cmake = CMakeBuild()
stage += shell(commands=[
png_cmake.configure_step(directory='/opt/pngwriter',
opts=['-DBUILD_SHARED_LIBS=ON']),
png_cmake.build_step(target='install')
])
# copy and install jitify
jitify_git = git()
stage += jitify_git.clone_step(
repository='https://github.com/NVIDIA/jitify.git', path='/opt/')
png_cmake = CMakeBuild()
stage += shell(commands=['cp /opt/jitify/jitify.hpp /usr/local/include'])
stage += shell(commands=['rm -rf /opt/pngwriter', 'rm -rf /opt/jitify'])
# check if the path to the notebook is specified, otherwise use the current directory
stage += runscript(commands=[
'if [ $# -gt 0 ]', 'then', 'cd $1', 'fi',
'jupyter-notebook --config=/jupyter_notebook_config.py'
])
print(stage.__str__())