def build(container_format='singularity',
os_release='ubuntu',
os_version='20.04'):
config.set_container_format(container_format)
image = f'{os_release}:{os_version}'
stage0 = Stage(name='stage0')
stage0 += baseimage(image=image, _bootstrap='docker')
stage0 += environment(variables={
'LC_ALL': 'en_AU.UTF-8',
'LANGUAGE': 'en_AU.UTF-8',
})
stage0 += label(metadata={
'maintainer': 'Luhan Cheng',
'email': '*****@*****.**'
})
stage0 += shell(commands=[
'rm -f /bin/sh && ln -s /bin/bash /bin/sh',
'rm -f /usr/bin/sh && ln -s /usr/bin/bash /usr/bin/sh', '/bin/bash',
'ln -s /usr/lib/x86_64-linux-gnu/libffi.so.7 /usr/lib/x86_64-linux-gnu/libffi.so.6'
])
stage0 += packages(apt=[
'wget', 'git', 'software-properties-common', 'build-essential',
'locales', 'zlib1g-dev'
])
stage0 += shell(commands=['locale-gen en_AU.UTF-8'])
stage0 += comment('Installing vglrun and TurboVNC')
stage0 += packages(apt=[
'ubuntu-desktop', 'vim', 'mesa-utils', 'python3-pip', 'python3-pyqt5',
'pyqt5-dev', 'python3-tk'
])
stage0 += shell(commands=[
'wget https://swift.rc.nectar.org.au/v1/AUTH_810/CVL-Singularity-External-Files/turbovnc_2.2.5_amd64.deb && dpkg -i turbovnc_2.2.5_amd64.deb && rm turbovnc_2.2.5_amd64.deb',
'wget https://swift.rc.nectar.org.au/v1/AUTH_810/CVL-Singularity-External-Files/virtualgl_2.6.4_amd64.deb && dpkg -i virtualgl_2.6.4_amd64.deb && rm virtualgl_2.6.4_amd64.deb',
'apt update', 'apt -y upgrade'
])
stage0 += primitives.copy(src='./ChimeraX-1.2.5.tar.gz',
dest='/opt/ChimeraX-1.2.5.tar.gz')
stage0 += shell(commands=[
'tar xfv /opt/ChimeraX-1.2.5.tar.gz -C /opt',
'rm /opt/ChimeraX-1.2.5.tar.gz'
])
stage0 += environment(
variables=from_prefix('/opt/chimerax-1.2.5-rc-2021.05.24'))
return stage0
def recipe() -> str:
"""Generate the recipe
:returns: singularity recipe
:rtype: str
"""
hpccm.config.set_container_format('singularity')
hpccm.config.set_singularity_version('3.3')
stage = hpccm.Stage()
stage += label(metadata={'CLING ALPAKA VERSION': str(version)})
stage += environment(variables={'CLING_ALPAKA_VERSION': version})
# the baseimage of xeus-cling-cuda is Ubuntu 16.04 with CUDA 8
if not ap_gn.add_alpaka_dep_layer(stage, '16.04', True, []):
print('adding the alpaka dependencies layer failed', file=sys.stderr)
exit(1)
install_alpaka(stage)
build_jupyter_kernel(stage)
# baseimage support nothing else than dockerhub
# so, manually add the baseimage command to the recipe
recipe = stage.__str__()
recipe = 'Bootstrap: library\nFrom: sehrig/default/xeus-cling-cuda-cxx:2.2\n\n' + recipe
return recipe
def build(container_format='singularity',
os_release='ubuntu',
os_version='20.04'):
config.set_container_format(container_format)
image = f'{os_release}:{os_version}'
stage0 = Stage(name='stage0')
stage0 += baseimage(image=image, _bootstrap='docker')
stage0 += environment(variables={
'LC_ALL': 'en_AU.UTF-8',
'LANGUAGE': 'en_AU.UTF-8',
})
stage0 += label(metadata={
'maintainer': 'Luhan Cheng',
'email': '*****@*****.**'
})
stage0 += shell(commands=[
'rm -f /bin/sh && ln -s /bin/bash /bin/sh',
'rm -f /usr/bin/sh && ln -s /usr/bin/bash /usr/bin/sh',
'/bin/bash',
])
stage0 += packages(apt=[
'wget', 'git', 'software-properties-common', 'build-essential',
'locales', 'zlib1g-dev'
])
stage0 += shell(commands=['locale-gen en_AU.UTF-8'])
return stage0
def singleStageBuild(stage):
stage += label(metadata={'MAINTAINER': 'Simeon Ehrig'})
stage += label(metadata={'EMAIL': '*****@*****.**'})
stage += label(metadata={'Version': str(container_version)})
stage += environment(variables={'GOL_VERSION': str(container_version)})
# copy example inside container
stage += copy(src='notebook', dest='/')
# copy and build the pnwriter library
stage += copy(src='pngwriter', dest='/opt')
stage += packages(ospackages=['libpng-dev'])
cmake = CMakeBuild(prefix='/notebook/pngwriter')
stage += shell(commands=[
cmake.configure_step(directory='/opt/pngwriter',
opts=['-DBUILD_SHARED_LIBS=ON']),
cmake.build_step(target='install'), 'rm -rf /opt/pngwriter'
])
# Copy notebook examples and pngwriter lib to the host's /tmp file system to obtain a writable file system.
stage += runscript(commands=[
'if [ ! -d /tmp/GOL-xeus-cling-cuda ]; then \n'
' mkdir /tmp/GOL-xeus-cling-cuda &&'
' cp -r /notebook/ /tmp/GOL-xeus-cling-cuda\n fi',
'cd /tmp/GOL-xeus-cling-cuda/notebook', 'jupyter-lab'
])
print(stage.__str__())
def build(image="Characterisation-Virtual-Laboratory/CharacterisationVL-Software:2004", _bootstrap='shub', mash_version='v2.2.2', capnp_version='0.8.0', quicktree_version='v2.5'):
stage0 = hpccm.Stage()
hpccm.config.set_container_format("singularity")
stage0 += baseimage(image=image, _bootstrap=_bootstrap, _distro='ubuntu20')
stage0 += label(metadata={'maintainer': 'Luhan Cheng', 'email': '*****@*****.**'})
stage0 += packages(ospackages=['cpanminus', 'libexpat1-dev', 'sqlite3', 'libsqlite3-dev', 'autoconf'])
stage0 += generic_build(
repository='https://github.com/khowe/quicktree',
branch=quicktree_version,
build=['make'],
install=[
'mv quicktree /usr/local/bin',
'mv include/* /usr/local/include/'
],
)
stage0 += boost()
stage0 += generic_autotools(
url=f'https://capnproto.org/capnproto-c++-{capnp_version}.tar.gz'
)
stage0 += shell(commands=['cpanm -l /usr/local/perl5 --notest BioPerl Bio::Sketch::Mash DBD::SQLite DBI'])
stage0 += generic_autotools(
repository=f'https://github.com/marbl/Mash',
preconfigure=['./bootstrap.sh'],
branch=mash_version,
with_capnp='/usr/local/',
with_boost='/usr/local/boost/',
)
stage0 += environment(variables={'PERL5LIB': '$PERL5LIB:/usr/local', **from_prefix('/usr/local/mashtree')})
stage0 += shell(commands=['cpanm -f -l /usr/local/mashtree Mashtree'])
return stage0
def build(container_format='singularity', os_release='ubuntu', os_version='20.04', gpu=True, cuda_version='11.0', desktop=True):
config.set_container_format(container_format)
image = f'{os_release}:{os_version}'
stage0 = Stage(name='stage0')
stage0 += baseimage(image=image, _bootstrap='docker')
stage0 += environment(variables={
'LC_ALL': 'en_AU.UTF-8',
'LANGUAGE': 'en_AU.UTF-8',
})
stage0 += label(metadata={'maintainer': 'Luhan Cheng', 'email': '*****@*****.**'})
stage0 += shell(commands=[
'rm -f /bin/sh && ln -s /bin/bash /bin/sh',
'rm -f /usr/bin/sh && ln -s /usr/bin/bash /usr/bin/sh',
'/bin/bash',
])
stage0 += packages(apt=['wget', 'git', 'software-properties-common', 'build-essential', 'locales', 'zlib1g-dev', 'dirmngr', 'gpg-agent'])
stage0 += shell(commands=['locale-gen en_AU.UTF-8'])
stage0 += shell(commands=[
"wget -O- http://neuro.debian.net/lists/bionic.us-ca.full | tee /etc/apt/sources.list.d/neurodebian.sources.list",
"apt-key adv --recv-keys --keyserver hkps://keyserver.ubuntu.com 0xA5D32F012649A5A9 2>&1",
])
stage0 += packages(apt=["fsl-complete"])
return stage0
def build(container_format='singularity', os_release='ubuntu', os_version='20.04'):
config.set_container_format(container_format)
image = f'{os_release}:{os_version}'
stage0 = Stage(name='stage0')
stage0 += baseimage(image=image, _bootstrap='docker')
stage0 += environment(variables={
'LC_ALL': 'en_AU.UTF-8',
'LANGUAGE': 'en_AU.UTF-8',
})
stage0 += label(metadata={'maintainer': 'Luhan Cheng', 'email': '*****@*****.**'})
stage0 += shell(commands=[
'rm -f /bin/sh && ln -s /bin/bash /bin/sh',
'rm -f /usr/bin/sh && ln -s /usr/bin/bash /usr/bin/sh',
'/bin/bash',
])
stage0 += packages(apt=['wget', 'git', 'software-properties-common', 'build-essential', 'locales', 'zlib1g-dev', 'bc', 'cmake',
'bison', 'flex', 'libx11-dev', 'x11proto-core-dev', 'libxi6', 'libxi-dev', 'libxmu6', 'libxmu-dev',
'libxmu-headers', 'libgl1-mesa-dev', 'libglu1-mesa-dev', 'libjpeg-dev', 'libevent-dev', 'libncurses-dev',
'pkg-config', 'libopenblas-dev'])
stage0 += shell(commands=['locale-gen en_AU.UTF-8'])
stage0 += comment('Installing vglrun and TurboVNC')
stage0 += packages(apt=['ubuntu-desktop', 'vim', 'mesa-utils', 'python3-pip', 'python3-pyqt5', 'pyqt5-dev', 'python3-tk'])
stage0 += shell(commands=[
'wget https://swift.rc.nectar.org.au/v1/AUTH_810/CVL-Singularity-External-Files/turbovnc_2.2.5_amd64.deb && dpkg -i turbovnc_2.2.5_amd64.deb && rm turbovnc_2.2.5_amd64.deb',
'wget https://swift.rc.nectar.org.au/v1/AUTH_810/CVL-Singularity-External-Files/virtualgl_2.6.4_amd64.deb && dpkg -i virtualgl_2.6.4_amd64.deb && rm virtualgl_2.6.4_amd64.deb',
'apt update',
'apt -y upgrade'
])
stage0 += generic_cmake(
repository='https://github.com/BIC-MNI/minc-toolkit-v2.git',
branch='release-1.9.18.1',
recursive=True,
prefix='/opt/minc/1.9.18',
preconfigure=['sed -i -e \'s/http\:\/\/mirrors\.ibiblio\.org\/gnu\/ftp\/gnu\/gsl/http\:\/\/ftp\.\sun\.ac\.za\/ftp\/pub\/mirrors\/ftp\.gnu\.org\/gsl/g\' cmake-modules/BuildGSL.cmake', ],
cmake_opts=['-DCMAKE_BUILD_TYPE:STRING=Release', '-DCMAKE_INSTALL_PREFIX:PATH=/opt/minc/1.9.18', '-DMT_BUILD_ABC:BOOL=ON',
'-DMT_BUILD_ANTS:BOOL=ON', '-DMT_BUILD_C3D:BOOL=ON', '-DMT_BUILD_ELASTIX:BOOL=ON', '-DMT_BUILD_IM:BOOL=OFF',
'-DMT_BUILD_ITK_TOOLS:BOOL=ON', '-DMT_BUILD_LITE:BOOL=OFF', '-DMT_BUILD_SHARED_LIBS:BOOL=ON', '-DMT_BUILD_VISUAL_TOOLS:BOOL=ON',
'-DMT_USE_OPENMP:BOOL=ON', '-DUSE_SYSTEM_FFTW3D:BOOL=OFF', '-DUSE_SYSTEM_FFTW3F:BOOL=OFF', '-DUSE_SYSTEM_GLUT:BOOL=OFF',
'-DUSE_SYSTEM_GSL:BOOL=OFF', '-DUSE_SYSTEM_HDF5:BOOL=OFF', '-DUSE_SYSTEM_ITK:BOOL=OFF', '-DUSE_SYSTEM_NETCDF:BOOL=OFF',
'-DUSE_SYSTEM_NIFTI:BOOL=OFF', '-DUSE_SYSTEM_PCRE:BOOL=OFF', '-DUSE_SYSTEM_ZLIB:BOOL=OFF']
)
stage0 += environment(variables=from_prefix('/opt/minc/1.9.18.1'))
return stage0
def build(container_format='singularity',
os_release='ubuntu',
os_version='20.04'):
config.set_container_format(container_format)
image = f'{os_release}:{os_version}'
stage0 = Stage(name='stage0')
stage0 += baseimage(image=image, _bootstrap='docker')
stage0 += environment(variables={
'LC_ALL': 'en_AU.UTF-8',
'LANGUAGE': 'en_AU.UTF-8',
})
stage0 += label(metadata={
'maintainer': 'Luhan Cheng',
'email': '*****@*****.**'
})
stage0 += shell(commands=[
'rm -f /bin/sh && ln -s /bin/bash /bin/sh',
'rm -f /usr/bin/sh && ln -s /usr/bin/bash /usr/bin/sh',
'/bin/bash',
])
stage0 += packages(apt=[
'wget', 'git', 'software-properties-common', 'build-essential',
'locales', 'zlib1g-dev'
])
stage0 += shell(commands=['locale-gen en_AU.UTF-8'])
stage0 += comment('Installing vglrun and TurboVNC')
stage0 += packages(apt=[
'ubuntu-desktop', 'vim', 'mesa-utils', 'python3-pip', 'python3-pyqt5',
'pyqt5-dev', 'python3-tk'
])
stage0 += shell(commands=[
'wget https://swift.rc.nectar.org.au/v1/AUTH_810/CVL-Singularity-External-Files/turbovnc_2.2.5_amd64.deb && dpkg -i turbovnc_2.2.5_amd64.deb && rm turbovnc_2.2.5_amd64.deb',
'wget https://swift.rc.nectar.org.au/v1/AUTH_810/CVL-Singularity-External-Files/virtualgl_2.6.4_amd64.deb && dpkg -i virtualgl_2.6.4_amd64.deb && rm virtualgl_2.6.4_amd64.deb',
'apt update', 'apt -y upgrade'
])
# coin3d
stage0 += packages(apt=[
'mesa-common-dev', 'libglu1-mesa-dev', 'freeglut3-dev', 'autoconf',
'autogen', 'doxygen'
])
compiler = gnu(version="10")
stage0 += compiler
stage0 += boost()
stage0 += generic_cmake(
repository="https://github.com/coin3d/coin",
recursive=True,
cmake_opts=[
'-G "Unix Makefiles"', "-DCMAKE_BUILD_TYPE=Release",
"-DCOIN_BUILD_DOCUMENTATION=OFF",
"-DCMAKE_INSTALL_PREFIX=/usr/local/coin3d"
],
postinstall=['cd cpack.d', 'cpack --config debian.cmake'])
return stage0
def build(container_format='singularity',
os_release='ubuntu',
os_version='20.04',
cuda_version='11.0'):
config.set_container_format(container_format)
image = f'nvcr.io/nvidia/cuda:{cuda_version}-devel-{os_release}{os_version}'
stage0 = Stage(name='stage0')
stage0 += baseimage(image=image, _bootstrap='docker')
stage0 += environment(variables={
'LC_ALL': 'en_AU.UTF-8',
'LANGUAGE': 'en_AU.UTF-8',
})
stage0 += label(metadata={
'maintainer': 'Luhan Cheng',
'email': '*****@*****.**'
})
stage0 += shell(commands=[
'rm -f /bin/sh && ln -s /bin/bash /bin/sh',
'rm -f /usr/bin/sh && ln -s /usr/bin/bash /usr/bin/sh',
'/bin/bash',
])
stage0 += environment(variables=from_prefix('/usr/local/cuda'))
stage0 += packages(apt=[
'wget', 'git', 'software-properties-common', 'build-essential',
'locales', 'zlib1g-dev'
])
stage0 += shell(commands=['locale-gen en_AU.UTF-8'])
stage0 += comment('Installing vglrun and TurboVNC')
stage0 += packages(apt=[
'ubuntu-desktop', 'vim', 'mesa-utils', 'python3-pip', 'python3-pyqt5',
'pyqt5-dev', 'python3-tk'
])
stage0 += shell(commands=[
'wget https://swift.rc.nectar.org.au/v1/AUTH_810/CVL-Singularity-External-Files/turbovnc_2.2.5_amd64.deb && dpkg -i turbovnc_2.2.5_amd64.deb && rm turbovnc_2.2.5_amd64.deb',
'wget https://swift.rc.nectar.org.au/v1/AUTH_810/CVL-Singularity-External-Files/virtualgl_2.6.4_amd64.deb && dpkg -i virtualgl_2.6.4_amd64.deb && rm virtualgl_2.6.4_amd64.deb',
'apt update', 'apt -y upgrade'
])
stage0 += conda(eula=True,
channels=['schrodinger'],
packages=['pymol-bundle=2.4.1'])
stage0 += environment(variables=from_prefix('/usr/local/anaconda'))
return stage0
def build(version='2.1.1'):
stage0 = CVL_ubuntu_stage()
stage0 += generic_build(
repository='https://github.com/DerrickWood/kraken2.git',
branch=f'v{version}',
install=['./install_kraken2.sh /usr/local/kraken2'],
)
stage0 += environment(variables=add_binary('/usr/local/kraken2/bin'))
return stage0
def build(container_format='singularity', os_release='ubuntu', os_version='20.04'):
config.set_container_format(container_format)
image = f'{os_release}:{os_version}'
stage0 = Stage(name='stage0')
stage0 += baseimage(image=image, _bootstrap='docker')
stage0 += environment(variables={
'LC_ALL': 'en_AU.UTF-8',
'LANGUAGE': 'en_AU.UTF-8',
})
stage0 += label(metadata={'maintainer': 'Luhan Cheng', 'email': '*****@*****.**'})
stage0 += shell(commands=['rm /usr/bin/sh', 'ln -s /usr/bin/bash /usr/bin/sh', '/usr/bin/bash'])
stage0 += packages(apt=['wget', 'git', 'software-properties-common', 'build-essential', 'locales', 'zlib1g-dev'])
stage0 += shell(commands=['locale-gen en_AU.UTF-8'])
stage0 += comment('Installing vglrun and TurboVNC')
stage0 += packages(apt=['ubuntu-desktop', 'vim', 'mesa-utils', 'python3-pip', 'python3-pyqt5', 'pyqt5-dev', 'python3-tk'])
stage0 += shell(commands=[
'wget https://swift.rc.nectar.org.au/v1/AUTH_810/CVL-Singularity-External-Files/turbovnc_2.2.5_amd64.deb && dpkg -i turbovnc_2.2.5_amd64.deb && rm turbovnc_2.2.5_amd64.deb',
'wget https://swift.rc.nectar.org.au/v1/AUTH_810/CVL-Singularity-External-Files/virtualgl_2.6.4_amd64.deb && dpkg -i virtualgl_2.6.4_amd64.deb && rm virtualgl_2.6.4_amd64.deb',
'apt update',
'apt -y upgrade'
])
stage0 += shell(commands=[
'curl -sL https://deb.nodesource.com/setup_12.x | bash -',
'apt install -y nodejs npm'
])
prefix = '/var/www'
stage0 += shell(commands=[
f'mkdir {prefix} && cd {prefix}',
'git clone --branch 1.16.10-release https://github.com/gmod/jbrowse .',
'./setup.sh',
])
stage0 += environment(variables={
'PATH': f'{prefix}/bin:$PATH'
})
stage0 += runscript(commands=[
f'cd {prefix}',
'npm run start $*'
])
return stage0
def build(version='1.3'):
stage0 = CVL_ubuntu_stage()
stage0 += generic_build(
repository='https://github.com/lh3/seqtk.git',
branch=f'v{version}',
build=['make'],
install=[
'mkdir -p /usr/local/seqtk/bin && cp seqtk /usr/local/seqtk/bin/'
])
stage0 += environment(variables=add_binary('/usr/local/seqtk/bin'))
return stage0
def build(container_format='singularity', os_release='ubuntu', os_version='20.04'):
config.set_container_format(container_format)
image = f'{os_release}:{os_version}'
stage0 = Stage(name='stage0')
stage0 += baseimage(image=image, _bootstrap='docker')
stage0 += environment(variables={
'LC_ALL': 'en_AU.UTF-8',
'LANGUAGE': 'en_AU.UTF-8',
})
stage0 += label(metadata={'maintainer': 'Luhan Cheng', 'email': '*****@*****.**'})
stage0 += shell(commands=['rm /usr/bin/sh', 'ln -s /usr/bin/bash /usr/bin/sh', '/usr/bin/bash'])
stage0 += packages(apt=['wget', 'git', 'software-properties-common', 'build-essential', 'locales', 'zlib1g-dev',
'perl', 'openssh-server', 'libssl-dev', 'gnuplot', 'globus-gridftp-server-progs', 'rsync', 'acl-dev'])
stage0 += gnu(version='10')
stage0 += shell(commands=[
'yes | cpan Data::MessagePack',
'yes | cpan IO::Socket::SSL'
])
stage0 += generic_build(
repository='https://github.com/eeertekin/bbcp.git',
commit='64af832',
prefix='/usr/local/bbcp/',
build=['make -C src'],
install=['mv bin/amd64_linux/bbcp /usr/local/bbcp/']
)
stage0 += environment(variables=add_binary('/usr/local/bbcp'))
stage0 += generic_build(
repository='https://github.com/pkolano/shift.git',
commit='ffb7f5f',
prefix='/usr/local/shift',
directory='shift/c',
build=['make nolustre'],
install=['mdkir /usr/local/shift/bin && mv c/shift-bin /usr/local/shift/bin']
)
return stage0
def build(container_format='singularity', os_release='ubuntu', os_version='20.04', cuda_version='11.0'):
config.set_container_format(container_format)
image = f'{os_release}:{os_version}'
stage0 = Stage(name='stage0')
stage0 += baseimage(image=image, _bootstrap='docker')
stage0 += environment(variables={
'LC_ALL': 'en_AU.UTF-8',
'LANGUAGE': 'en_AU.UTF-8',
})
stage0 += label(metadata={'maintainer': 'Luhan Cheng', 'email': '*****@*****.**'})
stage0 += shell(commands=['rm /usr/bin/sh', 'ln -s /usr/bin/bash /usr/bin/sh', '/usr/bin/bash'])
stage0 += packages(apt=['wget', 'git', 'software-properties-common', 'build-essential', 'locales'])
stage0 += shell(commands=['locale-gen en_AU.UTF-8'])
stage0 += comment('Installing vglrun and TurboVNC')
stage0 += packages(apt=['ubuntu-desktop', 'vim', 'mesa-utils', 'python3-pip', 'python3-pyqt5', 'pyqt5-dev', 'python3-tk'])
stage0 += shell(commands=[
'wget https://swift.rc.nectar.org.au/v1/AUTH_810/CVL-Singularity-External-Files/turbovnc_2.2.5_amd64.deb && dpkg -i turbovnc_2.2.5_amd64.deb && rm turbovnc_2.2.5_amd64.deb',
'wget https://swift.rc.nectar.org.au/v1/AUTH_810/CVL-Singularity-External-Files/virtualgl_2.6.4_amd64.deb && dpkg -i virtualgl_2.6.4_amd64.deb && rm virtualgl_2.6.4_amd64.deb',
'apt update',
'apt -y upgrade'
])
stage0 += conda(
eula=True,
packages=['python=3.8.5', 'openjdk', 'pip'],
)
stage0 += environment(variables=from_prefix('/usr/local/anaconda'))
stage0 += shell(commands=['pip install --upgrade minerl'])
stage0 += shell(commands=[
'wget https://launcher.mojang.com/download/Minecraft.deb && dpkg -i Minecraft.deb && rm Minecraft.deb',
])
stage0 += runscript(commands=['source /usr/local/anaconda/etc/profile.d/conda.sh', '/usr/local/anaconda/bin/python3 $*'])
return stage0
def build(version='1.0.1'):
stage0 = CVL_ubuntu_stage()
stage0 += packages(apt=['r-base'])
stage0 += generic_build(
repository='https://github.com/danknights/sourcetracker.git',
branch=f'v{version}',
install=[
'mkdir -p /usr/local/sourcetracker/bin && mv src/SourceTracker.r /usr/local/sourcetracker/bin'
])
stage0 += environment(
variables={
'SOURCETRACKER_PATH': '/usr/local/sourcetracker/bin',
'PATH': '/usr/local/sourcetracker/bin:$PATH'
})
return stage0
def build(container_format='singularity',
os_release='ubuntu',
os_version='20.04'):
config.set_container_format(container_format)
image = f'{os_release}:{os_version}'
stage0 = Stage(name='stage0')
stage0 += baseimage(image=image, _bootstrap='docker')
stage0 += environment(variables={
'LC_ALL': 'en_AU.UTF-8',
'LANGUAGE': 'en_AU.UTF-8',
})
stage0 += label(metadata={
'maintainer': 'Luhan Cheng',
'email': '*****@*****.**'
})
stage0 += shell(commands=[
'rm -f /bin/sh && ln -s /bin/bash /bin/sh',
'rm -f /usr/bin/sh && ln -s /usr/bin/bash /usr/bin/sh',
'/bin/bash',
])
stage0 += packages(apt=[
'wget', 'git', 'software-properties-common', 'build-essential',
'locales', 'zlib1g-dev'
])
stage0 += shell(commands=['locale-gen en_AU.UTF-8'])
stage0 += comment('Installing vglrun and TurboVNC')
stage0 += packages(apt=[
'ubuntu-desktop', 'vim', 'mesa-utils', 'python3-pip', 'python3-pyqt5',
'pyqt5-dev', 'python3-tk', 'cmake', 'libgtk2.0-dev', 'libasound2-dev',
'libavformat-dev', 'libjack-jackd2-dev', 'uuid-dev'
])
stage0 += shell(commands=[
'wget https://swift.rc.nectar.org.au/v1/AUTH_810/CVL-Singularity-External-Files/turbovnc_2.2.5_amd64.deb && dpkg -i turbovnc_2.2.5_amd64.deb && rm turbovnc_2.2.5_amd64.deb',
'wget https://swift.rc.nectar.org.au/v1/AUTH_810/CVL-Singularity-External-Files/virtualgl_2.6.4_amd64.deb && dpkg -i virtualgl_2.6.4_amd64.deb && rm virtualgl_2.6.4_amd64.deb',
'apt update', 'apt -y upgrade'
])
stage0 += shell(commands=['pip install conan'])
stage0 += generic_cmake(repository='https://github.com/audacity/audacity/',
branch='Audacity-3.0.5')
return stage0
def build(container_format='singularity',
os_release='ubuntu',
os_version='20.04'):
config.set_container_format(container_format)
image = f'{os_release}:{os_version}'
stage0 = Stage(name='stage0')
stage0 += baseimage(image=image, _bootstrap='docker')
stage0 += environment(variables={
'LC_ALL': 'en_AU.UTF-8',
'LANGUAGE': 'en_AU.UTF-8',
})
stage0 += label(metadata={
'maintainer': 'Luhan Cheng',
'email': '*****@*****.**'
})
stage0 += shell(commands=[
'rm -f /bin/sh && ln -s /bin/bash /bin/sh',
'rm -f /usr/bin/sh && ln -s /usr/bin/bash /usr/bin/sh',
'/bin/bash',
])
stage0 += packages(apt=[
'wget', 'git', 'software-properties-common', 'build-essential',
'locales', 'zlib1g-dev'
])
stage0 += shell(commands=['locale-gen en_AU.UTF-8'])
stage0 += packages(apt=[
'perl', 'libmodule-install-perl', 'libxml-parser-perl', 'cpanminus'
])
stage0 += shell(commands=[
'cpanm inc::latest',
'cpanm BioPerl Bio::SeqIO Getopt::Euclid List::Util Math::Random::MT version'
])
stage0 += shell(commands=[
'wget https://sourceforge.net/projects/biogrinder/files/biogrinder/Grinder-0.5.4/Grinder-0.5.4.tar.gz && tar -xf Grinder-0.5.4.tar.gz && rm Grinder-0.5.4.tar.gz',
'cd Grinder-0.5.4', 'perl Makefile.PL', 'make', 'make install'
])
return stage0
##
# hpccm --recipe relion_3.0.7_staged.py
# --format singularity
# --singularity-version 3.2 > recipe.def
# Start "base recipe, known as '18.04_cuda9'"
# add docstring to Dockerfile
Stage0 += comment(__doc__.strip(), reformat=False)
#Stage0 = hpccm.Stage()
Stage0.name = '18.04_cuda9'
Stage0 += baseimage(image="ubuntu:18.04", _as=Stage0.name)
# Setting up environment vars for container build and runtime.
Stage0 += environment(variables={
"LC_ALL": "en_AU.UTF-8",
"LANGUAGE": "en_AU.UTF-8"
})
# Set apt repos and packages to install
base_packages = apt_get(
repositories=[
"deb http://us.archive.ubuntu.com/ubuntu/ bionic main restricted",
"deb http://us.archive.ubuntu.com/ubuntu/ bionic-updates main restricted",
"deb http://us.archive.ubuntu.com/ubuntu/ bionic universe",
"deb http://us.archive.ubuntu.com/ubuntu/ bionic-updates universe",
],
ospackages=[
"software-properties-common", "locales", "wget", "ubuntu-desktop",
"vim", "git", "cmake", "freeglut3-dev", "build-essential",
"libx11-dev", "libxmu-dev", "libxi-dev", "libglu1-mesa",
"libglu1-mesa-dev", "python-pip", "python-pyqt5", "pyqt5-dev",
def build(container_format='singularity',
flavour="devel",
os_release='ubuntu',
os_version='20.04',
cuda_version='11.3.0',
python_version='3.8',
anaconda_version='4.9.2',
alphafold_path="/opt/alphafold",
version='2.1.2'):
config.set_container_format(container_format)
image = f'nvcr.io/nvidia/cuda:{cuda_version}-{flavour}-{os_release}{os_version}'
stage0 = Stage(name='stage0')
stage0 += baseimage(image=image, _bootstrap='docker')
stage0 += environment(variables={
'LC_ALL': 'en_AU.UTF-8',
'LANGUAGE': 'en_AU.UTF-8',
})
stage0 += label(metadata={
'maintainer': 'Luhan Cheng',
'email': '*****@*****.**'
})
stage0 += shell(commands=[
'rm -f /bin/sh && ln -s /bin/bash /bin/sh',
'rm -f /usr/bin/sh && ln -s /usr/bin/bash /usr/bin/sh',
'/bin/bash',
])
stage0 += environment(variables=from_prefix('/usr/local/cuda'))
stage0 += packages(apt=[
'wget', 'git', 'software-properties-common', 'build-essential',
'locales', 'zlib1g-dev', 'vim'
])
stage0 += shell(commands=['locale-gen en_AU.UTF-8'])
stage0 += conda(eula=True,
packages=[
f'python={python_version}', "openmm==7.5.1",
"cudnn==8.2.1.32", "cudatoolkit==11.0.3",
"pdbfixer==1.7", "hmmer==3.3.2", "hhsuite==3.3.0",
"kalign2==2.04"
],
channels=['conda-forge', 'bioconda'],
version=anaconda_version)
stage0 += environment(variables=from_prefix('/usr/local/anaconda'))
stage0 += shell(commands=['conda activate base'])
stage0 += shell(commands=[
f"mkdir -p {alphafold_path}",
f'git clone -b v{version} https://github.com/deepmind/alphafold.git {alphafold_path}'
])
stage0 += environment(variables={"ALPHAFOLD_PATH": f"{alphafold_path}"})
stage0 += shell(commands=[
f'wget -q -P {alphafold_path}/alphafold/common/ https://git.scicore.unibas.ch/schwede/openstructure/-/raw/7102c63615b64735c4941278d92b554ec94415f8/modules/mol/alg/src/stereo_chemical_props.txt'
])
stage0 += pip(packages=[
"absl-py==0.13.0", "biopython==1.79", "chex==0.0.7", "dm-haiku==0.0.4",
"dm-tree==0.1.6", "immutabledict==2.0.0", "ml-collections==0.1.0",
"numpy==1.19.5", "scipy==1.7.0", "tensorflow==2.5.0", "pandas==1.3.4",
"tensorflow-cpu==2.5.0"
],
pip='pip3')
stage0 += pip(packages=[
"jax",
"jaxlib==0.1.69+cuda111 -f https://storage.googleapis.com/jax-releases/jax_releases.html"
],
upgrade=True,
pip='pip3')
stage0 += pip(packages=[f'-r {alphafold_path}/requirements.txt'],
pip='pip3')
stage0 += shell(commands=[
f'cd /usr/local/anaconda/lib/python{python_version}/site-packages/ && patch -p0 < {alphafold_path}/docker/openmm.patch'
])
stage0 += shell(commands=[
f'cd {alphafold_path}',
'wget https://raw.githubusercontent.com/0luhancheng0/hpccm-containers/main/src/hpccm_containers/alphafold/run_alphafold.sh && chmod a+x run_alphafold.sh'
])
stage0 += runscript(commands=[
'source /usr/local/anaconda/etc/profile.d/conda.sh',
f"cd {alphafold_path}", f'{alphafold_path}/run_alphafold.sh $@'
])
return stage0
def build(container_format='singularity',
os_release='ubuntu',
os_version='20.04',
cuda_version='11.0'):
config.set_container_format(container_format)
image = f'nvcr.io/nvidia/cuda:{cuda_version}-devel-{os_release}{os_version}'
stage0 = Stage(name='stage0')
stage0 += baseimage(image=image, _bootstrap='docker')
stage0 += environment(variables={
'LC_ALL': 'en_AU.UTF-8',
'LANGUAGE': 'en_AU.UTF-8',
})
stage0 += label(metadata={
'maintainer': 'Luhan Cheng',
'email': '*****@*****.**'
})
stage0 += shell(commands=[
'rm -f /bin/sh && ln -s /bin/bash /bin/sh',
'rm -f /usr/bin/sh && ln -s /usr/bin/bash /usr/bin/sh',
'/bin/bash',
])
stage0 += environment(variables=from_prefix('/usr/local/cuda'))
stage0 += packages(apt=[
'wget', 'git', 'software-properties-common', 'build-essential',
'locales', 'zlib1g-dev'
])
stage0 += shell(commands=['locale-gen en_AU.UTF-8'])
stage0 += comment('Installing vglrun and TurboVNC')
stage0 += packages(apt=[
'ubuntu-desktop', 'vim', 'mesa-utils', 'python3-pip', 'python3-pyqt5',
'pyqt5-dev', 'python3-tk'
])
stage0 += shell(commands=[
'wget https://swift.rc.nectar.org.au/v1/AUTH_810/CVL-Singularity-External-Files/turbovnc_2.2.5_amd64.deb && dpkg -i turbovnc_2.2.5_amd64.deb && rm turbovnc_2.2.5_amd64.deb',
'wget https://swift.rc.nectar.org.au/v1/AUTH_810/CVL-Singularity-External-Files/virtualgl_2.6.4_amd64.deb && dpkg -i virtualgl_2.6.4_amd64.deb && rm virtualgl_2.6.4_amd64.deb',
'apt update', 'apt -y upgrade'
])
stage0 += comment('Installing pre-requisites')
stage0 += primitives.copy(
src="./ccp4-7.1.014-shelx-arpwarp-linux64.tar.gz",
dest="/opt/ccp4-7.1.014-shelx-arpwarp-linux64.tar.gz")
stage0 += shell(commands=[
'cd /opt && tar -xf ccp4-7.1.014-shelx-arpwarp-linux64.tar.gz && rm ccp4-7.1.014-shelx-arpwarp-linux64.tar.gz',
'touch $HOME/.agree2ccp4v6',
'cd ccp4-7.1',
'./BINARY.setup',
])
stage0 += environment(variables=add_binary('/opt/ccp4-7.1/bin'))
stage0 += comment('Installing CCP-EM')
stage0 += primitives.copy(src="./ccpem-1.5.0-linux-x86_64.tar.gz",
dest="/opt/ccpem-1.5.0-linux-x86_64.tar.gz")
stage0 += primitives.copy(src="./input.txt", dest="/opt/input.txt")
stage0 += shell(commands=[
'touch $HOME/.agree2ccpemv1',
'cd /opt && tar -xf ccpem-1.5.0-linux-x86_64.tar.gz && rm ccpem-1.5.0-linux-x86_64.tar.gz',
'cd ccpem-1.5.0', './install_ccpem.sh',
'cat /opt/input.txt | bash install_modeller.sh'
])
return stage0
def build(python_version='3.6', freesurfer_version='6.0.1'):
stage0 = Stage()
config.set_container_format('singularity')
stage0 += baseimage(image='nvidia/cuda:10.0-cudnn7-runtime-ubuntu16.04',
_bootstrap='docker')
stage0 += shell(
commands=['rm /bin/sh', 'ln -s /bin/bash /bin/sh', '/bin/bash'])
stage0 += packages(apt=[
'build-essential', 'cmake', 'git', 'vim', 'wget', 'ca-certificates',
'bzip2', 'libx11-6', 'libjpeg-dev', 'libpng-dev', 'bc', 'tar', 'zip',
'gawk', 'tcsh', 'time', 'libgomp1', 'libglu1-mesa', 'libglu1-mesa-dev',
'perl-modules'
])
stage0 += comment(
"https://github.com/Deep-MI/FastSurfer/blob/master/Docker/Dockerfile")
stage0 += shell(commands=[
f"wget -qO- https://surfer.nmr.mgh.harvard.edu/pub/dist/freesurfer/{freesurfer_version}/freesurfer-Linux-centos6_x86_64-stable-pub-v{freesurfer_version}.tar.gz | tar zxv --no-same-owner -C /opt \
--exclude='freesurfer/trctrain' \
--exclude='freesurfer/subjects/fsaverage_sym' \
--exclude='freesurfer/subjects/fsaverage3' \
--exclude='freesurfer/subjects/fsaverage4' \
--exclude='freesurfer/subjects/fsaverage5' \
--exclude='freesurfer/subjects/fsaverage6' \
--exclude='freesurfer/subjects/cvs_avg35' \
--exclude='freesurfer/subjects/cvs_avg35_inMNI152' \
--exclude='freesurfer/subjects/bert' \
--exclude='freesurfer/subjects/V1_average' \
--exclude='freesurfer/average/mult-comp-cor' \
--exclude='freesurfer/lib/cuda' \
--exclude='freesurfer/lib/qt'",
])
stage0 += conda(
channels=['pytorch', 'conda-forge'],
packages=[
f'python={python_version}', 'scipy', 'numpy', 'matplotlib', 'h5py',
'scikit-image', 'pytorch', 'cudatoolkit=10.0',
'pytorch=1.2.0=py3.6_cuda10.0.130_cudnn7.6.2_0',
'torchvision=0.4.0', 'scikit-sparse', 'nibabel=2.5.1',
'pillow=7.1.1'
],
eula=True,
)
stage0 += generic_build(
repository='https://github.com/Deep-MI/FastSurfer.git',
# branch='c5894bd',
build=['git checkout c5894bd'],
install=['mkdir /fastsurfer && mv * /fastsurfer/'])
stage0 += environment(
variables={
'OS': 'Linux',
'FS_OVERRIDE': 0,
'FIX_VERTEX_AREA': '',
'SUBJECTS_DIR': '/opt/freesurfer/subjects',
'FSF_OUTPUT_FORMAT': 'nii.gz',
'MNI_DIR': '/opt/freesurfer/mni',
'LOCAL_DIR': '/opt/freesurfer/local',
'FREESURFER_HOME': '/opt/freesurfer',
'FSFAST_HOME': '/opt/freesurfer/fsfast',
'PERL5LIB': '/opt/freesurfer/mni/lib/perl5/5.8.5',
'MNI_PERL4LIB': '/opt/freesurfer/mni/lib/perl5/5.8.5',
'PYTHONNUMBUFFERED': 0,
'PATH':
'/fastsurfer/recon_surf:/fastsurfer:/usr/local/anaconda/bin:/opt/freesurfer/bin:/opt/freesurfer/fsfast/bin:/opt/freesurfer/tktools:/opt/freesurfer/mni/bin:$PATH',
**from_prefix('/usr/local/cuda')
})
stage0 += runscript(commands=[
'cd /fastsurfer', 'source /usr/local/anaconda/etc/profile.d/conda.sh',
'source $FREESURFER_HOME/SetUpFreeSurfer.sh', "$*"
])
return stage0
def add_alpaka_dep_layer(stage: hpccm.Stage,
ubuntu_version: str,
cuda_support: bool,
extra_compiler: List[str],
alpaka=False) -> bool:
"""Add all dependencies to an hpccm stage that are necessary to build and run Alpaka.
:param stage: At least a baseimage
:type stage: hpccm.Stage
:param ubuntu_version: Ubuntu version number: '16.04' or '18.04'
:type ubuntu_version: str
:param cuda_support: Set True, if the Stage supports CUDA
:type cuda_support: bool
:param extra_compiler: List of compilers, which are installed additional to the system compiler. Supported are: gcc:[5-9], clang:[5.0-7.0, 8-9]
:type extra_compiler: str
:param alpaka: install alpaka in /usr/local
:type alpaka: bool
:returns: Returns True if function was successful
:rtype: bool
"""
if ubuntu_version != '16.04' and ubuntu_version != '18.04':
print('not supported Ubuntu version: ' + ubuntu_version,
file=sys.stderr)
print('supported are: 16.04, 18.04', file=sys.stderr)
return False
apt_package_list = [
'gcc', 'g++', 'make', 'software-properties-common', 'wget',
'libc6-dev', 'libomp-dev', 'unzip', 'git'
]
if ubuntu_version == '16.04':
apt_package_list.append('gnupg-agent')
if ubuntu_version == '18.04':
apt_package_list.append('gpg-agent')
stage += packages(ospackages=apt_package_list)
stage += cmake(eula=True, version='3.16.0')
# install extra compiler
if extra_compiler is not None:
for com in extra_compiler:
if com.startswith('gcc'):
stage += gnu(extra_repository=True, version=com[len('gcc:'):])
if com.startswith('clang'):
add_clang(stage, ubuntu_version, version=com[len('clang:'):])
#install boost
stage += shell(
commands=['add-apt-repository -y ppa:mhier/libboost-latest'])
stage += packages(ospackages=['boost1.67'])
if cuda_support:
stage += environment(
variables={
'LD_LIBRARY_PATH': '$LD_LIBRARY_PATH:/usr/local/cuda/lib64'
})
# alpaka use a function direct from the cuda driver library
# in the container, the cuda libraries are not at the default path
stage += environment(
variables={
'LIBRARY_PATH': '$LIBRARY_PATH:/usr/local/cuda/lib64/stubs'
})
stage += environment(
variables={'CMAKE_PREFIX_PATH': '/usr/local/cuda/lib64/stubs/'})
if alpaka:
git_alpaka = git()
cmake_alpaka = CMakeBuild()
alpaka_commands = []
alpaka_commands.append(
git_alpaka.clone_step(
repository='https://github.com/alpaka-group/alpaka.git',
path='/opt'))
alpaka_commands.append(
cmake_alpaka.configure_step(
build_directory='build',
directory='/opt/alpaka',
opts=['-Dalpaka_BUILD_EXAMPLES=OFF', '-DBUILD_TESTING=OFF']))
alpaka_commands.append(cmake_alpaka.build_step(target='install'))
alpaka_commands.append('rm -rf /opt/alpaka')
stage += shell(commands=alpaka_commands)
return True
def build(container_format='singularity',
os_release='ubuntu',
os_version='20.04'):
config.set_container_format(container_format)
image = f'{os_release}:{os_version}'
stage0 = Stage(name='stage0')
stage0 += baseimage(image=image, _bootstrap='docker')
stage0 += environment(variables={
'LC_ALL': 'en_AU.UTF-8',
'LANGUAGE': 'en_AU.UTF-8',
})
stage0 += label(metadata={
'maintainer': 'Luhan Cheng',
'email': '*****@*****.**'
})
stage0 += shell(commands=[
'rm /usr/bin/sh', 'ln -s /usr/bin/bash /usr/bin/sh', '/usr/bin/bash'
])
stage0 += packages(apt=[
'wget', 'git', 'software-properties-common', 'build-essential',
'locales', 'zlib1g-dev', 'perl', 'ncbi-blast+', 'libx11-dev'
])
stage0 += shell(commands=['locale-gen en_AU.UTF-8'])
stage0 += gnu(version='10')
stage0 += generic_build(
url=
'http://search.cpan.org/CPAN/authors/id/Y/YA/YANICK/Parallel-ForkManager-1.19.tar.gz',
prefix='/usr/local/parallel-forkmanager',
build=['perl Makefile.PL', 'make install'],
install=['mv * /usr/local/parallel-forkmanager/'])
stage0 += environment(
variables=add_binary('/usr/local/parallel-forkmanager'))
stage0 += generic_build(
url=
'ftp://ftp.ebi.ac.uk/pub/software/clustalw2/2.1/clustalw-2.1-linux-x86_64-libcppstatic.tar.gz',
prefix='/usr/local/clustalw',
install=['mv clustalw2 /usr/local/clustalw/clustalw'])
stage0 += environment(variables=add_binary('/usr/local/clustalw'))
stage0 += generic_autotools(
url='ftp://emboss.open-bio.org/pub/EMBOSS/EMBOSS-6.6.0.tar.gz',
prefix='/usr/local/emboss',
)
stage0 += environment(variables=from_prefix('/usr/local/emboss'))
stage0 += generic_autotools(
url=
'https://www.tbi.univie.ac.at/RNA/download/sourcecode/2_4_x/ViennaRNA-2.4.17.tar.gz',
prefix='/usr/local/viennarna')
stage0 += environment(variables=from_prefix('/usr/local/viennarna'))
stage0 += generic_build(
url='https://github.com/weizhongli/cdhit/archive/V4.8.1.tar.gz',
prefix='/usr/local/cdhit',
directory='cdhit-4.8.1',
build=['make'],
install=['mv * /usr/local/cdhit/'])
stage0 += environment(variables=add_binary('/usr/local/cdhit'))
stage0 += generic_build(
repository='https://github.com/ambarishbiswas/CRISPRDetect_2.2.git',
commit='0f8249f',
prefix='/usr/local/crisprdetect',
install=['mv * /usr/local/crisprdetect/'])
stage0 += environment(variables=add_binary('/usr/local/crisprdetect'))
return stage0
def build(container_format='singularity',
os='ubuntu20.04',
cuda_version='11.0',
gpu_version='V100',
mkl_version='2020.0-088',
version='psmp',
tag='8.1.0'):
image = f'nvcr.io/nvidia/cuda:{cuda_version}-devel-{os}'
config.set_container_format(container_format)
stage0 = Stage(name='stage0')
stage0 += baseimage(image=image, _bootstrap='docker')
stage0 += label(metadata={
'maintainer': 'Luhan Cheng',
'email': '*****@*****.**'
})
stage0 += comment(
'Toolchain installation translated from https://github.com/cp2k/cp2k/blob/master/tools/toolchain/Dockerfile.cuda_mkl'
)
stage0 += packages(apt=[
'git', 'gfortran', 'mpich', 'libmpich-dev',
'software-properties-common', 'python-dev'
])
stage0 += python()
stage0 += shell(commands=[
f'git clone --depth 1 --branch v{tag} https://github.com/cp2k/cp2k.git /cp2k',
'cd /cp2k', 'git submodule update --init --recursive'
])
stage0 += shell(
commands=['/cp2k/tools/toolchain/install_requirements_ubuntu.sh'])
stage0 += environment(
variables={
'CUDA_PATH': '/usr/local/cuda',
'LD_LIBRARY_PATH': '/usr/local/cuda/:${LD_LIBRARY_PATH}',
'MKLROOT': '/opt/intel/compilers_and_libraries/linux/mkl',
**from_prefix('/usr/local/cuda')
})
stage0 += mkl(eula=True, version=mkl_version)
stage0 += workdir(directory='/opt/cp2k-toolchain')
stage0 += shell(commands=[
'cd /opt/cp2k-toolchain',
'mkdir scripts && cp -r /cp2k/tools/toolchain/scripts/* ./scripts/',
'cp /cp2k/tools/toolchain/install_cp2k_toolchain.sh .',
])
stage0 += shell(commands=[
'cd /opt/cp2k-toolchain',
f'./install_cp2k_toolchain.sh --mpi-mode=openmpi --math-mode=mkl --with-reflapack=no --with-scalapack=no --with-elpa=no --gpu-ver={gpu_version}',
'rm -rf ./build'
])
stage0 += shell(commands=[
'cp /opt/cp2k-toolchain/install/arch/* /cp2k/arch/', 'cd /cp2k',
"sed 's/source /. /g' /opt/cp2k-toolchain/install/setup > /opt/cp2k-toolchain/install/setup ",
'. /opt/cp2k-toolchain/install/setup',
f'make ARCH=local VERSION="{version}" | tee /var/tmp/log.txt'
])
stage0 += environment(variables={'PATH': '/cp2k/exe/local:$PATH'})
stage0 += shell(commands=[
'ln -s /usr/lib/x86_64-linux-gnu/libncursesw.so.6 /usr/lib/x86_64-linux-gnu/libncursesw.so.5',
'ln -s /usr/lib/x86_64-linux-gnu/libtinfo.so.6 /usr/lib/x86_64-linux-gnu/libtinfo.so.5'
])
stage0 += environment(
variables={
'LD_LIBRARY_PATH': '/usr/lib/x86_64-linux-gnu/:$LD_LIBRARY_PATH'
})
return stage0
def build(container_format='singularity',
os_release='ubuntu',
os_version='20.04',
cuda_version='11.0.3'):
config.set_container_format(container_format)
# image = f'{os_release}:{os_version}'
image = f'nvcr.io/nvidia/cuda:{cuda_version}-devel-{os_release}{os_version}'
stage0 = Stage(name='stage0')
stage0 += baseimage(image=image, _bootstrap='docker')
stage0 += environment(variables={
'LC_ALL': 'en_AU.UTF-8',
'LANGUAGE': 'en_AU.UTF-8',
})
stage0 += label(metadata={
'maintainer': 'Luhan Cheng',
'email': '*****@*****.**'
})
stage0 += shell(commands=[
'rm -f /bin/sh && ln -s /bin/bash /bin/sh',
'rm -f /usr/bin/sh && ln -s /usr/bin/bash /usr/bin/sh',
'/bin/bash',
])
stage0 += packages(apt=[
'wget', 'git', 'software-properties-common', 'build-essential',
'locales', 'zlib1g-dev'
])
stage0 += shell(commands=['locale-gen en_AU.UTF-8'])
stage0 += packages(apt=[
'freeglut3-dev', 'build-essential', 'libx11-dev', 'libxmu-dev',
'libxi-dev', 'libglu1-mesa', 'libglu1-mesa-dev', 'mesa-utils',
'libgl1-mesa-glx'
])
stage0 += environment(variables=from_prefix('/usr/local/cuda'))
stage0 += comment('Installing vglrun and TurboVNC')
stage0 += packages(apt=[
'ubuntu-desktop', 'vim', 'mesa-utils', 'python3-pip', 'python3-pyqt5',
'pyqt5-dev', 'python3-tk'
])
stage0 += shell(commands=[
'wget https://swift.rc.nectar.org.au/v1/AUTH_810/CVL-Singularity-External-Files/turbovnc_2.2.5_amd64.deb && dpkg -i turbovnc_2.2.5_amd64.deb && rm turbovnc_2.2.5_amd64.deb',
'wget https://swift.rc.nectar.org.au/v1/AUTH_810/CVL-Singularity-External-Files/virtualgl_2.6.4_amd64.deb && dpkg -i virtualgl_2.6.4_amd64.deb && rm virtualgl_2.6.4_amd64.deb',
'apt update', 'apt -y upgrade'
])
stage0 += packages(apt=[
'make', 'libopenmpi-dev', 'python3-tk', 'libfftw3-dev', 'libhdf5-dev',
'libtiff-dev', 'libjpeg-dev', 'libsqlite3-dev', 'openjdk-8-jdk'
])
stage0 += environment(
variables={
'LD_LIBRARY_PATH':
"$LD_LIBRARY_PATH:/usr/lib/x86_64-linux-gnu/hdf5/serial",
})
prefix = '/usr/local/scipion'
stage0 += shell(commands=[
'python3 -m pip install --user scipion-installer',
f'python3 -m scipioninstaller {prefix} -j 4'
])
stage0 += environment(variables=from_prefix(prefix))
return stage0
def main(): # pragma: no cover
cli = Cli_Args()
args = cli.parse_args()
if args.deploy != '':
args.build = True
args.convert = True
cwd = os.getcwd()
c = list(
itertools.product(args.format, args.ogs, args.pm, args.ompi,
args.cmake_args))
if not args.print and not args.cleanup:
print('Creating {} image definition(s)...'.format(len(c)))
for build in c:
__format = build[0]
ogs_version = build[1]
ogscm.config.set_package_manager(build[2])
ompi = build[3]
cmake_args = build[4].strip().split(' ')
# args checking
if len(c) > 1 and args.file != '':
print(
'--file can only be used when generating a single image definition'
)
quit(1)
if (len(c) > 1 and args.sif_file != '') or (args.sif_file != ''
and args.convert == False):
print('--sif_file can only be used when generating a single image '
'definition and --convert is given')
quit(1)
if (ogs_version == 'off' or ogs_version
== 'clean') and len(cmake_args) > 0 and cmake_args[0] != '':
cmake_args = []
print('--cmake_args cannot be used with --ogs off! Ignoring!')
if __format == 'singularity':
if args.runtime_only:
args.runtime_only = False
print(
'--runtime-only cannot be used with --format singularity! '
'Ignoring!')
if args.upload:
print('--upload cannot be used with --format singularity! '
'Ignoring!')
if args.convert:
print('--convert cannot be used with --format singularity! '
'Ignoring!')
info = container_info(build, args)
if args.cleanup:
info.cleanup()
exit(0)
info.make_dirs()
if ompi != 'off':
if args.base_image == 'ubuntu:20.04':
args.base_image = 'centos:8'
print(
'Setting base_image to \'centos:8\'. OpenMPI is supported on CentOS only.'
)
# Create definition
hpccm.config.set_container_format(__format)
# ------------------------------ recipe -------------------------------
Stage0 = hpccm.Stage()
Stage0 += raw(docker='# syntax=docker/dockerfile:experimental')
if args.runtime_only:
Stage0.name = 'build'
Stage0 += baseimage(image=args.base_image, _as='build')
Stage0 += comment(f"Generated with ogs-container-maker {__version__}",
reformat=False)
Stage0 += packages(ospackages=['wget', 'tar', 'curl', 'make'])
# base compiler
if args.compiler != 'off':
if args.compiler_version == '':
if args.compiler == 'clang':
args.compiler_version = '8'
else:
if hpccm.config.g_linux_distro == linux_distro.CENTOS:
args.compiler_version = '10' # required for std::filesystem
else:
args.compiler_version = None # Use default
if args.compiler == 'clang':
compiler = llvm(extra_repository=True,
extra_tools=True,
version=args.compiler_version)
else:
compiler = gnu(fortran=False,
extra_repository=True,
version=args.compiler_version)
toolchain = compiler.toolchain
Stage0 += compiler
# Upgrade stdc++ lib after installing new compiler
# https://stackoverflow.com/a/46613656/80480
if args.compiler == 'gcc' and args.compiler_version is not None:
Stage0 += packages(apt=['libstdc++6'])
# Prepare runtime stage
Stage1 = hpccm.Stage()
Stage1.baseimage(image=args.base_image)
# Install scif in all stages
Stage0 += pip(packages=['scif'], pip='pip3')
Stage1 += pip(packages=['scif'], pip='pip3')
if ompi != 'off':
mpicc = object
if False: # eve:
# Stage0 += ofed() OR mlnx_ofed(); is installed later on from debian archive
# Stage0 += knem()
Stage0 += ucx(version='1.5.1',
cuda=False) # knem='/usr/local/knem'
Stage0 += packages(ospackages=['libpmi2-0-dev'
]) # req. for --with-pmi
# req. for --with-psm2
Stage0 += packages(ospackages=['libnuma1'])
psm_deb_url = 'http://snapshot.debian.org/archive/debian/20181231T220010Z/pool/main'
psm2_version = '11.2.68-4'
Stage0 += shell(commands=[
'cd /tmp',
f'wget -nv {psm_deb_url}/libp/libpsm2/libpsm2-2_{psm2_version}_amd64.deb',
f'wget -nv {psm_deb_url}/libp/libpsm2/libpsm2-dev_{psm2_version}_amd64.deb',
'dpkg --install *.deb'
])
# libibverbs
# Available versions: http://snapshot.debian.org/binary/ibacm/
# ibverbs_version = '21.0-1'
# works on eve, eve has 17.2-3 installed nut this version is not available in snapshot.debian
ib_deb_url = 'http://snapshot.debian.org/archive/debian/20180430T215634Z/pool/main'
ibverbs_version = '17.1-2'
ibverbs_packages = [
'ibacm', 'ibverbs-providers', 'ibverbs-utils',
'libibumad-dev', 'libibumad3', 'libibverbs-dev',
'libibverbs1', 'librdmacm-dev', 'librdmacm1', 'rdma-core',
'rdmacm-utils'
]
ibverbs_cmds = ['cd /tmp']
for package in ibverbs_packages:
ibverbs_cmds.extend([
f'wget -nv {ib_deb_url}/r/rdma-core/{package}_{ibverbs_version}_amd64.deb'
])
ibverbs_cmds.append('dpkg --install *.deb')
Stage0 += packages(ospackages=[
'libnl-3-200', 'libnl-route-3-200', 'libnl-route-3-dev',
'udev', 'perl'
])
Stage0 += shell(commands=ibverbs_cmds)
mpicc = openmpi(
version=ompi,
cuda=False,
toolchain=toolchain,
ldconfig=True,
ucx='/usr/local/ucx',
configure_opts=[
'--disable-getpwuid',
'--sysconfdir=/mnt/0',
'--with-slurm', # used on taurus
'--with-pmi=/usr/include/slurm-wlm',
'CPPFLAGS=\'-I /usr/include/slurm-wlm\'',
'--with-pmi-libdir=/usr/lib/x86_64-linux-gnu',
# '--with-pmix',
'--with-psm2',
'--disable-pty-support',
'--enable-mca-no-build=btl-openib,plm-slurm',
# eve:
'--with-sge',
'--enable-mpirun-prefix-by-default',
'--enable-orterun-prefix-by-default',
])
else:
ucx_version = '1.8.1'
Stage0 += ucx(version=ucx_version, cuda=False)
Stage0 += slurm_pmi2(version='17.02.11')
pmix_version = True
if version.parse(ompi) >= version.parse('4'):
Stage0 += pmix(version='3.1.5')
pmix_version = '/usr/local/pmix'
mpicc = openmpi(version=ompi,
cuda=False,
infiniband=False,
pmi='/usr/local/slurm-pmi2',
pmix=pmix_version,
ucx='/usr/local/ucx')
toolchain = mpicc.toolchain
Stage0 += mpicc
# OpenMPI expects this program to exist, even if it's not used.
# Default is "ssh : rsh", but that's not installed.
Stage0 += shell(commands=[
'mkdir /mnt/0',
"echo 'plm_rsh_agent = false' >> /mnt/0/openmpi-mca-params.conf"
])
Stage0 += label(
metadata={
'org.opengeosys.mpi': 'openmpi',
'org.opengeosys.mpi.version': ompi
})
if args.mpi_benchmarks:
# osu_app = scif(name='osu', file=f"{info.out_dir}/osu.scif")
Stage0 += osu_benchmarks(toolchain=toolchain)
Stage0 += shell(commands=[
'mkdir -p /usr/local/mpi-examples',
'cd /usr/local/mpi-examples',
'curl -O https://raw.githubusercontent.com/hpc/charliecloud/674b3b4e4ad243be5565f200d8f5fb92b7544480/examples/mpihello/hello.c',
'curl -O https://computing.llnl.gov/tutorials/mpi/samples/C/mpi_bandwidth.c',
'curl -O https://raw.githubusercontent.com/mpitutorial/mpitutorial/gh-pages/tutorials/mpi-send-and-receive/code/ring.c',
'mpicc -o /usr/local/bin/mpi-hello /usr/local/mpi-examples/hello.c',
'mpicc -o /usr/local/bin/mpi-ring /usr/local/mpi-examples/ring.c',
'mpicc -o /usr/local/bin/mpi-bandwidth /usr/local/mpi-examples/mpi_bandwidth.c',
])
Stage1 += copy(_from='build',
src='/usr/local/bin/mpi-*',
dest='/usr/local/bin/')
# Stage0 += mlnx_ofed()
if ogs_version != 'clean':
Stage0 += ogs_base()
if args.gui:
Stage0 += packages(apt=[
'mesa-common-dev', 'libgl1-mesa-dev', 'libglu1-mesa-dev',
'libxt-dev'
],
yum=[
'mesa-libOSMesa-devel', 'mesa-libGL-devel',
'mesa-libGLU-devel', 'libXt-devel'
])
Stage1 += packages(
apt=[
'libosmesa6', 'libgl1-mesa-glx', 'libglu1-mesa', 'libxt6',
'libopengl0'
],
yum=['mesa-libOSMesa', 'mesa-libGL', 'mesa-libGLU', 'libXt'])
if ogs_version != 'clean':
if ogscm.config.g_package_manager == package_manager.CONAN:
Stage0 += cmake(eula=True, version='3.16.6')
conan_user_home = '/opt/conan'
if args.dev:
conan_user_home = ''
Stage0 += pm_conan(user_home=conan_user_home)
Stage0 += environment(variables={'CONAN_SYSREQUIRES_SUDO': 0})
elif ogscm.config.g_package_manager == package_manager.SYSTEM:
Stage0 += cmake(eula=True, version='3.16.6')
Stage0 += boost(version='1.66.0', bootstrap_opts=['headers'])
Stage0 += environment(
variables={'BOOST_ROOT': '/usr/local/boost'})
vtk_cmake_args = [
'-DModule_vtkIOXML=ON', '-DModule_vtkIOXdmf3=ON',
'-DVTK_Group_Rendering=OFF', '-DVTK_Group_StandAlone=OFF'
]
if args.gui:
Stage0 += packages(apt=[
'libgeotiff-dev', 'libshp-dev', 'libnetcdf-c++4-dev',
'libqt5x11extras5-dev', 'libqt5xmlpatterns5-dev',
'qt5-default'
],
yum=[
'libgeotiff-devel',
'shapelib-devel', 'netcdf-devel',
'qt5-qtbase-devel',
'qt5-qtxmlpatterns-devel',
'qt5-qtx11extras-devel'
])
Stage1 += packages(apt=[
'geotiff-bin', 'shapelib', 'libnetcdf-c++4',
'libqt5x11extras5', 'libqt5xmlpatterns5', 'qt5-default'
],
yum=[
'libgeotiff', 'shapelib', 'netcdf',
'qt5-qtbase', 'qt5-qtxmlpatterns',
'qt5-qtx11extras'
])
vtk_cmake_args = [
'-DModule_vtkIOXdmf3=ON',
'-DVTK_BUILD_QT_DESIGNER_PLUGIN=OFF',
'-DVTK_Group_Qt=ON', '-DVTK_QT_VERSION=5'
]
if hpccm.config.g_linux_distro == linux_distro.CENTOS:
# otherwise linker error, maybe due to gcc 10?
vtk_cmake_args.extend([
'-DBUILD_SHARED_LIBS=OFF',
'-DCMAKE_POSITION_INDEPENDENT_CODE=ON'
])
if args.insitu:
if args.gui:
print('--gui can not be used with --insitu!')
exit(1)
Stage0 += paraview(cmake_args=['-DPARAVIEW_USE_PYTHON=ON'],
edition='CATALYST',
ldconfig=True,
toolchain=toolchain,
version="v5.8.1")
else:
if toolchain.CC == 'mpicc':
vtk_cmake_args.extend([
'-D Module_vtkIOParallelXML=ON',
'-D Module_vtkParallelMPI=ON'
])
Stage0 += generic_cmake(
cmake_opts=vtk_cmake_args,
devel_environment={'VTK_ROOT': '/usr/local/vtk'},
directory='VTK-8.2.0',
ldconfig=True,
prefix='/usr/local/vtk',
toolchain=toolchain,
url=
'https://www.vtk.org/files/release/8.2/VTK-8.2.0.tar.gz'
)
if ompi != 'off':
Stage0 += packages(yum=['diffutils'])
Stage0 += generic_autotools(
configure_opts=[
f'CC={toolchain.CC}', f'CXX={toolchain.CXX}',
'--CFLAGS=\'-O3\'', '--CXXFLAGS=\'-O3\'',
'--FFLAGS=\'-O3\'', '--with-debugging=no',
'--with-fc=0', '--download-f2cblaslapack=1'
],
devel_environment={'PETSC_DIR': '/usr/local/petsc'},
directory='petsc-3.11.3',
ldconfig=True,
preconfigure=[
"sed -i -- 's/python/python3/g' configure"
],
prefix='/usr/local/petsc',
toolchain=toolchain,
url='http://ftp.mcs.anl.gov/pub/petsc/release-snapshots/'
'petsc-lite-3.11.3.tar.gz')
Stage0 += generic_cmake(
devel_environment={
'Eigen3_ROOT': '/usr/local/eigen',
'Eigen3_DIR': '/usr/local/eigen'
},
directory='eigen-3.3.7',
prefix='/usr/local/eigen',
url=
'https://gitlab.com/libeigen/eigen/-/archive/3.3.7/eigen-3.3.7.tar.gz'
)
if args.cvode:
Stage0 += generic_cmake(
cmake_opts=[
'-D EXAMPLES_INSTALL=OFF', '-D BUILD_SHARED_LIBS=OFF',
'-D CMAKE_POSITION_INDEPENDENT_CODE=ON'
],
devel_environment={'CVODE_ROOT': '/usr/local/cvode'},
directory='cvode-2.8.2',
prefix='/usr/local/cvode',
url='https://github.com/ufz/cvode/archive/2.8.2.tar.gz')
if args.cppcheck:
Stage0 += generic_cmake(
devel_environment={'PATH': '/usr/local/cppcheck/bin:$PATH'},
directory='cppcheck-809a769c690d8ab6fef293e41a29c8490512866e',
prefix='/usr/local/cppcheck',
runtime_environment={'PATH': '/usr/local/cppcheck/bin:$PATH'},
url=
'https://github.com/danmar/cppcheck/archive/809a769c690d8ab6fef293e41a29c8490512866e.tar.gz'
)
if args.iwyy and args.compiler == 'clang':
Stage0 += packages(ospackages=[
'libncurses5-dev', 'zlib1g-dev',
f"llvm-{args.compiler_version}-dev",
f"libclang-{args.compiler_version}-dev"
])
Stage0 += generic_cmake(
cmake_opts=[
f"-D IWYU_LLVM_ROOT_PATH=/usr/lib/llvm-{args.compiler_version}"
],
devel_environment={'PATH': '/usr/local/iwyy/bin:$PATH'},
directory=
f"include-what-you-use-clang_{args.compiler_version}.0",
prefix='/usr/local/iwyy',
runtime_environment={'PATH': '/usr/local/iwyy/bin:$PATH'},
url="https://github.com/include-what-you-use/include-what-"
f"you-use/archive/clang_{args.compiler_version}.0.tar.gz")
if args.docs:
Stage0 += packages(
ospackages=['doxygen', 'graphviz', 'texlive-base'])
if args.gcovr:
Stage0 += pip(pip='pip3', packages=['gcovr'])
if args.dev:
Stage0 += packages(ospackages=[
'neovim', 'gdb', 'silversearcher-ag', 'ssh-client', 'less'
])
if args.pip:
Stage0 += pip(packages=args.pip, pip='pip3')
Stage1 += pip(packages=args.pip, pip='pip3')
if args.packages:
Stage0 += packages(ospackages=args.packages)
if args.tfel:
Stage0 += generic_cmake(
directory='tfel-TFEL-3.3.0',
ldconfig=True,
url='https://github.com/thelfer/tfel/archive/TFEL-3.3.0.tar.gz',
prefix='/usr/local/tfel')
Stage0 += environment(variables={'TFELHOME': '/usr/local/tfel'})
definition_file_path = os.path.join(info.out_dir, info.definition_file)
if args.ccache:
Stage0 += ccache(cache_size='15G')
if ogs_version != 'off' and ogs_version != 'clean':
mount_args = ''
if args.ccache:
mount_args = f'{mount_args} --mount=type=cache,target=/opt/ccache,id=ccache'
if args.cvode:
cmake_args.append('-DOGS_USE_CVODE=ON')
if args.gui:
cmake_args.append('-DOGS_BUILD_GUI=ON')
if args.insitu:
cmake_args.append('-DOGS_INSITU=ON')
Stage0 += raw(docker=f"ARG OGS_COMMIT_HASH={info.commit_hash}")
scif_file = f"{info.out_dir}/ogs.scif"
if info.ogsdir:
context_path_size = len(ogs_version)
print(f"chdir to {ogs_version}")
os.chdir(ogs_version)
mount_args = f'{mount_args} --mount=type=bind,target=/scif/apps/ogs/src,rw'
scif_file = f"{info.out_dir[context_path_size+1:]}/ogs.scif"
definition_file_path = f"{info.out_dir[context_path_size+1:]}/{info.definition_file}"
ogs_app = scif(_arguments=mount_args, name='ogs', file=scif_file)
ogs_app += ogs(repo=info.repo,
branch=info.branch,
commit=info.commit_hash,
git_version=info.git_version,
toolchain=toolchain,
prefix='/scif/apps/ogs',
cmake_args=cmake_args,
parallel=args.parallel,
remove_build=True,
remove_source=True)
Stage0 += ogs_app
stages_string = str(Stage0)
if args.runtime_only:
Stage1 += Stage0.runtime(exclude=['boost'])
if args.compiler == 'gcc' and args.compiler_version != None:
Stage1 += packages(apt=['libstdc++6'])
stages_string += "\n\n" + str(Stage1)
# ---------------------------- recipe end -----------------------------
with open(definition_file_path, 'w') as f:
print(stages_string, file=f)
if args.print:
print(stages_string)
else:
print(
f'Created definition {os.path.abspath(definition_file_path)}')
# Create image
if not args.build:
continue
if __format == 'singularity':
subprocess.run(
f"sudo `which singularity` build --force {info.images_out_dir}/{info.img_file}.sif"
f"{definition_file_path}",
shell=True)
subprocess.run(
f"sudo chown $USER:$USER {info.images_out_dir}/{info.img_file}.sif",
shell=True)
# TODO: adapt this to else
continue
build_cmd = (f"DOCKER_BUILDKIT=1 docker build {args.build_args} "
f"-t {info.tag} -f {definition_file_path} .")
print(f"Running: {build_cmd}")
subprocess.run(build_cmd, shell=True)
inspect_out = subprocess.check_output(
f"docker inspect {info.tag} | grep Id",
shell=True).decode(sys.stdout.encoding)
image_id = re.search('sha256:(\w*)', inspect_out).group(1)
image_id_short = image_id[0:12]
if args.upload:
subprocess.run(f"docker push {info.tag}", shell=True)
if args.sif_file:
image_file = f'{info.images_out_dir}/{args.sif_file}'
else:
image_file = f'{info.images_out_dir}/{info.img_file}-{image_id_short}.sif'
if args.convert and not os.path.exists(image_file):
subprocess.run(
f"cd {cwd} && singularity build --force {image_file} docker-daemon:{info.tag}",
shell=True)
# Deploy image
if not args.deploy:
continue
deploy_config_filename = f'{cwd}/config/deploy_hosts.yml'
if not os.path.isfile(deploy_config_filename):
print(
f'ERROR: {deploy_config_filename} not found but required for deploying!'
)
exit(1)
with open(deploy_config_filename, 'r') as ymlfile:
deploy_config = yaml.load(ymlfile, Loader=yaml.FullLoader)
if not args.deploy == 'ALL' and not args.deploy in deploy_config:
print(f'ERROR: Deploy host "{args.deploy}" not found in config!')
exit(1)
deploy_hosts = {}
if args.deploy == 'ALL':
deploy_hosts = deploy_config
else:
deploy_hosts[args.deploy] = deploy_config[args.deploy]
for deploy_host in deploy_hosts:
deploy_info = deploy_hosts[deploy_host]
print(f'Deploying to {deploy_info} ...')
proxy_cmd = ''
user_cmd = ''
if 'user' in deploy_info:
user_cmd = f"{deploy_info['user']}@"
if 'proxy' in deploy_info:
proxy_cmd = f"-e 'ssh -A -J {user_cmd}{deploy_info['proxy']}'"
print(proxy_cmd)
print(
subprocess.check_output(
f"rsync -c -v {proxy_cmd} {image_file} {user_cmd}{deploy_info['host']}:{deploy_info['dest_dir']}",
shell=True).decode(sys.stdout.encoding))
