def build(container_format='singularity', os_release='ubuntu', os_version='20.04'): config.set_container_format(container_format) image = f'{os_release}:{os_version}' stage0 = Stage(name='stage0') stage0 += baseimage(image=image, _bootstrap='docker') stage0 += environment(variables={ 'LC_ALL': 'en_AU.UTF-8', 'LANGUAGE': 'en_AU.UTF-8', }) stage0 += label(metadata={ 'maintainer': 'Luhan Cheng', 'email': '*****@*****.**' }) stage0 += shell(commands=[ 'rm -f /bin/sh && ln -s /bin/bash /bin/sh', 'rm -f /usr/bin/sh && ln -s /usr/bin/bash /usr/bin/sh', '/bin/bash', 'ln -s /usr/lib/x86_64-linux-gnu/libffi.so.7 /usr/lib/x86_64-linux-gnu/libffi.so.6' ]) stage0 += packages(apt=[ 'wget', 'git', 'software-properties-common', 'build-essential', 'locales', 'zlib1g-dev' ]) stage0 += shell(commands=['locale-gen en_AU.UTF-8']) stage0 += comment('Installing vglrun and TurboVNC') stage0 += packages(apt=[ 'ubuntu-desktop', 'vim', 'mesa-utils', 'python3-pip', 'python3-pyqt5', 'pyqt5-dev', 'python3-tk' ]) stage0 += shell(commands=[ 'wget https://swift.rc.nectar.org.au/v1/AUTH_810/CVL-Singularity-External-Files/turbovnc_2.2.5_amd64.deb && dpkg -i turbovnc_2.2.5_amd64.deb && rm turbovnc_2.2.5_amd64.deb', 'wget https://swift.rc.nectar.org.au/v1/AUTH_810/CVL-Singularity-External-Files/virtualgl_2.6.4_amd64.deb && dpkg -i virtualgl_2.6.4_amd64.deb && rm virtualgl_2.6.4_amd64.deb', 'apt update', 'apt -y upgrade' ]) stage0 += primitives.copy(src='./ChimeraX-1.2.5.tar.gz', dest='/opt/ChimeraX-1.2.5.tar.gz') stage0 += shell(commands=[ 'tar xfv /opt/ChimeraX-1.2.5.tar.gz -C /opt', 'rm /opt/ChimeraX-1.2.5.tar.gz' ]) stage0 += environment( variables=from_prefix('/opt/chimerax-1.2.5-rc-2021.05.24')) return stage0
def recipe() -> str: """Generate the recipe :returns: singularity recipe :rtype: str """ hpccm.config.set_container_format('singularity') hpccm.config.set_singularity_version('3.3') stage = hpccm.Stage() stage += label(metadata={'CLING ALPAKA VERSION': str(version)}) stage += environment(variables={'CLING_ALPAKA_VERSION': version}) # the baseimage of xeus-cling-cuda is Ubuntu 16.04 with CUDA 8 if not ap_gn.add_alpaka_dep_layer(stage, '16.04', True, []): print('adding the alpaka dependencies layer failed', file=sys.stderr) exit(1) install_alpaka(stage) build_jupyter_kernel(stage) # baseimage support nothing else than dockerhub # so, manually add the baseimage command to the recipe recipe = stage.__str__() recipe = 'Bootstrap: library\nFrom: sehrig/default/xeus-cling-cuda-cxx:2.2\n\n' + recipe return recipe
def build(container_format='singularity', os_release='ubuntu', os_version='20.04'): config.set_container_format(container_format) image = f'{os_release}:{os_version}' stage0 = Stage(name='stage0') stage0 += baseimage(image=image, _bootstrap='docker') stage0 += environment(variables={ 'LC_ALL': 'en_AU.UTF-8', 'LANGUAGE': 'en_AU.UTF-8', }) stage0 += label(metadata={ 'maintainer': 'Luhan Cheng', 'email': '*****@*****.**' }) stage0 += shell(commands=[ 'rm -f /bin/sh && ln -s /bin/bash /bin/sh', 'rm -f /usr/bin/sh && ln -s /usr/bin/bash /usr/bin/sh', '/bin/bash', ]) stage0 += packages(apt=[ 'wget', 'git', 'software-properties-common', 'build-essential', 'locales', 'zlib1g-dev' ]) stage0 += shell(commands=['locale-gen en_AU.UTF-8']) return stage0
def singleStageBuild(stage): stage += label(metadata={'MAINTAINER': 'Simeon Ehrig'}) stage += label(metadata={'EMAIL': '*****@*****.**'}) stage += label(metadata={'Version': str(container_version)}) stage += environment(variables={'GOL_VERSION': str(container_version)}) # copy example inside container stage += copy(src='notebook', dest='/') # copy and build the pnwriter library stage += copy(src='pngwriter', dest='/opt') stage += packages(ospackages=['libpng-dev']) cmake = CMakeBuild(prefix='/notebook/pngwriter') stage += shell(commands=[ cmake.configure_step(directory='/opt/pngwriter', opts=['-DBUILD_SHARED_LIBS=ON']), cmake.build_step(target='install'), 'rm -rf /opt/pngwriter' ]) # Copy notebook examples and pngwriter lib to the host's /tmp file system to obtain a writable file system. stage += runscript(commands=[ 'if [ ! -d /tmp/GOL-xeus-cling-cuda ]; then \n' ' mkdir /tmp/GOL-xeus-cling-cuda &&' ' cp -r /notebook/ /tmp/GOL-xeus-cling-cuda\n fi', 'cd /tmp/GOL-xeus-cling-cuda/notebook', 'jupyter-lab' ]) print(stage.__str__())
def build(image="Characterisation-Virtual-Laboratory/CharacterisationVL-Software:2004", _bootstrap='shub', mash_version='v2.2.2', capnp_version='0.8.0', quicktree_version='v2.5'): stage0 = hpccm.Stage() hpccm.config.set_container_format("singularity") stage0 += baseimage(image=image, _bootstrap=_bootstrap, _distro='ubuntu20') stage0 += label(metadata={'maintainer': 'Luhan Cheng', 'email': '*****@*****.**'}) stage0 += packages(ospackages=['cpanminus', 'libexpat1-dev', 'sqlite3', 'libsqlite3-dev', 'autoconf']) stage0 += generic_build( repository='https://github.com/khowe/quicktree', branch=quicktree_version, build=['make'], install=[ 'mv quicktree /usr/local/bin', 'mv include/* /usr/local/include/' ], ) stage0 += boost() stage0 += generic_autotools( url=f'https://capnproto.org/capnproto-c++-{capnp_version}.tar.gz' ) stage0 += shell(commands=['cpanm -l /usr/local/perl5 --notest BioPerl Bio::Sketch::Mash DBD::SQLite DBI']) stage0 += generic_autotools( repository=f'https://github.com/marbl/Mash', preconfigure=['./bootstrap.sh'], branch=mash_version, with_capnp='/usr/local/', with_boost='/usr/local/boost/', ) stage0 += environment(variables={'PERL5LIB': '$PERL5LIB:/usr/local', **from_prefix('/usr/local/mashtree')}) stage0 += shell(commands=['cpanm -f -l /usr/local/mashtree Mashtree']) return stage0
def build(container_format='singularity', os_release='ubuntu', os_version='20.04', gpu=True, cuda_version='11.0', desktop=True): config.set_container_format(container_format) image = f'{os_release}:{os_version}' stage0 = Stage(name='stage0') stage0 += baseimage(image=image, _bootstrap='docker') stage0 += environment(variables={ 'LC_ALL': 'en_AU.UTF-8', 'LANGUAGE': 'en_AU.UTF-8', }) stage0 += label(metadata={'maintainer': 'Luhan Cheng', 'email': '*****@*****.**'}) stage0 += shell(commands=[ 'rm -f /bin/sh && ln -s /bin/bash /bin/sh', 'rm -f /usr/bin/sh && ln -s /usr/bin/bash /usr/bin/sh', '/bin/bash', ]) stage0 += packages(apt=['wget', 'git', 'software-properties-common', 'build-essential', 'locales', 'zlib1g-dev', 'dirmngr', 'gpg-agent']) stage0 += shell(commands=['locale-gen en_AU.UTF-8']) stage0 += shell(commands=[ "wget -O- http://neuro.debian.net/lists/bionic.us-ca.full | tee /etc/apt/sources.list.d/neurodebian.sources.list", "apt-key adv --recv-keys --keyserver hkps://keyserver.ubuntu.com 0xA5D32F012649A5A9 2>&1", ]) stage0 += packages(apt=["fsl-complete"]) return stage0
def build(container_format='singularity', os_release='ubuntu', os_version='20.04'): config.set_container_format(container_format) image = f'{os_release}:{os_version}' stage0 = Stage(name='stage0') stage0 += baseimage(image=image, _bootstrap='docker') stage0 += environment(variables={ 'LC_ALL': 'en_AU.UTF-8', 'LANGUAGE': 'en_AU.UTF-8', }) stage0 += label(metadata={'maintainer': 'Luhan Cheng', 'email': '*****@*****.**'}) stage0 += shell(commands=[ 'rm -f /bin/sh && ln -s /bin/bash /bin/sh', 'rm -f /usr/bin/sh && ln -s /usr/bin/bash /usr/bin/sh', '/bin/bash', ]) stage0 += packages(apt=['wget', 'git', 'software-properties-common', 'build-essential', 'locales', 'zlib1g-dev', 'bc', 'cmake', 'bison', 'flex', 'libx11-dev', 'x11proto-core-dev', 'libxi6', 'libxi-dev', 'libxmu6', 'libxmu-dev', 'libxmu-headers', 'libgl1-mesa-dev', 'libglu1-mesa-dev', 'libjpeg-dev', 'libevent-dev', 'libncurses-dev', 'pkg-config', 'libopenblas-dev']) stage0 += shell(commands=['locale-gen en_AU.UTF-8']) stage0 += comment('Installing vglrun and TurboVNC') stage0 += packages(apt=['ubuntu-desktop', 'vim', 'mesa-utils', 'python3-pip', 'python3-pyqt5', 'pyqt5-dev', 'python3-tk']) stage0 += shell(commands=[ 'wget https://swift.rc.nectar.org.au/v1/AUTH_810/CVL-Singularity-External-Files/turbovnc_2.2.5_amd64.deb && dpkg -i turbovnc_2.2.5_amd64.deb && rm turbovnc_2.2.5_amd64.deb', 'wget https://swift.rc.nectar.org.au/v1/AUTH_810/CVL-Singularity-External-Files/virtualgl_2.6.4_amd64.deb && dpkg -i virtualgl_2.6.4_amd64.deb && rm virtualgl_2.6.4_amd64.deb', 'apt update', 'apt -y upgrade' ]) stage0 += generic_cmake( repository='https://github.com/BIC-MNI/minc-toolkit-v2.git', branch='release-1.9.18.1', recursive=True, prefix='/opt/minc/1.9.18', preconfigure=['sed -i -e \'s/http\:\/\/mirrors\.ibiblio\.org\/gnu\/ftp\/gnu\/gsl/http\:\/\/ftp\.\sun\.ac\.za\/ftp\/pub\/mirrors\/ftp\.gnu\.org\/gsl/g\' cmake-modules/BuildGSL.cmake', ], cmake_opts=['-DCMAKE_BUILD_TYPE:STRING=Release', '-DCMAKE_INSTALL_PREFIX:PATH=/opt/minc/1.9.18', '-DMT_BUILD_ABC:BOOL=ON', '-DMT_BUILD_ANTS:BOOL=ON', '-DMT_BUILD_C3D:BOOL=ON', '-DMT_BUILD_ELASTIX:BOOL=ON', '-DMT_BUILD_IM:BOOL=OFF', '-DMT_BUILD_ITK_TOOLS:BOOL=ON', '-DMT_BUILD_LITE:BOOL=OFF', '-DMT_BUILD_SHARED_LIBS:BOOL=ON', '-DMT_BUILD_VISUAL_TOOLS:BOOL=ON', '-DMT_USE_OPENMP:BOOL=ON', '-DUSE_SYSTEM_FFTW3D:BOOL=OFF', '-DUSE_SYSTEM_FFTW3F:BOOL=OFF', '-DUSE_SYSTEM_GLUT:BOOL=OFF', '-DUSE_SYSTEM_GSL:BOOL=OFF', '-DUSE_SYSTEM_HDF5:BOOL=OFF', '-DUSE_SYSTEM_ITK:BOOL=OFF', '-DUSE_SYSTEM_NETCDF:BOOL=OFF', '-DUSE_SYSTEM_NIFTI:BOOL=OFF', '-DUSE_SYSTEM_PCRE:BOOL=OFF', '-DUSE_SYSTEM_ZLIB:BOOL=OFF'] ) stage0 += environment(variables=from_prefix('/opt/minc/1.9.18.1')) return stage0
def build(container_format='singularity', os_release='ubuntu', os_version='20.04'): config.set_container_format(container_format) image = f'{os_release}:{os_version}' stage0 = Stage(name='stage0') stage0 += baseimage(image=image, _bootstrap='docker') stage0 += environment(variables={ 'LC_ALL': 'en_AU.UTF-8', 'LANGUAGE': 'en_AU.UTF-8', }) stage0 += label(metadata={ 'maintainer': 'Luhan Cheng', 'email': '*****@*****.**' }) stage0 += shell(commands=[ 'rm -f /bin/sh && ln -s /bin/bash /bin/sh', 'rm -f /usr/bin/sh && ln -s /usr/bin/bash /usr/bin/sh', '/bin/bash', ]) stage0 += packages(apt=[ 'wget', 'git', 'software-properties-common', 'build-essential', 'locales', 'zlib1g-dev' ]) stage0 += shell(commands=['locale-gen en_AU.UTF-8']) stage0 += comment('Installing vglrun and TurboVNC') stage0 += packages(apt=[ 'ubuntu-desktop', 'vim', 'mesa-utils', 'python3-pip', 'python3-pyqt5', 'pyqt5-dev', 'python3-tk' ]) stage0 += shell(commands=[ 'wget https://swift.rc.nectar.org.au/v1/AUTH_810/CVL-Singularity-External-Files/turbovnc_2.2.5_amd64.deb && dpkg -i turbovnc_2.2.5_amd64.deb && rm turbovnc_2.2.5_amd64.deb', 'wget https://swift.rc.nectar.org.au/v1/AUTH_810/CVL-Singularity-External-Files/virtualgl_2.6.4_amd64.deb && dpkg -i virtualgl_2.6.4_amd64.deb && rm virtualgl_2.6.4_amd64.deb', 'apt update', 'apt -y upgrade' ]) # coin3d stage0 += packages(apt=[ 'mesa-common-dev', 'libglu1-mesa-dev', 'freeglut3-dev', 'autoconf', 'autogen', 'doxygen' ]) compiler = gnu(version="10") stage0 += compiler stage0 += boost() stage0 += generic_cmake( repository="https://github.com/coin3d/coin", recursive=True, cmake_opts=[ '-G "Unix Makefiles"', "-DCMAKE_BUILD_TYPE=Release", "-DCOIN_BUILD_DOCUMENTATION=OFF", "-DCMAKE_INSTALL_PREFIX=/usr/local/coin3d" ], postinstall=['cd cpack.d', 'cpack --config debian.cmake']) return stage0
def build(container_format='singularity', os_release='ubuntu', os_version='20.04', cuda_version='11.0'): config.set_container_format(container_format) image = f'nvcr.io/nvidia/cuda:{cuda_version}-devel-{os_release}{os_version}' stage0 = Stage(name='stage0') stage0 += baseimage(image=image, _bootstrap='docker') stage0 += environment(variables={ 'LC_ALL': 'en_AU.UTF-8', 'LANGUAGE': 'en_AU.UTF-8', }) stage0 += label(metadata={ 'maintainer': 'Luhan Cheng', 'email': '*****@*****.**' }) stage0 += shell(commands=[ 'rm -f /bin/sh && ln -s /bin/bash /bin/sh', 'rm -f /usr/bin/sh && ln -s /usr/bin/bash /usr/bin/sh', '/bin/bash', ]) stage0 += environment(variables=from_prefix('/usr/local/cuda')) stage0 += packages(apt=[ 'wget', 'git', 'software-properties-common', 'build-essential', 'locales', 'zlib1g-dev' ]) stage0 += shell(commands=['locale-gen en_AU.UTF-8']) stage0 += comment('Installing vglrun and TurboVNC') stage0 += packages(apt=[ 'ubuntu-desktop', 'vim', 'mesa-utils', 'python3-pip', 'python3-pyqt5', 'pyqt5-dev', 'python3-tk' ]) stage0 += shell(commands=[ 'wget https://swift.rc.nectar.org.au/v1/AUTH_810/CVL-Singularity-External-Files/turbovnc_2.2.5_amd64.deb && dpkg -i turbovnc_2.2.5_amd64.deb && rm turbovnc_2.2.5_amd64.deb', 'wget https://swift.rc.nectar.org.au/v1/AUTH_810/CVL-Singularity-External-Files/virtualgl_2.6.4_amd64.deb && dpkg -i virtualgl_2.6.4_amd64.deb && rm virtualgl_2.6.4_amd64.deb', 'apt update', 'apt -y upgrade' ]) stage0 += conda(eula=True, channels=['schrodinger'], packages=['pymol-bundle=2.4.1']) stage0 += environment(variables=from_prefix('/usr/local/anaconda')) return stage0
def build(version='2.1.1'): stage0 = CVL_ubuntu_stage() stage0 += generic_build( repository='https://github.com/DerrickWood/kraken2.git', branch=f'v{version}', install=['./install_kraken2.sh /usr/local/kraken2'], ) stage0 += environment(variables=add_binary('/usr/local/kraken2/bin')) return stage0
def build(container_format='singularity', os_release='ubuntu', os_version='20.04'): config.set_container_format(container_format) image = f'{os_release}:{os_version}' stage0 = Stage(name='stage0') stage0 += baseimage(image=image, _bootstrap='docker') stage0 += environment(variables={ 'LC_ALL': 'en_AU.UTF-8', 'LANGUAGE': 'en_AU.UTF-8', }) stage0 += label(metadata={'maintainer': 'Luhan Cheng', 'email': '*****@*****.**'}) stage0 += shell(commands=['rm /usr/bin/sh', 'ln -s /usr/bin/bash /usr/bin/sh', '/usr/bin/bash']) stage0 += packages(apt=['wget', 'git', 'software-properties-common', 'build-essential', 'locales', 'zlib1g-dev']) stage0 += shell(commands=['locale-gen en_AU.UTF-8']) stage0 += comment('Installing vglrun and TurboVNC') stage0 += packages(apt=['ubuntu-desktop', 'vim', 'mesa-utils', 'python3-pip', 'python3-pyqt5', 'pyqt5-dev', 'python3-tk']) stage0 += shell(commands=[ 'wget https://swift.rc.nectar.org.au/v1/AUTH_810/CVL-Singularity-External-Files/turbovnc_2.2.5_amd64.deb && dpkg -i turbovnc_2.2.5_amd64.deb && rm turbovnc_2.2.5_amd64.deb', 'wget https://swift.rc.nectar.org.au/v1/AUTH_810/CVL-Singularity-External-Files/virtualgl_2.6.4_amd64.deb && dpkg -i virtualgl_2.6.4_amd64.deb && rm virtualgl_2.6.4_amd64.deb', 'apt update', 'apt -y upgrade' ]) stage0 += shell(commands=[ 'curl -sL https://deb.nodesource.com/setup_12.x | bash -', 'apt install -y nodejs npm' ]) prefix = '/var/www' stage0 += shell(commands=[ f'mkdir {prefix} && cd {prefix}', 'git clone --branch 1.16.10-release https://github.com/gmod/jbrowse .', './setup.sh', ]) stage0 += environment(variables={ 'PATH': f'{prefix}/bin:$PATH' }) stage0 += runscript(commands=[ f'cd {prefix}', 'npm run start $*' ]) return stage0
def build(version='1.3'): stage0 = CVL_ubuntu_stage() stage0 += generic_build( repository='https://github.com/lh3/seqtk.git', branch=f'v{version}', build=['make'], install=[ 'mkdir -p /usr/local/seqtk/bin && cp seqtk /usr/local/seqtk/bin/' ]) stage0 += environment(variables=add_binary('/usr/local/seqtk/bin')) return stage0
def build(container_format='singularity', os_release='ubuntu', os_version='20.04'): config.set_container_format(container_format) image = f'{os_release}:{os_version}' stage0 = Stage(name='stage0') stage0 += baseimage(image=image, _bootstrap='docker') stage0 += environment(variables={ 'LC_ALL': 'en_AU.UTF-8', 'LANGUAGE': 'en_AU.UTF-8', }) stage0 += label(metadata={'maintainer': 'Luhan Cheng', 'email': '*****@*****.**'}) stage0 += shell(commands=['rm /usr/bin/sh', 'ln -s /usr/bin/bash /usr/bin/sh', '/usr/bin/bash']) stage0 += packages(apt=['wget', 'git', 'software-properties-common', 'build-essential', 'locales', 'zlib1g-dev', 'perl', 'openssh-server', 'libssl-dev', 'gnuplot', 'globus-gridftp-server-progs', 'rsync', 'acl-dev']) stage0 += gnu(version='10') stage0 += shell(commands=[ 'yes | cpan Data::MessagePack', 'yes | cpan IO::Socket::SSL' ]) stage0 += generic_build( repository='https://github.com/eeertekin/bbcp.git', commit='64af832', prefix='/usr/local/bbcp/', build=['make -C src'], install=['mv bin/amd64_linux/bbcp /usr/local/bbcp/'] ) stage0 += environment(variables=add_binary('/usr/local/bbcp')) stage0 += generic_build( repository='https://github.com/pkolano/shift.git', commit='ffb7f5f', prefix='/usr/local/shift', directory='shift/c', build=['make nolustre'], install=['mdkir /usr/local/shift/bin && mv c/shift-bin /usr/local/shift/bin'] ) return stage0
def build(container_format='singularity', os_release='ubuntu', os_version='20.04', cuda_version='11.0'): config.set_container_format(container_format) image = f'{os_release}:{os_version}' stage0 = Stage(name='stage0') stage0 += baseimage(image=image, _bootstrap='docker') stage0 += environment(variables={ 'LC_ALL': 'en_AU.UTF-8', 'LANGUAGE': 'en_AU.UTF-8', }) stage0 += label(metadata={'maintainer': 'Luhan Cheng', 'email': '*****@*****.**'}) stage0 += shell(commands=['rm /usr/bin/sh', 'ln -s /usr/bin/bash /usr/bin/sh', '/usr/bin/bash']) stage0 += packages(apt=['wget', 'git', 'software-properties-common', 'build-essential', 'locales']) stage0 += shell(commands=['locale-gen en_AU.UTF-8']) stage0 += comment('Installing vglrun and TurboVNC') stage0 += packages(apt=['ubuntu-desktop', 'vim', 'mesa-utils', 'python3-pip', 'python3-pyqt5', 'pyqt5-dev', 'python3-tk']) stage0 += shell(commands=[ 'wget https://swift.rc.nectar.org.au/v1/AUTH_810/CVL-Singularity-External-Files/turbovnc_2.2.5_amd64.deb && dpkg -i turbovnc_2.2.5_amd64.deb && rm turbovnc_2.2.5_amd64.deb', 'wget https://swift.rc.nectar.org.au/v1/AUTH_810/CVL-Singularity-External-Files/virtualgl_2.6.4_amd64.deb && dpkg -i virtualgl_2.6.4_amd64.deb && rm virtualgl_2.6.4_amd64.deb', 'apt update', 'apt -y upgrade' ]) stage0 += conda( eula=True, packages=['python=3.8.5', 'openjdk', 'pip'], ) stage0 += environment(variables=from_prefix('/usr/local/anaconda')) stage0 += shell(commands=['pip install --upgrade minerl']) stage0 += shell(commands=[ 'wget https://launcher.mojang.com/download/Minecraft.deb && dpkg -i Minecraft.deb && rm Minecraft.deb', ]) stage0 += runscript(commands=['source /usr/local/anaconda/etc/profile.d/conda.sh', '/usr/local/anaconda/bin/python3 $*']) return stage0
def build(version='1.0.1'): stage0 = CVL_ubuntu_stage() stage0 += packages(apt=['r-base']) stage0 += generic_build( repository='https://github.com/danknights/sourcetracker.git', branch=f'v{version}', install=[ 'mkdir -p /usr/local/sourcetracker/bin && mv src/SourceTracker.r /usr/local/sourcetracker/bin' ]) stage0 += environment( variables={ 'SOURCETRACKER_PATH': '/usr/local/sourcetracker/bin', 'PATH': '/usr/local/sourcetracker/bin:$PATH' }) return stage0
def build(container_format='singularity', os_release='ubuntu', os_version='20.04'): config.set_container_format(container_format) image = f'{os_release}:{os_version}' stage0 = Stage(name='stage0') stage0 += baseimage(image=image, _bootstrap='docker') stage0 += environment(variables={ 'LC_ALL': 'en_AU.UTF-8', 'LANGUAGE': 'en_AU.UTF-8', }) stage0 += label(metadata={ 'maintainer': 'Luhan Cheng', 'email': '*****@*****.**' }) stage0 += shell(commands=[ 'rm -f /bin/sh && ln -s /bin/bash /bin/sh', 'rm -f /usr/bin/sh && ln -s /usr/bin/bash /usr/bin/sh', '/bin/bash', ]) stage0 += packages(apt=[ 'wget', 'git', 'software-properties-common', 'build-essential', 'locales', 'zlib1g-dev' ]) stage0 += shell(commands=['locale-gen en_AU.UTF-8']) stage0 += comment('Installing vglrun and TurboVNC') stage0 += packages(apt=[ 'ubuntu-desktop', 'vim', 'mesa-utils', 'python3-pip', 'python3-pyqt5', 'pyqt5-dev', 'python3-tk', 'cmake', 'libgtk2.0-dev', 'libasound2-dev', 'libavformat-dev', 'libjack-jackd2-dev', 'uuid-dev' ]) stage0 += shell(commands=[ 'wget https://swift.rc.nectar.org.au/v1/AUTH_810/CVL-Singularity-External-Files/turbovnc_2.2.5_amd64.deb && dpkg -i turbovnc_2.2.5_amd64.deb && rm turbovnc_2.2.5_amd64.deb', 'wget https://swift.rc.nectar.org.au/v1/AUTH_810/CVL-Singularity-External-Files/virtualgl_2.6.4_amd64.deb && dpkg -i virtualgl_2.6.4_amd64.deb && rm virtualgl_2.6.4_amd64.deb', 'apt update', 'apt -y upgrade' ]) stage0 += shell(commands=['pip install conan']) stage0 += generic_cmake(repository='https://github.com/audacity/audacity/', branch='Audacity-3.0.5') return stage0
def build(container_format='singularity', os_release='ubuntu', os_version='20.04'): config.set_container_format(container_format) image = f'{os_release}:{os_version}' stage0 = Stage(name='stage0') stage0 += baseimage(image=image, _bootstrap='docker') stage0 += environment(variables={ 'LC_ALL': 'en_AU.UTF-8', 'LANGUAGE': 'en_AU.UTF-8', }) stage0 += label(metadata={ 'maintainer': 'Luhan Cheng', 'email': '*****@*****.**' }) stage0 += shell(commands=[ 'rm -f /bin/sh && ln -s /bin/bash /bin/sh', 'rm -f /usr/bin/sh && ln -s /usr/bin/bash /usr/bin/sh', '/bin/bash', ]) stage0 += packages(apt=[ 'wget', 'git', 'software-properties-common', 'build-essential', 'locales', 'zlib1g-dev' ]) stage0 += shell(commands=['locale-gen en_AU.UTF-8']) stage0 += packages(apt=[ 'perl', 'libmodule-install-perl', 'libxml-parser-perl', 'cpanminus' ]) stage0 += shell(commands=[ 'cpanm inc::latest', 'cpanm BioPerl Bio::SeqIO Getopt::Euclid List::Util Math::Random::MT version' ]) stage0 += shell(commands=[ 'wget https://sourceforge.net/projects/biogrinder/files/biogrinder/Grinder-0.5.4/Grinder-0.5.4.tar.gz && tar -xf Grinder-0.5.4.tar.gz && rm Grinder-0.5.4.tar.gz', 'cd Grinder-0.5.4', 'perl Makefile.PL', 'make', 'make install' ]) return stage0
## # hpccm --recipe relion_3.0.7_staged.py # --format singularity # --singularity-version 3.2 > recipe.def # Start "base recipe, known as '18.04_cuda9'" # add docstring to Dockerfile Stage0 += comment(__doc__.strip(), reformat=False) #Stage0 = hpccm.Stage() Stage0.name = '18.04_cuda9' Stage0 += baseimage(image="ubuntu:18.04", _as=Stage0.name) # Setting up environment vars for container build and runtime. Stage0 += environment(variables={ "LC_ALL": "en_AU.UTF-8", "LANGUAGE": "en_AU.UTF-8" }) # Set apt repos and packages to install base_packages = apt_get( repositories=[ "deb http://us.archive.ubuntu.com/ubuntu/ bionic main restricted", "deb http://us.archive.ubuntu.com/ubuntu/ bionic-updates main restricted", "deb http://us.archive.ubuntu.com/ubuntu/ bionic universe", "deb http://us.archive.ubuntu.com/ubuntu/ bionic-updates universe", ], ospackages=[ "software-properties-common", "locales", "wget", "ubuntu-desktop", "vim", "git", "cmake", "freeglut3-dev", "build-essential", "libx11-dev", "libxmu-dev", "libxi-dev", "libglu1-mesa", "libglu1-mesa-dev", "python-pip", "python-pyqt5", "pyqt5-dev",
def build(container_format='singularity', flavour="devel", os_release='ubuntu', os_version='20.04', cuda_version='11.3.0', python_version='3.8', anaconda_version='4.9.2', alphafold_path="/opt/alphafold", version='2.1.2'): config.set_container_format(container_format) image = f'nvcr.io/nvidia/cuda:{cuda_version}-{flavour}-{os_release}{os_version}' stage0 = Stage(name='stage0') stage0 += baseimage(image=image, _bootstrap='docker') stage0 += environment(variables={ 'LC_ALL': 'en_AU.UTF-8', 'LANGUAGE': 'en_AU.UTF-8', }) stage0 += label(metadata={ 'maintainer': 'Luhan Cheng', 'email': '*****@*****.**' }) stage0 += shell(commands=[ 'rm -f /bin/sh && ln -s /bin/bash /bin/sh', 'rm -f /usr/bin/sh && ln -s /usr/bin/bash /usr/bin/sh', '/bin/bash', ]) stage0 += environment(variables=from_prefix('/usr/local/cuda')) stage0 += packages(apt=[ 'wget', 'git', 'software-properties-common', 'build-essential', 'locales', 'zlib1g-dev', 'vim' ]) stage0 += shell(commands=['locale-gen en_AU.UTF-8']) stage0 += conda(eula=True, packages=[ f'python={python_version}', "openmm==7.5.1", "cudnn==8.2.1.32", "cudatoolkit==11.0.3", "pdbfixer==1.7", "hmmer==3.3.2", "hhsuite==3.3.0", "kalign2==2.04" ], channels=['conda-forge', 'bioconda'], version=anaconda_version) stage0 += environment(variables=from_prefix('/usr/local/anaconda')) stage0 += shell(commands=['conda activate base']) stage0 += shell(commands=[ f"mkdir -p {alphafold_path}", f'git clone -b v{version} https://github.com/deepmind/alphafold.git {alphafold_path}' ]) stage0 += environment(variables={"ALPHAFOLD_PATH": f"{alphafold_path}"}) stage0 += shell(commands=[ f'wget -q -P {alphafold_path}/alphafold/common/ https://git.scicore.unibas.ch/schwede/openstructure/-/raw/7102c63615b64735c4941278d92b554ec94415f8/modules/mol/alg/src/stereo_chemical_props.txt' ]) stage0 += pip(packages=[ "absl-py==0.13.0", "biopython==1.79", "chex==0.0.7", "dm-haiku==0.0.4", "dm-tree==0.1.6", "immutabledict==2.0.0", "ml-collections==0.1.0", "numpy==1.19.5", "scipy==1.7.0", "tensorflow==2.5.0", "pandas==1.3.4", "tensorflow-cpu==2.5.0" ], pip='pip3') stage0 += pip(packages=[ "jax", "jaxlib==0.1.69+cuda111 -f https://storage.googleapis.com/jax-releases/jax_releases.html" ], upgrade=True, pip='pip3') stage0 += pip(packages=[f'-r {alphafold_path}/requirements.txt'], pip='pip3') stage0 += shell(commands=[ f'cd /usr/local/anaconda/lib/python{python_version}/site-packages/ && patch -p0 < {alphafold_path}/docker/openmm.patch' ]) stage0 += shell(commands=[ f'cd {alphafold_path}', 'wget https://raw.githubusercontent.com/0luhancheng0/hpccm-containers/main/src/hpccm_containers/alphafold/run_alphafold.sh && chmod a+x run_alphafold.sh' ]) stage0 += runscript(commands=[ 'source /usr/local/anaconda/etc/profile.d/conda.sh', f"cd {alphafold_path}", f'{alphafold_path}/run_alphafold.sh $@' ]) return stage0
def build(container_format='singularity', os_release='ubuntu', os_version='20.04', cuda_version='11.0'): config.set_container_format(container_format) image = f'nvcr.io/nvidia/cuda:{cuda_version}-devel-{os_release}{os_version}' stage0 = Stage(name='stage0') stage0 += baseimage(image=image, _bootstrap='docker') stage0 += environment(variables={ 'LC_ALL': 'en_AU.UTF-8', 'LANGUAGE': 'en_AU.UTF-8', }) stage0 += label(metadata={ 'maintainer': 'Luhan Cheng', 'email': '*****@*****.**' }) stage0 += shell(commands=[ 'rm -f /bin/sh && ln -s /bin/bash /bin/sh', 'rm -f /usr/bin/sh && ln -s /usr/bin/bash /usr/bin/sh', '/bin/bash', ]) stage0 += environment(variables=from_prefix('/usr/local/cuda')) stage0 += packages(apt=[ 'wget', 'git', 'software-properties-common', 'build-essential', 'locales', 'zlib1g-dev' ]) stage0 += shell(commands=['locale-gen en_AU.UTF-8']) stage0 += comment('Installing vglrun and TurboVNC') stage0 += packages(apt=[ 'ubuntu-desktop', 'vim', 'mesa-utils', 'python3-pip', 'python3-pyqt5', 'pyqt5-dev', 'python3-tk' ]) stage0 += shell(commands=[ 'wget https://swift.rc.nectar.org.au/v1/AUTH_810/CVL-Singularity-External-Files/turbovnc_2.2.5_amd64.deb && dpkg -i turbovnc_2.2.5_amd64.deb && rm turbovnc_2.2.5_amd64.deb', 'wget https://swift.rc.nectar.org.au/v1/AUTH_810/CVL-Singularity-External-Files/virtualgl_2.6.4_amd64.deb && dpkg -i virtualgl_2.6.4_amd64.deb && rm virtualgl_2.6.4_amd64.deb', 'apt update', 'apt -y upgrade' ]) stage0 += comment('Installing pre-requisites') stage0 += primitives.copy( src="./ccp4-7.1.014-shelx-arpwarp-linux64.tar.gz", dest="/opt/ccp4-7.1.014-shelx-arpwarp-linux64.tar.gz") stage0 += shell(commands=[ 'cd /opt && tar -xf ccp4-7.1.014-shelx-arpwarp-linux64.tar.gz && rm ccp4-7.1.014-shelx-arpwarp-linux64.tar.gz', 'touch $HOME/.agree2ccp4v6', 'cd ccp4-7.1', './BINARY.setup', ]) stage0 += environment(variables=add_binary('/opt/ccp4-7.1/bin')) stage0 += comment('Installing CCP-EM') stage0 += primitives.copy(src="./ccpem-1.5.0-linux-x86_64.tar.gz", dest="/opt/ccpem-1.5.0-linux-x86_64.tar.gz") stage0 += primitives.copy(src="./input.txt", dest="/opt/input.txt") stage0 += shell(commands=[ 'touch $HOME/.agree2ccpemv1', 'cd /opt && tar -xf ccpem-1.5.0-linux-x86_64.tar.gz && rm ccpem-1.5.0-linux-x86_64.tar.gz', 'cd ccpem-1.5.0', './install_ccpem.sh', 'cat /opt/input.txt | bash install_modeller.sh' ]) return stage0
def build(python_version='3.6', freesurfer_version='6.0.1'): stage0 = Stage() config.set_container_format('singularity') stage0 += baseimage(image='nvidia/cuda:10.0-cudnn7-runtime-ubuntu16.04', _bootstrap='docker') stage0 += shell( commands=['rm /bin/sh', 'ln -s /bin/bash /bin/sh', '/bin/bash']) stage0 += packages(apt=[ 'build-essential', 'cmake', 'git', 'vim', 'wget', 'ca-certificates', 'bzip2', 'libx11-6', 'libjpeg-dev', 'libpng-dev', 'bc', 'tar', 'zip', 'gawk', 'tcsh', 'time', 'libgomp1', 'libglu1-mesa', 'libglu1-mesa-dev', 'perl-modules' ]) stage0 += comment( "https://github.com/Deep-MI/FastSurfer/blob/master/Docker/Dockerfile") stage0 += shell(commands=[ f"wget -qO- https://surfer.nmr.mgh.harvard.edu/pub/dist/freesurfer/{freesurfer_version}/freesurfer-Linux-centos6_x86_64-stable-pub-v{freesurfer_version}.tar.gz | tar zxv --no-same-owner -C /opt \ --exclude='freesurfer/trctrain' \ --exclude='freesurfer/subjects/fsaverage_sym' \ --exclude='freesurfer/subjects/fsaverage3' \ --exclude='freesurfer/subjects/fsaverage4' \ --exclude='freesurfer/subjects/fsaverage5' \ --exclude='freesurfer/subjects/fsaverage6' \ --exclude='freesurfer/subjects/cvs_avg35' \ --exclude='freesurfer/subjects/cvs_avg35_inMNI152' \ --exclude='freesurfer/subjects/bert' \ --exclude='freesurfer/subjects/V1_average' \ --exclude='freesurfer/average/mult-comp-cor' \ --exclude='freesurfer/lib/cuda' \ --exclude='freesurfer/lib/qt'", ]) stage0 += conda( channels=['pytorch', 'conda-forge'], packages=[ f'python={python_version}', 'scipy', 'numpy', 'matplotlib', 'h5py', 'scikit-image', 'pytorch', 'cudatoolkit=10.0', 'pytorch=1.2.0=py3.6_cuda10.0.130_cudnn7.6.2_0', 'torchvision=0.4.0', 'scikit-sparse', 'nibabel=2.5.1', 'pillow=7.1.1' ], eula=True, ) stage0 += generic_build( repository='https://github.com/Deep-MI/FastSurfer.git', # branch='c5894bd', build=['git checkout c5894bd'], install=['mkdir /fastsurfer && mv * /fastsurfer/']) stage0 += environment( variables={ 'OS': 'Linux', 'FS_OVERRIDE': 0, 'FIX_VERTEX_AREA': '', 'SUBJECTS_DIR': '/opt/freesurfer/subjects', 'FSF_OUTPUT_FORMAT': 'nii.gz', 'MNI_DIR': '/opt/freesurfer/mni', 'LOCAL_DIR': '/opt/freesurfer/local', 'FREESURFER_HOME': '/opt/freesurfer', 'FSFAST_HOME': '/opt/freesurfer/fsfast', 'PERL5LIB': '/opt/freesurfer/mni/lib/perl5/5.8.5', 'MNI_PERL4LIB': '/opt/freesurfer/mni/lib/perl5/5.8.5', 'PYTHONNUMBUFFERED': 0, 'PATH': '/fastsurfer/recon_surf:/fastsurfer:/usr/local/anaconda/bin:/opt/freesurfer/bin:/opt/freesurfer/fsfast/bin:/opt/freesurfer/tktools:/opt/freesurfer/mni/bin:$PATH', **from_prefix('/usr/local/cuda') }) stage0 += runscript(commands=[ 'cd /fastsurfer', 'source /usr/local/anaconda/etc/profile.d/conda.sh', 'source $FREESURFER_HOME/SetUpFreeSurfer.sh', "$*" ]) return stage0
def add_alpaka_dep_layer(stage: hpccm.Stage, ubuntu_version: str, cuda_support: bool, extra_compiler: List[str], alpaka=False) -> bool: """Add all dependencies to an hpccm stage that are necessary to build and run Alpaka. :param stage: At least a baseimage :type stage: hpccm.Stage :param ubuntu_version: Ubuntu version number: '16.04' or '18.04' :type ubuntu_version: str :param cuda_support: Set True, if the Stage supports CUDA :type cuda_support: bool :param extra_compiler: List of compilers, which are installed additional to the system compiler. Supported are: gcc:[5-9], clang:[5.0-7.0, 8-9] :type extra_compiler: str :param alpaka: install alpaka in /usr/local :type alpaka: bool :returns: Returns True if function was successful :rtype: bool """ if ubuntu_version != '16.04' and ubuntu_version != '18.04': print('not supported Ubuntu version: ' + ubuntu_version, file=sys.stderr) print('supported are: 16.04, 18.04', file=sys.stderr) return False apt_package_list = [ 'gcc', 'g++', 'make', 'software-properties-common', 'wget', 'libc6-dev', 'libomp-dev', 'unzip', 'git' ] if ubuntu_version == '16.04': apt_package_list.append('gnupg-agent') if ubuntu_version == '18.04': apt_package_list.append('gpg-agent') stage += packages(ospackages=apt_package_list) stage += cmake(eula=True, version='3.16.0') # install extra compiler if extra_compiler is not None: for com in extra_compiler: if com.startswith('gcc'): stage += gnu(extra_repository=True, version=com[len('gcc:'):]) if com.startswith('clang'): add_clang(stage, ubuntu_version, version=com[len('clang:'):]) #install boost stage += shell( commands=['add-apt-repository -y ppa:mhier/libboost-latest']) stage += packages(ospackages=['boost1.67']) if cuda_support: stage += environment( variables={ 'LD_LIBRARY_PATH': '$LD_LIBRARY_PATH:/usr/local/cuda/lib64' }) # alpaka use a function direct from the cuda driver library # in the container, the cuda libraries are not at the default path stage += environment( variables={ 'LIBRARY_PATH': '$LIBRARY_PATH:/usr/local/cuda/lib64/stubs' }) stage += environment( variables={'CMAKE_PREFIX_PATH': '/usr/local/cuda/lib64/stubs/'}) if alpaka: git_alpaka = git() cmake_alpaka = CMakeBuild() alpaka_commands = [] alpaka_commands.append( git_alpaka.clone_step( repository='https://github.com/alpaka-group/alpaka.git', path='/opt')) alpaka_commands.append( cmake_alpaka.configure_step( build_directory='build', directory='/opt/alpaka', opts=['-Dalpaka_BUILD_EXAMPLES=OFF', '-DBUILD_TESTING=OFF'])) alpaka_commands.append(cmake_alpaka.build_step(target='install')) alpaka_commands.append('rm -rf /opt/alpaka') stage += shell(commands=alpaka_commands) return True
def build(container_format='singularity', os_release='ubuntu', os_version='20.04'): config.set_container_format(container_format) image = f'{os_release}:{os_version}' stage0 = Stage(name='stage0') stage0 += baseimage(image=image, _bootstrap='docker') stage0 += environment(variables={ 'LC_ALL': 'en_AU.UTF-8', 'LANGUAGE': 'en_AU.UTF-8', }) stage0 += label(metadata={ 'maintainer': 'Luhan Cheng', 'email': '*****@*****.**' }) stage0 += shell(commands=[ 'rm /usr/bin/sh', 'ln -s /usr/bin/bash /usr/bin/sh', '/usr/bin/bash' ]) stage0 += packages(apt=[ 'wget', 'git', 'software-properties-common', 'build-essential', 'locales', 'zlib1g-dev', 'perl', 'ncbi-blast+', 'libx11-dev' ]) stage0 += shell(commands=['locale-gen en_AU.UTF-8']) stage0 += gnu(version='10') stage0 += generic_build( url= 'http://search.cpan.org/CPAN/authors/id/Y/YA/YANICK/Parallel-ForkManager-1.19.tar.gz', prefix='/usr/local/parallel-forkmanager', build=['perl Makefile.PL', 'make install'], install=['mv * /usr/local/parallel-forkmanager/']) stage0 += environment( variables=add_binary('/usr/local/parallel-forkmanager')) stage0 += generic_build( url= 'ftp://ftp.ebi.ac.uk/pub/software/clustalw2/2.1/clustalw-2.1-linux-x86_64-libcppstatic.tar.gz', prefix='/usr/local/clustalw', install=['mv clustalw2 /usr/local/clustalw/clustalw']) stage0 += environment(variables=add_binary('/usr/local/clustalw')) stage0 += generic_autotools( url='ftp://emboss.open-bio.org/pub/EMBOSS/EMBOSS-6.6.0.tar.gz', prefix='/usr/local/emboss', ) stage0 += environment(variables=from_prefix('/usr/local/emboss')) stage0 += generic_autotools( url= 'https://www.tbi.univie.ac.at/RNA/download/sourcecode/2_4_x/ViennaRNA-2.4.17.tar.gz', prefix='/usr/local/viennarna') stage0 += environment(variables=from_prefix('/usr/local/viennarna')) stage0 += generic_build( url='https://github.com/weizhongli/cdhit/archive/V4.8.1.tar.gz', prefix='/usr/local/cdhit', directory='cdhit-4.8.1', build=['make'], install=['mv * /usr/local/cdhit/']) stage0 += environment(variables=add_binary('/usr/local/cdhit')) stage0 += generic_build( repository='https://github.com/ambarishbiswas/CRISPRDetect_2.2.git', commit='0f8249f', prefix='/usr/local/crisprdetect', install=['mv * /usr/local/crisprdetect/']) stage0 += environment(variables=add_binary('/usr/local/crisprdetect')) return stage0
def build(container_format='singularity', os='ubuntu20.04', cuda_version='11.0', gpu_version='V100', mkl_version='2020.0-088', version='psmp', tag='8.1.0'): image = f'nvcr.io/nvidia/cuda:{cuda_version}-devel-{os}' config.set_container_format(container_format) stage0 = Stage(name='stage0') stage0 += baseimage(image=image, _bootstrap='docker') stage0 += label(metadata={ 'maintainer': 'Luhan Cheng', 'email': '*****@*****.**' }) stage0 += comment( 'Toolchain installation translated from https://github.com/cp2k/cp2k/blob/master/tools/toolchain/Dockerfile.cuda_mkl' ) stage0 += packages(apt=[ 'git', 'gfortran', 'mpich', 'libmpich-dev', 'software-properties-common', 'python-dev' ]) stage0 += python() stage0 += shell(commands=[ f'git clone --depth 1 --branch v{tag} https://github.com/cp2k/cp2k.git /cp2k', 'cd /cp2k', 'git submodule update --init --recursive' ]) stage0 += shell( commands=['/cp2k/tools/toolchain/install_requirements_ubuntu.sh']) stage0 += environment( variables={ 'CUDA_PATH': '/usr/local/cuda', 'LD_LIBRARY_PATH': '/usr/local/cuda/:${LD_LIBRARY_PATH}', 'MKLROOT': '/opt/intel/compilers_and_libraries/linux/mkl', **from_prefix('/usr/local/cuda') }) stage0 += mkl(eula=True, version=mkl_version) stage0 += workdir(directory='/opt/cp2k-toolchain') stage0 += shell(commands=[ 'cd /opt/cp2k-toolchain', 'mkdir scripts && cp -r /cp2k/tools/toolchain/scripts/* ./scripts/', 'cp /cp2k/tools/toolchain/install_cp2k_toolchain.sh .', ]) stage0 += shell(commands=[ 'cd /opt/cp2k-toolchain', f'./install_cp2k_toolchain.sh --mpi-mode=openmpi --math-mode=mkl --with-reflapack=no --with-scalapack=no --with-elpa=no --gpu-ver={gpu_version}', 'rm -rf ./build' ]) stage0 += shell(commands=[ 'cp /opt/cp2k-toolchain/install/arch/* /cp2k/arch/', 'cd /cp2k', "sed 's/source /. /g' /opt/cp2k-toolchain/install/setup > /opt/cp2k-toolchain/install/setup ", '. /opt/cp2k-toolchain/install/setup', f'make ARCH=local VERSION="{version}" | tee /var/tmp/log.txt' ]) stage0 += environment(variables={'PATH': '/cp2k/exe/local:$PATH'}) stage0 += shell(commands=[ 'ln -s /usr/lib/x86_64-linux-gnu/libncursesw.so.6 /usr/lib/x86_64-linux-gnu/libncursesw.so.5', 'ln -s /usr/lib/x86_64-linux-gnu/libtinfo.so.6 /usr/lib/x86_64-linux-gnu/libtinfo.so.5' ]) stage0 += environment( variables={ 'LD_LIBRARY_PATH': '/usr/lib/x86_64-linux-gnu/:$LD_LIBRARY_PATH' }) return stage0
def build(container_format='singularity', os_release='ubuntu', os_version='20.04', cuda_version='11.0.3'): config.set_container_format(container_format) # image = f'{os_release}:{os_version}' image = f'nvcr.io/nvidia/cuda:{cuda_version}-devel-{os_release}{os_version}' stage0 = Stage(name='stage0') stage0 += baseimage(image=image, _bootstrap='docker') stage0 += environment(variables={ 'LC_ALL': 'en_AU.UTF-8', 'LANGUAGE': 'en_AU.UTF-8', }) stage0 += label(metadata={ 'maintainer': 'Luhan Cheng', 'email': '*****@*****.**' }) stage0 += shell(commands=[ 'rm -f /bin/sh && ln -s /bin/bash /bin/sh', 'rm -f /usr/bin/sh && ln -s /usr/bin/bash /usr/bin/sh', '/bin/bash', ]) stage0 += packages(apt=[ 'wget', 'git', 'software-properties-common', 'build-essential', 'locales', 'zlib1g-dev' ]) stage0 += shell(commands=['locale-gen en_AU.UTF-8']) stage0 += packages(apt=[ 'freeglut3-dev', 'build-essential', 'libx11-dev', 'libxmu-dev', 'libxi-dev', 'libglu1-mesa', 'libglu1-mesa-dev', 'mesa-utils', 'libgl1-mesa-glx' ]) stage0 += environment(variables=from_prefix('/usr/local/cuda')) stage0 += comment('Installing vglrun and TurboVNC') stage0 += packages(apt=[ 'ubuntu-desktop', 'vim', 'mesa-utils', 'python3-pip', 'python3-pyqt5', 'pyqt5-dev', 'python3-tk' ]) stage0 += shell(commands=[ 'wget https://swift.rc.nectar.org.au/v1/AUTH_810/CVL-Singularity-External-Files/turbovnc_2.2.5_amd64.deb && dpkg -i turbovnc_2.2.5_amd64.deb && rm turbovnc_2.2.5_amd64.deb', 'wget https://swift.rc.nectar.org.au/v1/AUTH_810/CVL-Singularity-External-Files/virtualgl_2.6.4_amd64.deb && dpkg -i virtualgl_2.6.4_amd64.deb && rm virtualgl_2.6.4_amd64.deb', 'apt update', 'apt -y upgrade' ]) stage0 += packages(apt=[ 'make', 'libopenmpi-dev', 'python3-tk', 'libfftw3-dev', 'libhdf5-dev', 'libtiff-dev', 'libjpeg-dev', 'libsqlite3-dev', 'openjdk-8-jdk' ]) stage0 += environment( variables={ 'LD_LIBRARY_PATH': "$LD_LIBRARY_PATH:/usr/lib/x86_64-linux-gnu/hdf5/serial", }) prefix = '/usr/local/scipion' stage0 += shell(commands=[ 'python3 -m pip install --user scipion-installer', f'python3 -m scipioninstaller {prefix} -j 4' ]) stage0 += environment(variables=from_prefix(prefix)) return stage0
def main(): # pragma: no cover cli = Cli_Args() args = cli.parse_args() if args.deploy != '': args.build = True args.convert = True cwd = os.getcwd() c = list( itertools.product(args.format, args.ogs, args.pm, args.ompi, args.cmake_args)) if not args.print and not args.cleanup: print('Creating {} image definition(s)...'.format(len(c))) for build in c: __format = build[0] ogs_version = build[1] ogscm.config.set_package_manager(build[2]) ompi = build[3] cmake_args = build[4].strip().split(' ') # args checking if len(c) > 1 and args.file != '': print( '--file can only be used when generating a single image definition' ) quit(1) if (len(c) > 1 and args.sif_file != '') or (args.sif_file != '' and args.convert == False): print('--sif_file can only be used when generating a single image ' 'definition and --convert is given') quit(1) if (ogs_version == 'off' or ogs_version == 'clean') and len(cmake_args) > 0 and cmake_args[0] != '': cmake_args = [] print('--cmake_args cannot be used with --ogs off! Ignoring!') if __format == 'singularity': if args.runtime_only: args.runtime_only = False print( '--runtime-only cannot be used with --format singularity! ' 'Ignoring!') if args.upload: print('--upload cannot be used with --format singularity! ' 'Ignoring!') if args.convert: print('--convert cannot be used with --format singularity! ' 'Ignoring!') info = container_info(build, args) if args.cleanup: info.cleanup() exit(0) info.make_dirs() if ompi != 'off': if args.base_image == 'ubuntu:20.04': args.base_image = 'centos:8' print( 'Setting base_image to \'centos:8\'. OpenMPI is supported on CentOS only.' ) # Create definition hpccm.config.set_container_format(__format) # ------------------------------ recipe ------------------------------- Stage0 = hpccm.Stage() Stage0 += raw(docker='# syntax=docker/dockerfile:experimental') if args.runtime_only: Stage0.name = 'build' Stage0 += baseimage(image=args.base_image, _as='build') Stage0 += comment(f"Generated with ogs-container-maker {__version__}", reformat=False) Stage0 += packages(ospackages=['wget', 'tar', 'curl', 'make']) # base compiler if args.compiler != 'off': if args.compiler_version == '': if args.compiler == 'clang': args.compiler_version = '8' else: if hpccm.config.g_linux_distro == linux_distro.CENTOS: args.compiler_version = '10' # required for std::filesystem else: args.compiler_version = None # Use default if args.compiler == 'clang': compiler = llvm(extra_repository=True, extra_tools=True, version=args.compiler_version) else: compiler = gnu(fortran=False, extra_repository=True, version=args.compiler_version) toolchain = compiler.toolchain Stage0 += compiler # Upgrade stdc++ lib after installing new compiler # https://stackoverflow.com/a/46613656/80480 if args.compiler == 'gcc' and args.compiler_version is not None: Stage0 += packages(apt=['libstdc++6']) # Prepare runtime stage Stage1 = hpccm.Stage() Stage1.baseimage(image=args.base_image) # Install scif in all stages Stage0 += pip(packages=['scif'], pip='pip3') Stage1 += pip(packages=['scif'], pip='pip3') if ompi != 'off': mpicc = object if False: # eve: # Stage0 += ofed() OR mlnx_ofed(); is installed later on from debian archive # Stage0 += knem() Stage0 += ucx(version='1.5.1', cuda=False) # knem='/usr/local/knem' Stage0 += packages(ospackages=['libpmi2-0-dev' ]) # req. for --with-pmi # req. for --with-psm2 Stage0 += packages(ospackages=['libnuma1']) psm_deb_url = 'http://snapshot.debian.org/archive/debian/20181231T220010Z/pool/main' psm2_version = '11.2.68-4' Stage0 += shell(commands=[ 'cd /tmp', f'wget -nv {psm_deb_url}/libp/libpsm2/libpsm2-2_{psm2_version}_amd64.deb', f'wget -nv {psm_deb_url}/libp/libpsm2/libpsm2-dev_{psm2_version}_amd64.deb', 'dpkg --install *.deb' ]) # libibverbs # Available versions: http://snapshot.debian.org/binary/ibacm/ # ibverbs_version = '21.0-1' # works on eve, eve has 17.2-3 installed nut this version is not available in snapshot.debian ib_deb_url = 'http://snapshot.debian.org/archive/debian/20180430T215634Z/pool/main' ibverbs_version = '17.1-2' ibverbs_packages = [ 'ibacm', 'ibverbs-providers', 'ibverbs-utils', 'libibumad-dev', 'libibumad3', 'libibverbs-dev', 'libibverbs1', 'librdmacm-dev', 'librdmacm1', 'rdma-core', 'rdmacm-utils' ] ibverbs_cmds = ['cd /tmp'] for package in ibverbs_packages: ibverbs_cmds.extend([ f'wget -nv {ib_deb_url}/r/rdma-core/{package}_{ibverbs_version}_amd64.deb' ]) ibverbs_cmds.append('dpkg --install *.deb') Stage0 += packages(ospackages=[ 'libnl-3-200', 'libnl-route-3-200', 'libnl-route-3-dev', 'udev', 'perl' ]) Stage0 += shell(commands=ibverbs_cmds) mpicc = openmpi( version=ompi, cuda=False, toolchain=toolchain, ldconfig=True, ucx='/usr/local/ucx', configure_opts=[ '--disable-getpwuid', '--sysconfdir=/mnt/0', '--with-slurm', # used on taurus '--with-pmi=/usr/include/slurm-wlm', 'CPPFLAGS=\'-I /usr/include/slurm-wlm\'', '--with-pmi-libdir=/usr/lib/x86_64-linux-gnu', # '--with-pmix', '--with-psm2', '--disable-pty-support', '--enable-mca-no-build=btl-openib,plm-slurm', # eve: '--with-sge', '--enable-mpirun-prefix-by-default', '--enable-orterun-prefix-by-default', ]) else: ucx_version = '1.8.1' Stage0 += ucx(version=ucx_version, cuda=False) Stage0 += slurm_pmi2(version='17.02.11') pmix_version = True if version.parse(ompi) >= version.parse('4'): Stage0 += pmix(version='3.1.5') pmix_version = '/usr/local/pmix' mpicc = openmpi(version=ompi, cuda=False, infiniband=False, pmi='/usr/local/slurm-pmi2', pmix=pmix_version, ucx='/usr/local/ucx') toolchain = mpicc.toolchain Stage0 += mpicc # OpenMPI expects this program to exist, even if it's not used. # Default is "ssh : rsh", but that's not installed. Stage0 += shell(commands=[ 'mkdir /mnt/0', "echo 'plm_rsh_agent = false' >> /mnt/0/openmpi-mca-params.conf" ]) Stage0 += label( metadata={ 'org.opengeosys.mpi': 'openmpi', 'org.opengeosys.mpi.version': ompi }) if args.mpi_benchmarks: # osu_app = scif(name='osu', file=f"{info.out_dir}/osu.scif") Stage0 += osu_benchmarks(toolchain=toolchain) Stage0 += shell(commands=[ 'mkdir -p /usr/local/mpi-examples', 'cd /usr/local/mpi-examples', 'curl -O https://raw.githubusercontent.com/hpc/charliecloud/674b3b4e4ad243be5565f200d8f5fb92b7544480/examples/mpihello/hello.c', 'curl -O https://computing.llnl.gov/tutorials/mpi/samples/C/mpi_bandwidth.c', 'curl -O https://raw.githubusercontent.com/mpitutorial/mpitutorial/gh-pages/tutorials/mpi-send-and-receive/code/ring.c', 'mpicc -o /usr/local/bin/mpi-hello /usr/local/mpi-examples/hello.c', 'mpicc -o /usr/local/bin/mpi-ring /usr/local/mpi-examples/ring.c', 'mpicc -o /usr/local/bin/mpi-bandwidth /usr/local/mpi-examples/mpi_bandwidth.c', ]) Stage1 += copy(_from='build', src='/usr/local/bin/mpi-*', dest='/usr/local/bin/') # Stage0 += mlnx_ofed() if ogs_version != 'clean': Stage0 += ogs_base() if args.gui: Stage0 += packages(apt=[ 'mesa-common-dev', 'libgl1-mesa-dev', 'libglu1-mesa-dev', 'libxt-dev' ], yum=[ 'mesa-libOSMesa-devel', 'mesa-libGL-devel', 'mesa-libGLU-devel', 'libXt-devel' ]) Stage1 += packages( apt=[ 'libosmesa6', 'libgl1-mesa-glx', 'libglu1-mesa', 'libxt6', 'libopengl0' ], yum=['mesa-libOSMesa', 'mesa-libGL', 'mesa-libGLU', 'libXt']) if ogs_version != 'clean': if ogscm.config.g_package_manager == package_manager.CONAN: Stage0 += cmake(eula=True, version='3.16.6') conan_user_home = '/opt/conan' if args.dev: conan_user_home = '' Stage0 += pm_conan(user_home=conan_user_home) Stage0 += environment(variables={'CONAN_SYSREQUIRES_SUDO': 0}) elif ogscm.config.g_package_manager == package_manager.SYSTEM: Stage0 += cmake(eula=True, version='3.16.6') Stage0 += boost(version='1.66.0', bootstrap_opts=['headers']) Stage0 += environment( variables={'BOOST_ROOT': '/usr/local/boost'}) vtk_cmake_args = [ '-DModule_vtkIOXML=ON', '-DModule_vtkIOXdmf3=ON', '-DVTK_Group_Rendering=OFF', '-DVTK_Group_StandAlone=OFF' ] if args.gui: Stage0 += packages(apt=[ 'libgeotiff-dev', 'libshp-dev', 'libnetcdf-c++4-dev', 'libqt5x11extras5-dev', 'libqt5xmlpatterns5-dev', 'qt5-default' ], yum=[ 'libgeotiff-devel', 'shapelib-devel', 'netcdf-devel', 'qt5-qtbase-devel', 'qt5-qtxmlpatterns-devel', 'qt5-qtx11extras-devel' ]) Stage1 += packages(apt=[ 'geotiff-bin', 'shapelib', 'libnetcdf-c++4', 'libqt5x11extras5', 'libqt5xmlpatterns5', 'qt5-default' ], yum=[ 'libgeotiff', 'shapelib', 'netcdf', 'qt5-qtbase', 'qt5-qtxmlpatterns', 'qt5-qtx11extras' ]) vtk_cmake_args = [ '-DModule_vtkIOXdmf3=ON', '-DVTK_BUILD_QT_DESIGNER_PLUGIN=OFF', '-DVTK_Group_Qt=ON', '-DVTK_QT_VERSION=5' ] if hpccm.config.g_linux_distro == linux_distro.CENTOS: # otherwise linker error, maybe due to gcc 10? vtk_cmake_args.extend([ '-DBUILD_SHARED_LIBS=OFF', '-DCMAKE_POSITION_INDEPENDENT_CODE=ON' ]) if args.insitu: if args.gui: print('--gui can not be used with --insitu!') exit(1) Stage0 += paraview(cmake_args=['-DPARAVIEW_USE_PYTHON=ON'], edition='CATALYST', ldconfig=True, toolchain=toolchain, version="v5.8.1") else: if toolchain.CC == 'mpicc': vtk_cmake_args.extend([ '-D Module_vtkIOParallelXML=ON', '-D Module_vtkParallelMPI=ON' ]) Stage0 += generic_cmake( cmake_opts=vtk_cmake_args, devel_environment={'VTK_ROOT': '/usr/local/vtk'}, directory='VTK-8.2.0', ldconfig=True, prefix='/usr/local/vtk', toolchain=toolchain, url= 'https://www.vtk.org/files/release/8.2/VTK-8.2.0.tar.gz' ) if ompi != 'off': Stage0 += packages(yum=['diffutils']) Stage0 += generic_autotools( configure_opts=[ f'CC={toolchain.CC}', f'CXX={toolchain.CXX}', '--CFLAGS=\'-O3\'', '--CXXFLAGS=\'-O3\'', '--FFLAGS=\'-O3\'', '--with-debugging=no', '--with-fc=0', '--download-f2cblaslapack=1' ], devel_environment={'PETSC_DIR': '/usr/local/petsc'}, directory='petsc-3.11.3', ldconfig=True, preconfigure=[ "sed -i -- 's/python/python3/g' configure" ], prefix='/usr/local/petsc', toolchain=toolchain, url='http://ftp.mcs.anl.gov/pub/petsc/release-snapshots/' 'petsc-lite-3.11.3.tar.gz') Stage0 += generic_cmake( devel_environment={ 'Eigen3_ROOT': '/usr/local/eigen', 'Eigen3_DIR': '/usr/local/eigen' }, directory='eigen-3.3.7', prefix='/usr/local/eigen', url= 'https://gitlab.com/libeigen/eigen/-/archive/3.3.7/eigen-3.3.7.tar.gz' ) if args.cvode: Stage0 += generic_cmake( cmake_opts=[ '-D EXAMPLES_INSTALL=OFF', '-D BUILD_SHARED_LIBS=OFF', '-D CMAKE_POSITION_INDEPENDENT_CODE=ON' ], devel_environment={'CVODE_ROOT': '/usr/local/cvode'}, directory='cvode-2.8.2', prefix='/usr/local/cvode', url='https://github.com/ufz/cvode/archive/2.8.2.tar.gz') if args.cppcheck: Stage0 += generic_cmake( devel_environment={'PATH': '/usr/local/cppcheck/bin:$PATH'}, directory='cppcheck-809a769c690d8ab6fef293e41a29c8490512866e', prefix='/usr/local/cppcheck', runtime_environment={'PATH': '/usr/local/cppcheck/bin:$PATH'}, url= 'https://github.com/danmar/cppcheck/archive/809a769c690d8ab6fef293e41a29c8490512866e.tar.gz' ) if args.iwyy and args.compiler == 'clang': Stage0 += packages(ospackages=[ 'libncurses5-dev', 'zlib1g-dev', f"llvm-{args.compiler_version}-dev", f"libclang-{args.compiler_version}-dev" ]) Stage0 += generic_cmake( cmake_opts=[ f"-D IWYU_LLVM_ROOT_PATH=/usr/lib/llvm-{args.compiler_version}" ], devel_environment={'PATH': '/usr/local/iwyy/bin:$PATH'}, directory= f"include-what-you-use-clang_{args.compiler_version}.0", prefix='/usr/local/iwyy', runtime_environment={'PATH': '/usr/local/iwyy/bin:$PATH'}, url="https://github.com/include-what-you-use/include-what-" f"you-use/archive/clang_{args.compiler_version}.0.tar.gz") if args.docs: Stage0 += packages( ospackages=['doxygen', 'graphviz', 'texlive-base']) if args.gcovr: Stage0 += pip(pip='pip3', packages=['gcovr']) if args.dev: Stage0 += packages(ospackages=[ 'neovim', 'gdb', 'silversearcher-ag', 'ssh-client', 'less' ]) if args.pip: Stage0 += pip(packages=args.pip, pip='pip3') Stage1 += pip(packages=args.pip, pip='pip3') if args.packages: Stage0 += packages(ospackages=args.packages) if args.tfel: Stage0 += generic_cmake( directory='tfel-TFEL-3.3.0', ldconfig=True, url='https://github.com/thelfer/tfel/archive/TFEL-3.3.0.tar.gz', prefix='/usr/local/tfel') Stage0 += environment(variables={'TFELHOME': '/usr/local/tfel'}) definition_file_path = os.path.join(info.out_dir, info.definition_file) if args.ccache: Stage0 += ccache(cache_size='15G') if ogs_version != 'off' and ogs_version != 'clean': mount_args = '' if args.ccache: mount_args = f'{mount_args} --mount=type=cache,target=/opt/ccache,id=ccache' if args.cvode: cmake_args.append('-DOGS_USE_CVODE=ON') if args.gui: cmake_args.append('-DOGS_BUILD_GUI=ON') if args.insitu: cmake_args.append('-DOGS_INSITU=ON') Stage0 += raw(docker=f"ARG OGS_COMMIT_HASH={info.commit_hash}") scif_file = f"{info.out_dir}/ogs.scif" if info.ogsdir: context_path_size = len(ogs_version) print(f"chdir to {ogs_version}") os.chdir(ogs_version) mount_args = f'{mount_args} --mount=type=bind,target=/scif/apps/ogs/src,rw' scif_file = f"{info.out_dir[context_path_size+1:]}/ogs.scif" definition_file_path = f"{info.out_dir[context_path_size+1:]}/{info.definition_file}" ogs_app = scif(_arguments=mount_args, name='ogs', file=scif_file) ogs_app += ogs(repo=info.repo, branch=info.branch, commit=info.commit_hash, git_version=info.git_version, toolchain=toolchain, prefix='/scif/apps/ogs', cmake_args=cmake_args, parallel=args.parallel, remove_build=True, remove_source=True) Stage0 += ogs_app stages_string = str(Stage0) if args.runtime_only: Stage1 += Stage0.runtime(exclude=['boost']) if args.compiler == 'gcc' and args.compiler_version != None: Stage1 += packages(apt=['libstdc++6']) stages_string += "\n\n" + str(Stage1) # ---------------------------- recipe end ----------------------------- with open(definition_file_path, 'w') as f: print(stages_string, file=f) if args.print: print(stages_string) else: print( f'Created definition {os.path.abspath(definition_file_path)}') # Create image if not args.build: continue if __format == 'singularity': subprocess.run( f"sudo `which singularity` build --force {info.images_out_dir}/{info.img_file}.sif" f"{definition_file_path}", shell=True) subprocess.run( f"sudo chown $USER:$USER {info.images_out_dir}/{info.img_file}.sif", shell=True) # TODO: adapt this to else continue build_cmd = (f"DOCKER_BUILDKIT=1 docker build {args.build_args} " f"-t {info.tag} -f {definition_file_path} .") print(f"Running: {build_cmd}") subprocess.run(build_cmd, shell=True) inspect_out = subprocess.check_output( f"docker inspect {info.tag} | grep Id", shell=True).decode(sys.stdout.encoding) image_id = re.search('sha256:(\w*)', inspect_out).group(1) image_id_short = image_id[0:12] if args.upload: subprocess.run(f"docker push {info.tag}", shell=True) if args.sif_file: image_file = f'{info.images_out_dir}/{args.sif_file}' else: image_file = f'{info.images_out_dir}/{info.img_file}-{image_id_short}.sif' if args.convert and not os.path.exists(image_file): subprocess.run( f"cd {cwd} && singularity build --force {image_file} docker-daemon:{info.tag}", shell=True) # Deploy image if not args.deploy: continue deploy_config_filename = f'{cwd}/config/deploy_hosts.yml' if not os.path.isfile(deploy_config_filename): print( f'ERROR: {deploy_config_filename} not found but required for deploying!' ) exit(1) with open(deploy_config_filename, 'r') as ymlfile: deploy_config = yaml.load(ymlfile, Loader=yaml.FullLoader) if not args.deploy == 'ALL' and not args.deploy in deploy_config: print(f'ERROR: Deploy host "{args.deploy}" not found in config!') exit(1) deploy_hosts = {} if args.deploy == 'ALL': deploy_hosts = deploy_config else: deploy_hosts[args.deploy] = deploy_config[args.deploy] for deploy_host in deploy_hosts: deploy_info = deploy_hosts[deploy_host] print(f'Deploying to {deploy_info} ...') proxy_cmd = '' user_cmd = '' if 'user' in deploy_info: user_cmd = f"{deploy_info['user']}@" if 'proxy' in deploy_info: proxy_cmd = f"-e 'ssh -A -J {user_cmd}{deploy_info['proxy']}'" print(proxy_cmd) print( subprocess.check_output( f"rsync -c -v {proxy_cmd} {image_file} {user_cmd}{deploy_info['host']}:{deploy_info['dest_dir']}", shell=True).decode(sys.stdout.encoding))