def test_baseimage_first(self):
"""Base image is always first"""
s = Stage()
s += shell(commands=['abc'])
s.name = 'bar'
s.baseimage('foo')
self.assertEqual(str(s), 'FROM foo AS bar\n\nRUN abc')
def test_multistage_as_override_docker(self):
"""Multistage naming"""
s0 = Stage()
s0 += baseimage(image='centos:7', _as='devel')
s0 += boost()
s1 = Stage()
s1 += s0.runtime(_from='build')
self.assertEqual(
str(s1), r'''# Boost
COPY --from=build /usr/local/boost /usr/local/boost
ENV LD_LIBRARY_PATH=/usr/local/boost/lib:$LD_LIBRARY_PATH''')
def test_multistage_noas_docker(self):
"""Multistage naming"""
s0 = Stage()
s0 += baseimage(image='centos:7')
s0 += boost()
s1 = Stage()
s1 += s0.runtime()
self.assertEqual(
str(s1), r'''# Boost
COPY --from=0 /usr/local/boost /usr/local/boost
ENV LD_LIBRARY_PATH=/usr/local/boost/lib:$LD_LIBRARY_PATH''')
def test_runtime_exclude(self):
"""Runtime from a previous stage with exclude"""
s0 = Stage()
s0 += gnu()
s0 += boost()
s1 = Stage()
s1 += s0.runtime(exclude=['boost'])
self.assertEqual(
str(s1), r'''# GNU compiler runtime
RUN yum install -y \
libgomp \
libgfortran && \
rm -rf /var/cache/yum/*''')
def test_runtime(self):
"""Runtime from a previous stage"""
s0 = Stage()
s0 += gnu()
s0 += shell(commands=['gcc -o hello hello.c'])
s1 = Stage()
s1 += s0.runtime()
self.assertEqual(
str(s1), r'''# GNU compiler runtime
RUN yum install -y \
libgomp \
libgfortran && \
rm -rf /var/cache/yum/*''')
def recipe(recipe_file,
ctype=container_type.DOCKER,
raise_exceptions=False,
single_stage=False,
userarg=None):
"""Recipe builder"""
# Make user arguments available
USERARG = {} # pylint: disable=unused-variable
if userarg:
USERARG = userarg # alias
# Consider just 2 stages for the time being
stages = [Stage(name='stage0'), Stage()]
Stage0 = stages[0] # alias # pylint: disable=unused-variable
Stage1 = stages[1] # alias # pylint: disable=unused-variable
try:
with open(recipe_file) as f:
# pylint: disable=exec-used
exec(compile(f.read(), recipe_file, 'exec'))
except Exception as e:
if raise_exceptions:
raise_from(e, e)
else:
logging.error(e)
exit(1)
# Set the global container type
hpccm.config.g_ctype = ctype
# Only process the first stage of a recipe
if single_stage:
del stages[1:]
else:
# Singularity does not support multi-stage builds. Ignore
# anything beyond the first stage.
if ctype == container_type.SINGULARITY and Stage1.is_defined():
logging.warning('This looks like a multi-stage recipe, but '
'Singularity does not support multi-stage builds. '
'Use --single-stage to get rid of this warning. '
'Only processing the first stage...')
del stages[1:]
r = []
for index, stage in enumerate(stages):
if index >= 1:
r.append('')
r.append(str(stage))
return '\n'.join(r)
def test_multistage_as_override_singularity(self):
"""Multistage naming"""
s0 = Stage()
s0 += baseimage(image='centos:7', _as='devel')
s0 += boost()
s1 = Stage()
s1 += s0.runtime(_from='build')
self.assertEqual(
str(s1), r'''# Boost
%files from build
/usr/local/boost /usr/local/boost
%environment
export LD_LIBRARY_PATH=/usr/local/boost/lib:$LD_LIBRARY_PATH
%post
export LD_LIBRARY_PATH=/usr/local/boost/lib:$LD_LIBRARY_PATH''')
def recipe(recipe_file,
ctype=container_type.DOCKER,
raise_exceptions=False,
single_stage=False,
singularity_version='2.6',
userarg=None):
"""Recipe builder
# Arguments
recipe_file: path to a recipe file (required).
ctype: Enum representing the container specification format. The
default is `container_type.DOCKER`.
raise_exceptions: If False, do not print stack traces when an
exception is raised. The default value is False.
single_stage: If True, only print the first stage of a multi-stage
recipe. The default is False.
singularity_version: Version of the Singularity definition file
format to use. Multi-stage support was added in version 3.2, but
the changes are incompatible with earlier versions of Singularity.
The default is '2.6'.
userarg: A dictionary of key / value pairs provided to the recipe
as the `USERARG` dictionary.
"""
# Make user arguments available
USERARG = {} # pylint: disable=unused-variable
if userarg:
USERARG = userarg # alias
# Consider just 2 stages for the time being
stages = [Stage(name='stage0'), Stage()]
Stage0 = stages[0] # alias # pylint: disable=unused-variable
Stage1 = stages[1] # alias # pylint: disable=unused-variable
# Set the global container type
hpccm.config.g_ctype = ctype
# Set the global Singularity version
hpccm.config.g_singularity_version = StrictVersion(singularity_version)
try:
with open(recipe_file) as f:
# pylint: disable=exec-used
exec(compile(f.read(), recipe_file, 'exec'),
dict(locals(), **globals()))
except Exception as e:
if raise_exceptions:
raise_from(e, e)
else:
logging.error(e)
exit(1)
# Only process the first stage of a recipe
if single_stage:
del stages[1:]
elif len(Stage1) > 0:
if (ctype == container_type.SINGULARITY
and hpccm.config.g_singularity_version < StrictVersion('3.2')):
# Singularity prior to version 3.2 did not support
# multi-stage builds. If the Singularity version is not
# sufficient to support multi-stage, provide advice to
# specify a sufficient Singularity version or disable
# multi-stage.
logging.warning('This looks like a multi-stage recipe. '
'Singularity 3.2 or later is required for '
'multi-stage builds. Use '
'--singularity-version=3.2 to enable this '
'feature or --single-stage to get rid of this '
'warning. Only processing the first stage...')
del stages[1:]
elif ctype == container_type.BASH:
logging.warning('This looks like a multi-stage recipe, but '
'bash does not support multi-stage builds. '
'Use --single-stage to get rid of this warning. '
'Only processing the first stage...')
del stages[1:]
r = []
for index, stage in enumerate(stages):
if index >= 1:
r.append('')
r.append(str(stage))
return '\n'.join(r)
def recipe(recipe_file,
ctype=container_type.DOCKER,
raise_exceptions=False,
single_stage=False,
userarg=None):
"""Recipe builder
# Arguments
recipe_file: path to a recipe file (required).
ctype: Enum representing the container specification format. The
default is `container_type.DOCKER`.
raise_exceptions: If False, do not print stack traces when an
exception is raised. The default value is False.
single_stage: If True, only print the first stage of a multi-stage
recipe. The default is False.
userarg: A dictionary of key / value pairs provided to the recipe
as the `USERARG` dictionary.
"""
# Make user arguments available
USERARG = {} # pylint: disable=unused-variable
if userarg:
USERARG = userarg # alias
# Consider just 2 stages for the time being
stages = [Stage(name='stage0'), Stage()]
Stage0 = stages[0] # alias # pylint: disable=unused-variable
Stage1 = stages[1] # alias # pylint: disable=unused-variable
# Set the global container type
hpccm.config.g_ctype = ctype
try:
with open(recipe_file) as f:
# pylint: disable=exec-used
exec(compile(f.read(), recipe_file, 'exec'),
dict(locals(), **globals()))
except Exception as e:
if raise_exceptions:
raise_from(e, e)
else:
logging.error(e)
exit(1)
# Only process the first stage of a recipe
if single_stage:
del stages[1:]
else:
# Singularity does not support multi-stage builds. Ignore
# anything beyond the first stage.
if ctype == container_type.SINGULARITY and len(Stage1) > 0:
logging.warning('This looks like a multi-stage recipe, but '
'Singularity does not support multi-stage builds. '
'Use --single-stage to get rid of this warning. '
'Only processing the first stage...')
del stages[1:]
r = []
for index, stage in enumerate(stages):
if index >= 1:
r.append('')
r.append(str(stage))
return '\n'.join(r)
def test_baseimage(self):
"""Base image specification"""
s = Stage()
s.name = 'bar'
s.baseimage('foo')
self.assertEqual(str(s), 'FROM foo AS bar')
def test_list(self):
"""List of layers"""
s = Stage()
self.assertFalse(s.is_defined())
s += [1, 2]
self.assertTrue(s.is_defined())
def test_value(self):
"""Single layer"""
s = Stage()
self.assertFalse(s.is_defined())
s += 1
self.assertTrue(s.is_defined())
def recipe(recipe_file, ctype=container_type.DOCKER, raise_exceptions=False,
single_stage=False, singularity_version='2.6', userarg=None,
working_directory='/var/tmp'):
"""Recipe builder
# Arguments
recipe_file: path to a recipe file (required).
ctype: Enum representing the container specification format. The
default is `container_type.DOCKER`.
raise_exceptions: If False, do not print stack traces when an
exception is raised. The default value is False.
single_stage: If True, only print the first stage of a multi-stage
recipe. The default is False.
singularity_version: Version of the Singularity definition file
format to use. Multi-stage support was added in version 3.2, but
the changes are incompatible with earlier versions of Singularity.
The default is '2.6'.
userarg: A dictionary of key / value pairs provided to the recipe
as the `USERARG` dictionary.
working_directory: path to use as the working directory in the
container specification
"""
# Make user arguments available
USERARG = {} # pylint: disable=unused-variable
if userarg:
USERARG = userarg # alias
# Consider just 2 stages for the time being
stages = [Stage(), Stage()]
Stage0 = stages[0] # alias # pylint: disable=unused-variable
Stage1 = stages[1] # alias # pylint: disable=unused-variable
# Set the global container type
hpccm.config.g_ctype = ctype
# Set the global Singularity version
hpccm.config.g_singularity_version = StrictVersion(singularity_version)
# Set the global working directory
hpccm.config.g_wd = working_directory
# Any included recipes that are specified using relative paths will
# need to prepend the path to the main recipe in order to be found.
# Save the path to the main recipe.
include.prepend_path = os.path.dirname(recipe_file)
# Load in the recipe file
include(recipe_file, _locals=locals(), _globals=globals(),
prepend_path=False, raise_exceptions=raise_exceptions)
# Only process the first stage of a recipe
if single_stage:
del stages[1:]
elif len(Stage1) > 0:
if (ctype == container_type.SINGULARITY and
hpccm.config.g_singularity_version < StrictVersion('3.2')):
# Singularity prior to version 3.2 did not support
# multi-stage builds. If the Singularity version is not
# sufficient to support multi-stage, provide advice to
# specify a sufficient Singularity version or disable
# multi-stage.
logging.warning('This looks like a multi-stage recipe. '
'Singularity 3.2 or later is required for '
'multi-stage builds. Use '
'--singularity-version=3.2 to enable this '
'feature or --single-stage to get rid of this '
'warning. Only processing the first stage...')
del stages[1:]
elif ctype == container_type.BASH:
logging.warning('This looks like a multi-stage recipe, but '
'bash does not support multi-stage builds. '
'Use --single-stage to get rid of this warning. '
'Only processing the first stage...')
del stages[1:]
r = []
for index, stage in enumerate(stages):
if index >= 1:
r.append('')
r.append(str(stage))
return '\n'.join(r)
def test_list(self):
"""List of layers"""
s = Stage()
self.assertEqual(len(s), 0)
s += [1, 2]
self.assertEqual(len(s), 2)
def test_value(self):
"""Single layer"""
s = Stage()
self.assertFalse(len(s))
s += 1
self.assertTrue(len(s))
def build(python_version='3.6', freesurfer_version='6.0.1'):
stage0 = Stage()
config.set_container_format('singularity')
stage0 += baseimage(image='nvidia/cuda:10.0-cudnn7-runtime-ubuntu16.04',
_bootstrap='docker')
stage0 += shell(
commands=['rm /bin/sh', 'ln -s /bin/bash /bin/sh', '/bin/bash'])
stage0 += packages(apt=[
'build-essential', 'cmake', 'git', 'vim', 'wget', 'ca-certificates',
'bzip2', 'libx11-6', 'libjpeg-dev', 'libpng-dev', 'bc', 'tar', 'zip',
'gawk', 'tcsh', 'time', 'libgomp1', 'libglu1-mesa', 'libglu1-mesa-dev',
'perl-modules'
])
stage0 += comment(
"https://github.com/Deep-MI/FastSurfer/blob/master/Docker/Dockerfile")
stage0 += shell(commands=[
f"wget -qO- https://surfer.nmr.mgh.harvard.edu/pub/dist/freesurfer/{freesurfer_version}/freesurfer-Linux-centos6_x86_64-stable-pub-v{freesurfer_version}.tar.gz | tar zxv --no-same-owner -C /opt \
--exclude='freesurfer/trctrain' \
--exclude='freesurfer/subjects/fsaverage_sym' \
--exclude='freesurfer/subjects/fsaverage3' \
--exclude='freesurfer/subjects/fsaverage4' \
--exclude='freesurfer/subjects/fsaverage5' \
--exclude='freesurfer/subjects/fsaverage6' \
--exclude='freesurfer/subjects/cvs_avg35' \
--exclude='freesurfer/subjects/cvs_avg35_inMNI152' \
--exclude='freesurfer/subjects/bert' \
--exclude='freesurfer/subjects/V1_average' \
--exclude='freesurfer/average/mult-comp-cor' \
--exclude='freesurfer/lib/cuda' \
--exclude='freesurfer/lib/qt'",
])
stage0 += conda(
channels=['pytorch', 'conda-forge'],
packages=[
f'python={python_version}', 'scipy', 'numpy', 'matplotlib', 'h5py',
'scikit-image', 'pytorch', 'cudatoolkit=10.0',
'pytorch=1.2.0=py3.6_cuda10.0.130_cudnn7.6.2_0',
'torchvision=0.4.0', 'scikit-sparse', 'nibabel=2.5.1',
'pillow=7.1.1'
],
eula=True,
)
stage0 += generic_build(
repository='https://github.com/Deep-MI/FastSurfer.git',
# branch='c5894bd',
build=['git checkout c5894bd'],
install=['mkdir /fastsurfer && mv * /fastsurfer/'])
stage0 += environment(
variables={
'OS': 'Linux',
'FS_OVERRIDE': 0,
'FIX_VERTEX_AREA': '',
'SUBJECTS_DIR': '/opt/freesurfer/subjects',
'FSF_OUTPUT_FORMAT': 'nii.gz',
'MNI_DIR': '/opt/freesurfer/mni',
'LOCAL_DIR': '/opt/freesurfer/local',
'FREESURFER_HOME': '/opt/freesurfer',
'FSFAST_HOME': '/opt/freesurfer/fsfast',
'PERL5LIB': '/opt/freesurfer/mni/lib/perl5/5.8.5',
'MNI_PERL4LIB': '/opt/freesurfer/mni/lib/perl5/5.8.5',
'PYTHONNUMBUFFERED': 0,
'PATH':
'/fastsurfer/recon_surf:/fastsurfer:/usr/local/anaconda/bin:/opt/freesurfer/bin:/opt/freesurfer/fsfast/bin:/opt/freesurfer/tktools:/opt/freesurfer/mni/bin:$PATH',
**from_prefix('/usr/local/cuda')
})
stage0 += runscript(commands=[
'cd /fastsurfer', 'source /usr/local/anaconda/etc/profile.d/conda.sh',
'source $FREESURFER_HOME/SetUpFreeSurfer.sh', "$*"
])
return stage0
